Commit 7c8c22a5 authored by Matija Obreza's avatar Matija Obreza
Browse files

Use new schema for method tables, recalculate metadatamethods and

accessiontraits from existing data.
parent ac9e512e
......@@ -34,4 +34,6 @@ public interface GenesysLowlevelRepository {
void updateFaoInstituteCountries();
void ensureMethodTable(Method method);
void refreshMetadataMethods();
}
......@@ -19,6 +19,7 @@ package org.genesys2.server.persistence.domain;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import javax.sql.DataSource;
......@@ -26,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.genesys2.server.model.genesys.Method;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.BadSqlGrammarException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Repository;
......@@ -62,7 +64,8 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
public void updateInstituteAccessionCount(long instituteId) {
LOG.info("Updating institute accession count for instituteId=" + instituteId);
int count = jdbcTemplate
.update("UPDATE faoinstitute FI inner join (select instituteId, count(*) X from accession where instituteId= ? group by instituteId) A on A.instituteId=FI.id set FI.accessionCount=A.X where FI.id = ?;", instituteId, instituteId);
.update("UPDATE faoinstitute FI inner join (select instituteId, count(*) X from accession where instituteId= ? group by instituteId) A on A.instituteId=FI.id set FI.accessionCount=A.X where FI.id = ?;",
instituteId, instituteId);
LOG.info("Institute accession counts updated: " + count);
}
......@@ -107,7 +110,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
// Common table elements
createTableSql.append("CREATE TABLE `").append(method.getId())
.append("` (`id` BIGINT NOT NULL AUTO_INCREMENT, `ALIS_Id` BIGINT NOT NULL, `Meta_id` INT NOT NULL, ");
.append("` (`id` BIGINT NOT NULL AUTO_INCREMENT, `accessionId` BIGINT NOT NULL, `metadataId` INT NOT NULL, ");
// Value column
createTableSql.append("`").append(method.getFieldName()).append("` ");
......@@ -115,12 +118,21 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
createTableSql.append(" NOT NULL");
// PK
createTableSql.append(", PRIMARY KEY (`id`), KEY `ALIS_Id` (`ALIS_Id`)");
createTableSql.append(", PRIMARY KEY (`id`)");
// FK metadata
// createTableSql.append(", INDEX `genesys_Metadata_").append(method.getId()).append("` (`Meta_Id` ASC)");
createTableSql.append(", CONSTRAINT `genesys_Metadata_").append(method.getId())
.append("` FOREIGN KEY (`methodId`) REFERENCES `metadata` (`Meta_id` ) ON DELETE NO ACTION ON UPDATE NO ACTION");
// FK accession
// createTableSql.append(", INDEX `genesys_Accession_idx` (`ALIS_Id` ASC)");
createTableSql.append(", CONSTRAINT `genesys_Accession_").append(method.getId())
.append("` FOREIGN KEY (`accessionId`) REFERENCES `accession` (`id` ) ON DELETE NO ACTION ON UPDATE NO ACTION");
// This is optional, but let's keep it
createTableSql.append(", KEY `").append(method.getFieldName()).append("` (`").append(method.getFieldName()).append("`)");
// createTableSql.append(", KEY `").append(method.getFieldName()).append("` (`").append(method.getFieldName()).append("`)");
createTableSql.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
LOG.warn("Executing: " + createTableSql.toString());
LOG.warn("Creating method table: " + createTableSql.toString());
jdbcTemplate.execute(createTableSql.toString());
}
LOG.info("Done");
......@@ -139,4 +151,51 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
}
throw new RuntimeException("Unsupported data type " + method.getFieldType());
}
@Override
@Transactional(readOnly = false)
public void refreshMetadataMethods() {
jdbcTemplate.update("delete from metadatamethod");
jdbcTemplate.update("delete from accessiontrait");
List<Long> methodIds = jdbcTemplate.queryForList("select id from methods", Long.class);
for (long methodId : methodIds) {
LOG.info("Updating metadatamethod for methodId=" + methodId);
// // Delete stuff without Metadata
// int deleted = jdbcTemplate.update("delete from `" + methodId +
// "` where Meta_Id in (select X.Meta_Id from (select distinct Meta_Id from `"
// + methodId +
// "`) X left outer join metadata M on M.Meta_Id=X.Meta_Id where M.Meta_Id is null)");
// if (deleted > 0) {
// LOG.warn("Deleted " + deleted +
// " observations with missing metadata from method=" + methodId);
// }
// Insert reference to metadatamethod
try {
int inserted = jdbcTemplate.update("insert into metadatamethod (metadataId, methodId) (select distinct metadataId, " + methodId + " from `"
+ methodId + "`)");
if (inserted == 0) {
LOG.warn("Method " + methodId + " has no entries?");
} else {
LOG.warn("Inserted " + inserted + " method references for metadata for methodId=" + methodId);
}
} catch (BadSqlGrammarException e) {
LOG.warn(e);
}
try {
int inserted2 = jdbcTemplate.update("insert into accessiontrait (metadataId, accessionId, methodId) (select distinct metadataId, accessionId, "
+ methodId + " from `" + methodId + "`)");
if (inserted2 == 0) {
LOG.warn("Method " + methodId + " has no entries?");
} else {
LOG.warn("Inserted " + inserted2 + " accession references methodId=" + methodId);
}
} catch (BadSqlGrammarException e) {
LOG.warn(e);
}
}
}
}
......@@ -101,8 +101,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
return this.jdbcTemplate.query(new PreparedStatementCreator() {
@Override
public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
PreparedStatement pst = conn.prepareStatement("select `ALIS_Id` as accessionId, `" + method.getFieldName() + "` as traitvalue from `"
+ method.getId() + "` where `Meta_Id`=?");
PreparedStatement pst = conn.prepareStatement("select accessionId, `" + method.getFieldName() + "` as traitvalue from `" + method.getId()
+ "` where metadataId=?");
pst.setLong(1, metadataId);
return pst;
}
......@@ -126,8 +126,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
for (AccessionTrait at : accessionTraits) {
Method method = methodRepository.findOne(at.getMethodId());
final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select Meta_id as metadataId, `" + method.getFieldName()
+ "` as traitvalue from `" + method.getId() + "` where `ALIS_Id`=?", new Object[] { at.getAccession().getId() });
final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select metadataId, `" + method.getFieldName() + "` as traitvalue from `"
+ method.getId() + "` where accessionId=?", new Object[] { at.getAccession().getId() });
if (rows == null || rows.size() == 0) {
// Skip
......@@ -137,14 +137,15 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
final List<ExperimentTrait> ret = new ArrayList<ExperimentTrait>();
for (final Map<String, Object> row : rows) {
final ExperimentTrait et = new ExperimentTrait((Integer) row.get("metadataId"), row.get("traitvalue"));
final ExperimentTrait et = new ExperimentTrait((Long) row.get("metadataId"), row.get("traitvalue"));
ret.add(et);
}
methodValues.put(method.getId(), ret);
}
LOG.warn("Done! " + methodValues);
if (LOG.isDebugEnabled())
LOG.debug("Done! " + methodValues);
return methodValues;
}
......@@ -162,8 +163,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
// FIXME We're loading all entries for metadata, should limit only
// to the accessions in the list!
final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select `ALIS_Id` as accessionId, `" + method.getFieldName()
+ "` as traitvalue from `" + method.getId() + "` where `Meta_Id`=?", new Object[] { metadata.getId() });
final List<Map<String, Object>> rows = this.jdbcTemplate.queryForList("select accessionId, `" + method.getFieldName() + "` as traitvalue from `"
+ method.getId() + "` where metadataId=?", new Object[] { metadata.getId() });
if (rows == null || rows.size() == 0) {
// Skip
......@@ -187,7 +188,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
}
}
LOG.warn("Done! ");
if (LOG.isDebugEnabled())
LOG.debug("Done! ");
return accessionMetadataValues;
}
......@@ -195,7 +197,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
@Transactional
public void delete(final Metadata metadata, final Accession accession, Method method) {
LOG.info("Deleting trait values in methodId=" + method.getId() + " for metadataId=" + metadata.getId() + " and accnId=" + accession.getId());
this.jdbcTemplate.update(String.format("DELETE FROM `%s` WHERE Alis_Id=? AND Meta_id=?", method.getId()), new PreparedStatementSetter() {
this.jdbcTemplate.update(String.format("DELETE FROM `%s` WHERE accessionId=? AND metadataId=?", method.getId()), new PreparedStatementSetter() {
@Override
public void setValues(PreparedStatement pst) throws SQLException {
pst.setLong(1, accession.getId());
......@@ -221,7 +223,8 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
}
}
this.jdbcTemplate.batchUpdate(String.format("INSERT INTO `%s` (Alis_Id, Meta_id, `%s`) VALUES (?, ?, ?)", method.getId(), method.getFieldName()),
this.jdbcTemplate.batchUpdate(
String.format("INSERT INTO `%s` (accessionId, metadataId, `%s`) VALUES (?, ?, ?)", method.getId(), method.getFieldName()),
new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement pst, int idx) throws SQLException {
......@@ -340,7 +343,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
* @return
*/
private int countTraitValues(long methodId, long metadataId) {
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE Meta_Id=?", methodId), Integer.class, metadataId);
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE metadataId=?", methodId), Integer.class, metadataId);
}
/**
......@@ -352,7 +355,7 @@ public class TraitValueRepositoryImpl implements TraitValueRepository {
* @return
*/
private int countTraitValues(long methodId, long metadataId, long accessionId) {
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE Meta_Id=? AND ALIS_Id=?", methodId), Integer.class, metadataId,
accessionId);
return this.jdbcTemplate.queryForObject(String.format("SELECT COUNT(*) FROM `%s` WHERE metadataId=? AND accessionId=?", methodId), Integer.class,
metadataId, accessionId);
}
}
......@@ -129,4 +129,6 @@ public interface GenesysService {
void saveGeo(List<AccessionGeo> all);
void refreshMetadataMethods();
}
......@@ -721,4 +721,11 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
zos.finish();
}
@Override
@Transactional
@PreAuthorize("hasRole('ADMINISTRATOR')")
public void refreshMetadataMethods() {
genesysLowlevelRepository.refreshMetadataMethods();
}
}
......@@ -114,6 +114,12 @@ public class AdminController {
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/refreshMetadataMethods")
public String refreshMetadataMethods() {
genesysService.refreshMetadataMethods();
return "redirect:/admin/";
}
@RequestMapping(method = RequestMethod.POST, value = "/updateInstituteCountryRefs")
public String updateInstituteCountryRefs() {
instituteService.updateCountryRefs();
......
......@@ -49,6 +49,11 @@
<input type="submit" class="btn btn-default" value="Update accession institute info" />
</form>
<h3>C&E</h3>
<form method="post" action="<c:url value="/admin/refreshMetadataMethods" />">
<input type="submit" class="btn btn-default" class="btn btn-default" value="Recalculate metadata methods" />
</form>
<h3>Content</h3>
<form method="post" action="<c:url value="/admin/sanitize" />">
<input type="submit" class="btn btn-default" value="Sanitize HTML content" />
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment