Commit 11620289 authored by Matija Obreza's avatar Matija Obreza
Browse files

Export more data as DwCA (missing termUrls!)

parent 697bda50
......@@ -39,4 +39,8 @@ public interface GenesysLowlevelRepository {
void refreshMetadataMethods();
void listAccessions(String instCode, RowCallbackHandler rowCallbackHandler);
void listAccessionsGeo(String code, RowCallbackHandler rowCallbackHandler);
void listAccessionsColl(String code, RowCallbackHandler rowCallbackHandler);
}
......@@ -161,7 +161,7 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
public void refreshMetadataMethods() {
jdbcTemplate.update("delete from metadatamethod");
jdbcTemplate.update("delete from accessiontrait");
List<Long> methodIds = jdbcTemplate.queryForList("select id from method", Long.class);
for (long methodId : methodIds) {
LOG.info("Updating metadatamethod for methodId=" + methodId);
......@@ -202,13 +202,45 @@ public class GenesysLowlevelRepositoryImpl implements GenesysLowlevelRepository
}
}
}
@Override
public void listAccessions(final String instCode, RowCallbackHandler rowCallbackHandler) {
jdbcTemplate.query(new PreparedStatementCreator() {
@Override
public PreparedStatement createPreparedStatement(Connection con) throws SQLException {
PreparedStatement stmt = con.prepareStatement("select a.id, a.uuid, a.instCode, a.acceNumb, a.genus, t.species, a.createdBy, a.createdDate, a.lastModifiedBy, a.lastModifiedDate from accession a left outer join taxonomy t on t.id=a.taxonomyId where a.instCode = ?");
PreparedStatement stmt = con
.prepareStatement("select a.id, a.uuid, a.instCode, a.acceNumb, a.genus, t.species, a.orgCty, a.acqSrc, a.acqDate, a.mlsStat, a.available, a.storage, a.sampStat, a.dublInst, a.createdBy, a.createdDate, a.lastModifiedBy, a.lastModifiedDate from accession a left outer join taxonomy t on t.id=a.taxonomyId where a.instCode = ?");
stmt.setString(1, instCode);
// Set mysql JConnector to stream results
stmt.setFetchSize(Integer.MIN_VALUE);
return stmt;
}
}, rowCallbackHandler);
}
@Override
public void listAccessionsGeo(final String instCode, RowCallbackHandler rowCallbackHandler) {
jdbcTemplate.query(new PreparedStatementCreator() {
@Override
public PreparedStatement createPreparedStatement(Connection con) throws SQLException {
PreparedStatement stmt = con
.prepareStatement("select a.id, g.latitude, g.longitude, g.altitude, g.version from accessiongeo g inner join accession a on a.id=g.accessionId where a.instCode = ?");
stmt.setString(1, instCode);
// Set mysql JConnector to stream results
stmt.setFetchSize(Integer.MIN_VALUE);
return stmt;
}
}, rowCallbackHandler);
}
@Override
public void listAccessionsColl(final String instCode, RowCallbackHandler rowCallbackHandler) {
jdbcTemplate.query(new PreparedStatementCreator() {
@Override
public PreparedStatement createPreparedStatement(Connection con) throws SQLException {
PreparedStatement stmt = con
.prepareStatement("select a.id, coll.instCode, coll.collNumb, coll.date, coll.site, coll.version from accessioncollect coll inner join accession a on a.id=coll.accessionId where a.instCode = ?");
stmt.setString(1, instCode);
// Set mysql JConnector to stream results
stmt.setFetchSize(Integer.MIN_VALUE);
......
......@@ -92,6 +92,8 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import au.com.bytecode.opencsv.CSVWriter;
import au.com.bytecode.opencsv.ResultSetHelper;
import au.com.bytecode.opencsv.ResultSetHelperService;
@Service
@Transactional(readOnly = true)
......@@ -150,7 +152,6 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
@Autowired
private AccessionAliasRepository accessionAliasRepository;
@Override
public long countByInstitute(FaoInstitute institute) {
......@@ -262,7 +263,7 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
public AllAccnames listAccessionNames(Accession accession) {
return accessionNamesRepository.findByAccession(accession);
}
@Override
public List<AccessionAlias> listAccessionAliases(Accession accession) {
return accessionAliasRepository.findByAccession(accession);
......@@ -642,43 +643,34 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
}
@Override
// TODO FIXME Need proper term URLs
public void writeAccessions(final FaoInstitute faoInstitute, OutputStream outputStream) throws IOException {
// UTF8 is used for encoding entry names
ZipOutputStream zos = new ZipOutputStream(outputStream);
zos.setComment("Genesys Accessions instCode=" + faoInstitute.getCode());
// Accessions
ZipEntry coreEntry = new ZipEntry("core.csv");
coreEntry.setComment("Accession information");
coreEntry.setTime(System.currentTimeMillis());
zos.putNextEntry(coreEntry);
writeAccessionsCore(faoInstitute, zos);
zos.closeEntry();
// Accessions
@SuppressWarnings("resource")
final CSVWriter csv = new CSVWriter(new BufferedWriter(new OutputStreamWriter(zos)), ',', '"', '\\', "\n");
csv.writeNext(new String[] { "Genesys ID", "UUID", "instCode", "acceNumb", "genus", "species", "createdBy", "createdDate", "lastModifiedBy",
"lastModifiedDate" });
// Write accession information
genesysLowlevelRepository.listAccessions(faoInstitute.getCode(), new RowCallbackHandler() {
int i = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
String createdBy = rs.getString(7);
createdBy = rs.wasNull() ? "" : createdBy;
String lastModifiedBy = rs.getString(9);
lastModifiedBy = rs.wasNull() ? "" : lastModifiedBy;
csv.writeNext(new String[] { String.valueOf(rs.getLong(1)), rs.getString(2), rs.getString(3), rs.getString(4), rs.getString(5),
rs.getString(6), createdBy, "" + rs.getTimestamp(8), lastModifiedBy, "" + rs.getTimestamp(10) });
if ((i++ % 5000) == 0) {
LOG.info("Writing accessions DWCA " + faoInstitute.getCode() + " " + i);
}
}
});
// AccessionGeo
ZipEntry entry = new ZipEntry("geo.csv");
entry.setComment("GIS information");
entry.setTime(System.currentTimeMillis());
zos.putNextEntry(entry);
writeAccessionsGeo(faoInstitute, zos);
zos.closeEntry();
csv.flush();
// AccessionCollect
entry = new ZipEntry("coll.csv");
entry.setComment("Collecting information");
entry.setTime(System.currentTimeMillis());
zos.putNextEntry(entry);
writeAccessionsColl(faoInstitute, zos);
zos.closeEntry();
ZipEntry metaEntry = new ZipEntry("meta.xml");
......@@ -696,11 +688,43 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
osw.write("\t<field index=\"3\" term=\"http://purl.org/germplasm/germplasmTerm#germplasmIdentifier\"/>\n");
osw.write("\t<field index=\"4\" term=\"http://rs.tdwg.org/dwc/terms/genus\"/>\n");
osw.write("\t<field index=\"5\" term=\"http://rs.tdwg.org/dwc/terms/species\"/>\n");
osw.write("\t<field index=\"6\" term=\"createdBy\"/>\n");
osw.write("\t<field index=\"7\" term=\"createdDate\"/>\n");
osw.write("\t<field index=\"8\" term=\"lastModifiedBy\"/>\n");
osw.write("\t<field index=\"9\" term=\"lastModifiedDate\"/>\n");
osw.write("\t<field index=\"6\" term=\"orgCty\"/>\n");
osw.write("\t<field index=\"7\" term=\"acqSrc\"/>\n");
osw.write("\t<field index=\"8\" term=\"acqDate\"/>\n");
osw.write("\t<field index=\"9\" term=\"mlsStat\"/>\n");
osw.write("\t<field index=\"10\" term=\"available\"/>\n");
osw.write("\t<field index=\"11\" term=\"storage\"/>\n");
osw.write("\t<field index=\"12\" term=\"sampStat\"/>\n");
osw.write("\t<field index=\"13\" term=\"dublInst\"/>\n");
osw.write("\t<field index=\"14\" term=\"createdBy\"/>\n");
osw.write("\t<field index=\"15\" term=\"createdDate\"/>\n");
osw.write("\t<field index=\"16\" term=\"lastModifiedBy\"/>\n");
osw.write("\t<field index=\"17\" term=\"lastModifiedDate\"/>\n");
osw.write("</core>\n");
osw.write("<extension encoding=\"UTF-8\" fieldsTerminatedBy=\",\" linesTerminatedBy=\"\\n\" fieldsEnclosedBy=\"&quot;\" ignoreHeaderLines=\"0\">\n");
osw.write("\t<files><location>");
osw.write("geo.csv");
osw.write("</location></files>\n");
osw.write("\t<coreid index=\"0\" />\n");
osw.write("\t<field index=\"1\" term=\"latitude\"/>\n");
osw.write("\t<field index=\"2\" term=\"longitude\"/>\n");
osw.write("\t<field index=\"3\" term=\"altitude\"/>\n");
osw.write("\t<field index=\"4\" term=\"version\"/>\n");
osw.write("</extension>\n");
osw.write("<extension encoding=\"UTF-8\" fieldsTerminatedBy=\",\" linesTerminatedBy=\"\\n\" fieldsEnclosedBy=\"&quot;\" ignoreHeaderLines=\"0\">\n");
osw.write("\t<files><location>");
osw.write("coll.csv");
osw.write("</location></files>\n");
osw.write("\t<coreid index=\"0\" />\n");
osw.write("\t<field index=\"1\" term=\"collCode\"/>\n");
osw.write("\t<field index=\"2\" term=\"collNumb\"/>\n");
osw.write("\t<field index=\"3\" term=\"collDate\"/>\n");
osw.write("\t<field index=\"4\" term=\"collSite\"/>\n");
osw.write("\t<field index=\"5\" term=\"version\"/>\n");
osw.write("</extension>\n");
osw.write("</archive>\n");
osw.flush();
zos.closeEntry();
......@@ -708,6 +732,90 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
zos.finish();
}
private void writeAccessionsCore(final FaoInstitute faoInstitute, ZipOutputStream zos) throws IOException {
@SuppressWarnings("resource")
final CSVWriter csv = new CSVWriter(new BufferedWriter(new OutputStreamWriter(zos)), ',', '"', '\\', "\n");
csv.writeNext(new String[] { "genesysId", "uuid", "instCode", "acceNumb", "genus", "species", "orgCty", "acqSrc", "acqDate", "mlsStat", "available",
"storage", "sampStat", "dublInst", "createdBy", "createdDate", "lastModifiedBy", "lastModifiedDate" });
final ResultSetHelper csvResultsetHelper = new ResultSetHelperService();
// Write accession information
genesysLowlevelRepository.listAccessions(faoInstitute.getCode(), new RowCallbackHandler() {
int i = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
try {
csv.writeNext(csvResultsetHelper.getColumnValues(rs));
} catch (IOException e) {
LOG.error("Reading resultset for DWCA", e);
}
if ((i++ % 5000) == 0) {
LOG.info("Writing accessions DWCA " + faoInstitute.getCode() + " " + i);
}
}
});
csv.flush();
}
private void writeAccessionsGeo(final FaoInstitute faoInstitute, ZipOutputStream zos) throws IOException {
@SuppressWarnings("resource")
final CSVWriter csv = new CSVWriter(new BufferedWriter(new OutputStreamWriter(zos)), ',', '"', '\\', "\n");
csv.writeNext(new String[] { "genesysId", "latitude", "longitude", "altitude", "version" });
final ResultSetHelper csvResultsetHelper = new ResultSetHelperService();
// Write accession information
genesysLowlevelRepository.listAccessionsGeo(faoInstitute.getCode(), new RowCallbackHandler() {
int i = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
try {
csv.writeNext(csvResultsetHelper.getColumnValues(rs));
} catch (IOException e) {
LOG.error("Reading resultset for DWCA", e);
}
if ((i++ % 5000) == 0) {
LOG.info("Writing geo DWCA " + faoInstitute.getCode() + " " + i);
}
}
});
csv.flush();
}
private void writeAccessionsColl(final FaoInstitute faoInstitute, ZipOutputStream zos) throws IOException {
@SuppressWarnings("resource")
final CSVWriter csv = new CSVWriter(new BufferedWriter(new OutputStreamWriter(zos)), ',', '"', '\\', "\n");
csv.writeNext(new String[] { "genesysId", "collCode", "collNumb", "collDate", "collSite", "version" });
final ResultSetHelper csvResultsetHelper = new ResultSetHelperService();
// Write accession information
genesysLowlevelRepository.listAccessionsColl(faoInstitute.getCode(), new RowCallbackHandler() {
int i = 0;
@Override
public void processRow(ResultSet rs) throws SQLException {
try {
csv.writeNext(csvResultsetHelper.getColumnValues(rs));
} catch (IOException e) {
LOG.error("Reading resultset for DWCA", e);
}
if ((i++ % 5000) == 0) {
LOG.info("Writing collecting DWCA " + faoInstitute.getCode() + " " + i);
}
}
});
csv.flush();
}
@Override
public void writeDataset(Metadata metadata, OutputStream outputStream) throws IOException {
// Methods: A CSV is generated for each method
......@@ -811,16 +919,16 @@ public class GenesysServiceImpl implements GenesysService, TraitService, Dataset
public void refreshMetadataMethods() {
genesysLowlevelRepository.refreshMetadataMethods();
}
@Override
@Transactional
public void saveAliases(List<AccessionAlias> aliases) {
accessionAliasRepository.save(aliases);
}
@Override
@Transactional
public void removeAliases(List<AccessionAlias> aliases) {
accessionAliasRepository.delete(aliases);
accessionAliasRepository.delete(aliases);
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment