Commit 4ee8792a authored by Matija Obreza's avatar Matija Obreza

Merge branch '19-access-as-data-source' into 'master'

Resolve "Access as data source"

Closes #19

See merge request !15
parents f87b137c db7edf6f
......@@ -164,7 +164,7 @@
<dependency>
<groupId>com.healthmarketscience.jackcess</groupId>
<artifactId>jackcess</artifactId>
<version>2.1.4</version>
<version>2.1.6</version>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
......
......@@ -316,13 +316,6 @@ public class AppWindow {
threadPool.shutdown();
}
public static boolean isMac() {
if (System.getProperty("os.name").equals("Mac OS X")) {
return true;
}
return false;
}
/**
* Create contents of the window.
*/
......@@ -833,8 +826,8 @@ public class AppWindow {
public void addSourceFile() {
FileDialog fd = new FileDialog(shlGenesysMagic, SWT.MULTI);
fd.setFilterNames(new String[] { "Supported source files", "Excel files", "CSV files" });
fd.setFilterExtensions(new String[] { "*.xlsx;*.xls;*.csv;*.tab", "*.xlsx;*.xls", "*.csv;*.tab" });
fd.setFilterNames(new String[] { "Supported source files", "Excel files", "CSV files", "MS Access" });
fd.setFilterExtensions(new String[] { "*.xlsx;*.xls;*.csv;*.tab;*.accdb", "*.xlsx;*.xls", "*.csv;*.tab", "*.accdb" });
String response = fd.open();
if (response != null) {
......
......@@ -203,6 +203,7 @@ public class DataSourceSheet extends AbstractModelObject implements IDataSourceS
column.setRdfTerm(columnDef.getRdfTerm());
column.setMultiple(columnDef.hasAllowMultiple());
column.setDescription(columnDef.getTitle());
column.setIncludeNull(columnDef.hasIncludeNull());
}
}
}
......
......@@ -154,6 +154,7 @@ public class DwcaBuilder extends AbstractModelObject {
column.put("pattern", sourceColumn.getPattern());
column.put("separator", sourceColumn.getSeparator());
column.put("groupPattern", sourceColumn.getGroupPattern());
column.put("includeNull", sourceColumn.isIncludeNull());
columns.put(column);
}
}
......@@ -273,6 +274,8 @@ public class DwcaBuilder extends AbstractModelObject {
sourceColumn.setDataType(ColumnDataType.valueOf(column.getString("dataType")));
if (column.has("multiple"))
sourceColumn.setMultiple(column.getBoolean("multiple"));
if (column.has("includeNull"))
sourceColumn.setIncludeNull(column.getBoolean("includeNull"));
if (loadedColumns.contains(sourceColumn)) {
columnList.add(sourceColumn);
......
......@@ -28,11 +28,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
......@@ -66,6 +61,11 @@ import org.genesys2.client.oauth.PleaseRetryException;
import org.genesys2.client.oauth.api.accession.Api1Constants;
import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import swing2swt.layout.BorderLayout;
public class PushDialog extends Dialog {
......@@ -89,7 +89,7 @@ public class PushDialog extends Dialog {
protected DataSourceLoader dataSourceLoader;
BlockingQueue<Runnable> linkedBlockingDeque = new LinkedBlockingDeque<Runnable>(4);
private ThreadPoolExecutor executorService = new ThreadPoolExecutor(1, 2, 30, TimeUnit.SECONDS, linkedBlockingDeque, new ThreadPoolExecutor.CallerRunsPolicy());
private ThreadPoolExecutor executorService = new ThreadPoolExecutor(1, 4, 30, TimeUnit.SECONDS, linkedBlockingDeque, new ThreadPoolExecutor.CallerRunsPolicy());
protected Object result;
protected Shell shell;
......@@ -482,8 +482,6 @@ public class PushDialog extends Dialog {
if (_log.isDebugEnabled())
_log.debug(serverResponse);
_log.info("Push done.");
try {
JsonNode ri = objectMapper.readTree(serverResponse);
ri.forEach(new Consumer<JsonNode>() {
......@@ -497,6 +495,8 @@ public class PushDialog extends Dialog {
}
});
_log.info("Push done.");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
......
......@@ -30,6 +30,7 @@ import org.genesys2.anno.model.JdbcDrivers;
import org.genesys2.anno.model.OAuthSettings;
import org.genesys2.anno.model.Settings;
import org.genesys2.anno.parser.CsvDataSourceParser;
import org.genesys2.anno.parser.MsAccessDataSourceParser;
import org.genesys2.anno.parser.XlsxDataSourceParser;
import org.genesys2.anno.reader.JDBCRowReader;
import org.genesys2.anno.util.ConnectionUtils;
......@@ -65,6 +66,7 @@ public class SpringConfig {
DataSourceLoaderImpl dataSourceLoader = new DataSourceLoaderImpl();
dataSourceLoader.registerParser(new XlsxDataSourceParser());
dataSourceLoader.registerParser(new CsvDataSourceParser());
dataSourceLoader.registerParser(new MsAccessDataSourceParser());
return dataSourceLoader;
}
......
......@@ -28,17 +28,18 @@ public class Column extends AbstractModelObject {
private String separator = ";";
private String pattern;
private String groupPattern;
private boolean includeNull = false;
private boolean includeNull = true;
public Column() {
}
public Column(String preferredName, ColumnDataType dataType, boolean multiple, boolean unique, String rdfTerm) {
public Column(String preferredName, ColumnDataType dataType, boolean multiple, boolean unique, String rdfTerm, boolean includeNull) {
this.preferredName = preferredName;
this.dataType = dataType;
this.multiple = multiple;
this.unique = unique;
this.rdfTerm = rdfTerm;
this.includeNull = includeNull;
}
public String getPreferredName() {
......
......@@ -28,6 +28,7 @@ public class ColumnDef extends AbstractModelObject {
private Set<ColumnValidator> validators = new HashSet<ColumnValidator>();
private String title;
private boolean allowMultiple = false;
private boolean includeNull = true;
public ColumnDef() {
}
......@@ -96,4 +97,13 @@ public class ColumnDef extends AbstractModelObject {
this.allowMultiple = allowMultiple;
return this;
}
public boolean hasIncludeNull() {
return this.includeNull;
}
public ColumnDef setIncludeNull(boolean includeNull) {
this.includeNull = includeNull;
return this;
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.anno.parser;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import org.genesys2.anno.gui.DataSourceSheet;
import org.genesys2.anno.gui.IDataSourceSheet;
import org.genesys2.anno.gui.UnsupportedDataFormatException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.healthmarketscience.jackcess.Cursor;
import com.healthmarketscience.jackcess.Database;
import com.healthmarketscience.jackcess.DatabaseBuilder;
import com.healthmarketscience.jackcess.Row;
import com.healthmarketscience.jackcess.Table;
/**
* The MsAccess DataSourceParser.
*
* @author Matija Obreza
*/
public class MsAccessDataSourceParser extends BasicDataSourceParser {
private static final Logger _log = LoggerFactory.getLogger(MsAccessDataSourceParser.class);
private final String[] supportedExtensions = new String[] { ".accdb" };
@Override
public String[] getSupportedExtensions() {
return supportedExtensions;
}
@Override
public Collection<? extends DataSourceSheet> findSheets(File sourceFile) throws UnsupportedDataFormatException {
try {
try (Database db = DatabaseBuilder.open(sourceFile)) {
_log.debug("CHARSET: " + db.getCharset());
db.getTableNames().stream().forEach(tableName -> System.out.println("Table: " + tableName));
return db.getTableNames().stream().sorted().map(tableName -> {
DataSourceSheet sheetDataSource = new DataSourceSheet(sourceFile);
sheetDataSource.setSheetName(tableName);
_log.debug("Creating sheet for table {}", tableName);
return sheetDataSource;
}).collect(Collectors.toList());
}
} catch (Throwable e) {
throw new UnsupportedDataFormatException(e.getMessage(), e);
}
}
@Override
public List<Object[]> loadRows(IDataSourceSheet dataSourceSheet, int maxRows, int startAt) throws UnsupportedDataFormatException, FileNotFoundException, IOException {
File sourceFile = dataSourceSheet.getSourceFile();
String sheetName = dataSourceSheet.getSheetName();
_log.debug("Reading {} from {}", sheetName, sourceFile);
List<Object[]> rows = new ArrayList<Object[]>(maxRows);
try {
try (Database db = DatabaseBuilder.open(sourceFile)) {
_log.debug("CHARSET: " + db.getCharset());
Table table = db.getTable(sheetName);
rows.add(table.getColumns().stream().map(column -> column.getName()).toArray());
Cursor cursor = table.getDefaultCursor();
cursor.moveNextRows(startAt);
for (int pos = maxRows; pos > 0; pos--) {
Row r = cursor.getNextRow();
if (r != null) {
rows.add(r.values().toArray());
}
}
}
return rows;
} catch (Throwable e) {
throw new UnsupportedDataFormatException(e.getMessage(), e);
}
}
@Override
public RowReader createRowReader(IDataSourceSheet dataSourceSheet) throws UnsupportedDataFormatException, IOException {
File sourceFile = dataSourceSheet.getSourceFile();
String sheetName = dataSourceSheet.getSheetName();
return new MsAccessRowReader(sourceFile, sheetName);
}
}
/*
* Copyright 2018 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys2.anno.parser;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.healthmarketscience.jackcess.Cursor;
import com.healthmarketscience.jackcess.Database;
import com.healthmarketscience.jackcess.DatabaseBuilder;
import com.healthmarketscience.jackcess.Row;
import com.healthmarketscience.jackcess.Table;
/**
* The Ms Access RowReader.
*
* @author Matija Obreza
*/
public class MsAccessRowReader implements RowReader {
private static final Logger _log = LoggerFactory.getLogger(MsAccessRowReader.class);
private Database db;
private Table table;
private Cursor cursor;
/**
* Instantiates a new ms access row reader.
*
* @param sourceFile
* the source file
* @param sheetName
* the sheet name
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public MsAccessRowReader(File sourceFile, String sheetName) throws IOException {
this.db = DatabaseBuilder.open(sourceFile);
_log.debug("CHARSET: " + db.getCharset());
this.table = db.getTable(sheetName);
this.cursor = table.getDefaultCursor();
}
/// This is called in case data starts beyond 1st row
@Override
public void setSkipRows(int skipRows) {
_log.info("Skipping rows {}", skipRows);
cursor.reset();
// try {
// cursor.moveNextRows(skipRows);
// } catch (IOException e) {
// _log.error("Could not skip rows", e);
// }
}
@Override
public int getRowCount() {
return table.getRowCount();
}
@Override
public void close() throws IOException {
db.close();
}
@Override
public List<Object[]> readRows(int rowsToRead) throws IOException {
List<Object[]> rows = new ArrayList<Object[]>(rowsToRead);
int rowCount = 0;
while ((rowCount < rowsToRead) && cursor.moveToNextRow()) {
rowCount++;
Row r = cursor.getCurrentRow();
if (r != null) {
rows.add(r.values().toArray());
}
}
return rows;
}
}
......@@ -74,7 +74,7 @@ public class MCPD {
columnDef = new ColumnDef("SUBTAUTHOR", "Subtaxon authority", ColumnDataType.TEXT, false, RdfMCPD.SUBTAUTHOR);
columns.add(columnDef);
columnDef = new ColumnDef("CROPNAME", "Common crop name", ColumnDataType.TEXT, false, RdfMCPD.CROPNAME);
columnDef = new ColumnDef("CROPNAME", "Common crop name", ColumnDataType.TEXT, false, RdfMCPD.CROPNAME).setIncludeNull(false);
columns.add(columnDef);
columnDef = new ColumnDef("ACCENAME", "Accession name", ColumnDataType.TEXT, false, RdfMCPD.ACCENAME).setAllowMultiple(true);
......@@ -158,20 +158,20 @@ public class MCPD {
columnDef = new ColumnDef("STORAGE", "Type of germplasm storage", ColumnDataType.TEXT, false, RdfMCPD.STORAGE).setAllowMultiple(true);
columns.add(columnDef);
columnDef = new ColumnDef("MLSSTAT", "MLS status of the accession", ColumnDataType.BOOLEAN, false, RdfMCPD.MLSSTAT);
columnDef = new ColumnDef("MLSSTAT", "MLS status of the accession", ColumnDataType.BOOLEAN, false, RdfMCPD.MLSSTAT).setIncludeNull(false);
columns.add(columnDef);
columnDef = new ColumnDef("REMARKS", "Remarks", ColumnDataType.TEXT, false, RdfMCPD.REMARKS).setAllowMultiple(true);
columns.add(columnDef);
// Genesys specific
columnDef = new ColumnDef("INTRUST", "FAO in Trust", ColumnDataType.BOOLEAN, false, RdfMCPD.FAOINTRUST);
columnDef = new ColumnDef("INTRUST", "FAO in Trust", ColumnDataType.BOOLEAN, false, RdfMCPD.FAOINTRUST).setIncludeNull(false);
columns.add(columnDef);
columnDef = new ColumnDef("AVAILABLE", "Accession is available for distribution", ColumnDataType.BOOLEAN, false, RdfMCPD.AVAILABLE);
columnDef = new ColumnDef("AVAILABLE", "Accession is available for distribution", ColumnDataType.BOOLEAN, false, RdfMCPD.AVAILABLE).setIncludeNull(false);
columns.add(columnDef);
columnDef = new ColumnDef("HISTORIC", "Historic entry", ColumnDataType.BOOLEAN, false, RdfMCPD.HISTORIC);
columnDef = new ColumnDef("HISTORIC", "Historic entry", ColumnDataType.BOOLEAN, false, RdfMCPD.HISTORIC).setIncludeNull(false);
columns.add(columnDef);
columnDef = new ColumnDef("ACCEURL", "Accession URL", ColumnDataType.TEXT, false, RdfMCPD.ACCEURL);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment