Commit 08b23a64 authored by Matija Obreza's avatar Matija Obreza
Browse files

Enhanced Excel metadata

- Using appropriate data types for rows
- Temp file
parent 3865ff81
......@@ -15,8 +15,8 @@
*/
package org.genesys.catalog.service;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import com.fasterxml.jackson.databind.node.ObjectNode;
......@@ -27,11 +27,11 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
*/
public interface MetadataService {
/**
* generate JSON with Excel metadata
*
* @param inputStream
* @return generated JSON object with Excel metadata
*/
ObjectNode getExcelMetadata(InputStream inputStream) throws IOException;
/**
* generate JSON with Excel metadata
*
* @param file
* @return generated JSON object with Excel metadata
*/
ObjectNode getExcelMetadata(File file) throws IOException;
}
......@@ -17,17 +17,13 @@ package org.genesys.catalog.service.impl;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.io.FileUtils;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
......@@ -37,6 +33,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* The Class MetadataServiceImpl.
*
......@@ -45,87 +46,105 @@ import org.springframework.stereotype.Service;
@Service
public class MetadataServiceImpl implements MetadataService {
/**
* The Constant LOG.
*/
private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceImpl.class);
/**
* The Constant ROWS_TO_READ.
*/
private static final int ROWS_TO_READ = 10;
/**
* {@inheritDoc}
*/
@Override
public ObjectNode getExcelMetadata(final InputStream inputStream) throws IOException {
final ObjectMapper mapper = new ObjectMapper();
final ObjectNode mainNode = mapper.createObjectNode();
final File tempFile = new File("temp");
try (final Workbook workbook = new XSSFWorkbook(tempFile)) {
FileUtils.copyInputStreamToFile(inputStream, tempFile);
final ArrayNode sheetsArrayNode = mapper.createArrayNode();
final int numberOfSheets = workbook.getNumberOfSheets();
for (int i = 0; i < numberOfSheets; i++) {
final Sheet datatypeSheet = workbook.getSheetAt(i);
final Iterator<Row> iterator = datatypeSheet.iterator();
final ArrayNode headersNode = mapper.createArrayNode();
final ArrayNode rowsNode = mapper.createArrayNode();
while (iterator.hasNext()) {
final Row currentRow = iterator.next();
final Iterator<Cell> cellIterator = currentRow.iterator();
final List<String> values = cellsValues(cellIterator);
if (currentRow.getRowNum() == 0) {
for (final String header : values) {
headersNode.add(header);
}
}
if (currentRow.getRowNum() < ROWS_TO_READ) {
final ArrayNode cellNode = mapper.createArrayNode();
for (final String cellValue : values) {
cellNode.add(cellValue);
}
rowsNode.add(cellNode);
}
}
final ObjectNode sheetNode = mapper.createObjectNode();
sheetNode.put("name", datatypeSheet.getSheetName());
sheetNode.set("headers", headersNode);
sheetNode.set("rows", rowsNode);
sheetNode.put("rowCount", (datatypeSheet.getLastRowNum() + 1));
sheetsArrayNode.add(sheetNode);
}
// leave metadata blank temporarily
mainNode.set("metadata", mapper.createObjectNode());
mainNode.set("sheets", sheetsArrayNode);
} catch (final IOException | InvalidFormatException e) {
LOG.error("Error writing metadata. {}", e.getMessage());
throw new IOException("Error writing metadata", e);
} finally {
FileUtils.deleteQuietly(tempFile);
}
return mainNode;
}
/**
* Get cells values from row
*
* @param cellIterator the cellIterator
* @return the list of cell values from row
*/
private List<String> cellsValues(final Iterator<Cell> cellIterator) {
final List<String> list = new ArrayList<>();
while (cellIterator.hasNext()) {
final Cell currentCell = cellIterator.next();
list.add(currentCell.toString());
}
return list;
}
/**
* The Constant LOG.
*/
private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceImpl.class);
/**
* The Constant ROWS_TO_READ.
*/
private static final int ROWS_TO_READ = 10;
private final ObjectMapper mapper = new ObjectMapper();
/**
* {@inheritDoc}
*/
@Override
public ObjectNode getExcelMetadata(final File excelFile) throws IOException {
final ObjectNode mainNode = mapper.createObjectNode();
try (final Workbook workbook = new XSSFWorkbook(excelFile)) {
final ArrayNode sheetsArrayNode = mapper.createArrayNode();
final int numberOfSheets = workbook.getNumberOfSheets();
for (int i = 0; i < numberOfSheets; i++) {
final Sheet datatypeSheet = workbook.getSheetAt(i);
final Iterator<Row> iterator = datatypeSheet.iterator();
JsonNode headersNode = mapper.createArrayNode();
final ArrayNode rowsNode = mapper.createArrayNode();
while (iterator.hasNext()) {
final Row currentRow = iterator.next();
final Iterator<Cell> cellIterator = currentRow.iterator();
final List<Object> values = cellsValues(cellIterator);
if (currentRow.getRowNum() == 0) {
headersNode = mapper.convertValue(values, JsonNode.class);
}
if (currentRow.getRowNum() < ROWS_TO_READ) {
rowsNode.add(mapper.convertValue(values, JsonNode.class));
} else {
// We reached the rows limit
break;
}
}
final ObjectNode sheetNode = mapper.createObjectNode();
sheetNode.put("name", datatypeSheet.getSheetName());
sheetNode.set("headers", headersNode);
sheetNode.set("rows", rowsNode);
sheetNode.put("rowCount", (datatypeSheet.getLastRowNum() + 1));
sheetsArrayNode.add(sheetNode);
}
// leave metadata blank temporarily
mainNode.set("metadata", mapper.createObjectNode());
mainNode.set("sheets", sheetsArrayNode);
} catch (final IOException | InvalidFormatException e) {
LOG.error("Error writing metadata. {}", e.getMessage());
throw new IOException("Error writing metadata", e);
}
return mainNode;
}
/**
* Get cells values from row
*
* @param cellIterator the cellIterator
* @return the list of cell values from row
*/
private List<Object> cellsValues(final Iterator<Cell> cellIterator) {
final List<Object> list = new ArrayList<>();
while (cellIterator.hasNext()) {
final Cell currentCell = cellIterator.next();
switch (currentCell.getCellTypeEnum()) {
case BLANK:
list.add(null);
break;
case ERROR:
list.add("#ERROR");
break;
case STRING:
list.add(currentCell.getStringCellValue());
break;
case BOOLEAN:
list.add(currentCell.getBooleanCellValue());
break;
case NUMERIC:
if (DateUtil.isCellDateFormatted(currentCell)) {
list.add(currentCell.getDateCellValue());
} else {
list.add(currentCell.getNumericCellValue());
}
break;
default:
list.add(currentCell.toString());
}
}
return list;
}
}
......@@ -15,20 +15,24 @@
*/
package org.genesys.catalog.server.controller.api.v0;
import java.io.File;
import java.io.IOException;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.io.FileUtils;
import org.genesys.catalog.service.MetadataService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* The Class MetadataController.
*
......@@ -39,24 +43,44 @@ import org.springframework.web.multipart.MultipartFile;
@PreAuthorize("isAuthenticated()")
public class MetadataController {
/**
* The Constant API_BASE.
*/
protected static final String API_BASE = "/api/v0/metadata";
/**
* The Constant API_BASE.
*/
protected static final String API_BASE = "/api/v0/metadata";
/**
* The Constant LOG.
*/
private static final Logger LOG = LoggerFactory.getLogger(MetadataController.class);
private static final String APPLICATION_OXML_SHEET = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String APPLICATION_MS_EXCEL = "application/vnd.ms-excel";
@Autowired
private MetadataService metadataService;
@PostMapping(consumes = { "multipart/form-data" })
public ObjectNode metadata(@RequestParam("file") final MultipartFile file) throws IOException {
/**
* The Constant LOG.
*/
private static final Logger LOG = LoggerFactory.getLogger(MetadataController.class);
// TODO store the file to the repository
@Autowired
private MetadataService metadataService;
if (APPLICATION_MS_EXCEL.equals(file.getContentType()) || APPLICATION_OXML_SHEET.equals(file.getContentType())) {
return excelMetadata(file);
} else {
throw new UnsupportedOperationException(file.getContentType());
}
}
@PostMapping(consumes = {"application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"})
public ObjectNode excelFileUpload(@RequestParam("file") final MultipartFile file) throws IOException {
LOG.info("Uploading Excel file {}", file.getOriginalFilename());
//TODO store the file to the repository
@PostMapping(consumes = { APPLICATION_MS_EXCEL, APPLICATION_OXML_SHEET })
public ObjectNode excelMetadata(@RequestBody final MultipartFile file) throws IOException {
LOG.info("Analyzing Excel file {}", file.getOriginalFilename());
return metadataService.getExcelMetadata(file.getInputStream());
}
final File tempFile = File.createTempFile("metadata", ".xlsx");
try {
FileUtils.copyInputStreamToFile(file.getInputStream(), tempFile);
return metadataService.getExcelMetadata(tempFile);
} finally {
FileUtils.deleteQuietly(tempFile);
}
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment