Commit 55b14421 authored by Maxym Borodenko's avatar Maxym Borodenko
Browse files

Update preview content

- changed signatures of methods
parent f5caba0f
......@@ -80,10 +80,10 @@ public class PreviewApi {
LOG.warn("Overriding encoding {} to {}", encoding, mediaType.getCharset().name());
encoding = mediaType.getCharset().name();
}
return metadataService.getTextData(uuid, expires, inputFile, null, encoding);
return metadataService.previewFromCSV(uuid, expires, inputFile, null, encoding);
}
return metadataService.getExcelData(uuid, expires, inputFile, null);
return metadataService.previewFromExcel(uuid, expires, inputFile, null);
}
@GetMapping(path = "/{uuid}")
......
......@@ -24,8 +24,8 @@ import org.springframework.web.multipart.MultipartFile;
public interface MetadataService {
Preview getExcelData(UUID uuid, Date expires, MultipartFile excelFile, Integer maxRows) throws IOException;
Preview previewFromExcel(UUID uuid, Date expires, MultipartFile excelFile, Integer maxRows) throws IOException;
Preview getTextData(UUID uuid, Date expires, MultipartFile inputFile, Integer maxRows, String encoding) throws IOException;
Preview previewFromCSV(UUID uuid, Date expires, MultipartFile inputFile, Integer maxRows, String encoding) throws IOException;
}
......@@ -30,12 +30,14 @@ public interface PreviewService {
Preview add(Preview preview);
void addRow(UUID refrenceUuid, int sheetIndex, int rowNum, List<Object> values);
void addRow(UUID referenceUuid, int sheetIndex, int rowNum, List<Object> values);
Preview getPreview(UUID referenceUuid);
List<Document> getData(UUID referenceUuid, long sheet, long startRow, int limit, String... selectedColumns);
void removeMongoRawdata(UUID referenceUuid);
List<StatisticsData> getStatisticsData(UUID referenceUuid, long sheet, long startRow, int limit, String... selectedFields);
Preview updateState(String id, State reading, String error);
......
......@@ -80,9 +80,13 @@ public class MetadataServiceImpl implements MetadataService {
@Override
@Transactional
public Preview getTextData(final UUID referenceUuid, Date expires, final MultipartFile csvFile, final Integer maxRows, String encoding) throws IOException {
public Preview previewFromCSV(final UUID referenceUuid, Date expires, final MultipartFile csvFile, final Integer maxRows, String encoding) throws IOException {
Preview preview = new Preview();
Preview preview = previewService.getPreview(referenceUuid);
if (preview == null) {
preview = new Preview();
}
LOG.warn("Preview with UUID={} already exists. Content will be updated.", referenceUuid);
preview.setName(csvFile.getOriginalFilename());
preview.setReferenceUuid(referenceUuid);
......@@ -101,10 +105,13 @@ public class MetadataServiceImpl implements MetadataService {
csvFile.transferTo(localFile);
// Save
previewService.add(preview);
Preview savedPreview = previewService.add(preview);
// remove old content of the preview
previewService.removeMongoRawdata(savedPreview.getReferenceUuid());
taskExecutor.execute(() -> {
previewService.updateState(preview.getId(), Preview.State.READING, null);
previewService.updateState(savedPreview.getId(), Preview.State.READING, null);
Charset charset=Charset.defaultCharset();
if (StringUtils.isNotBlank(encoding)) {
try {
......@@ -118,7 +125,7 @@ public class MetadataServiceImpl implements MetadataService {
final int sheetIndex = 0;
PreviewSheet sheetInfo = new PreviewSheet(sheetIndex, "CSV");
previewService.updateState(preview.getId(), Preview.State.READING, "Parsing CSV");
previewService.updateState(savedPreview.getId(), Preview.State.READING, "Parsing CSV");
AtomicInteger rowNum = new AtomicInteger(0);
Iterator<String[]> it = reader.iterator();
......@@ -144,13 +151,13 @@ public class MetadataServiceImpl implements MetadataService {
}
sheetInfo.rowCount = rowNum.get();
previewService.registerSheet(preview.getId(), sheetInfo);
previewService.registerSheet(savedPreview.getId(), sheetInfo);
previewService.updateState(preview.getId(), Preview.State.READY, null);
previewService.updateState(savedPreview.getId(), Preview.State.READY, null);
} catch (final Throwable e) {
LOG.error("Error writing metadata. {}", e.getMessage(), e);
previewService.updateState(preview.getId(), Preview.State.FAILED, e.getMessage());
previewService.updateState(savedPreview.getId(), Preview.State.FAILED, e.getMessage());
} finally {
localFile.toFile().delete();
}
......@@ -161,9 +168,12 @@ public class MetadataServiceImpl implements MetadataService {
@Override
@Transactional
public Preview getExcelData(final UUID referenceUuid, Date expires, final MultipartFile excelFile, final Integer maxRows) throws IOException {
public Preview previewFromExcel(final UUID referenceUuid, Date expires, final MultipartFile excelFile, final Integer maxRows) throws IOException {
Preview preview = new Preview();
Preview preview = previewService.getPreview(referenceUuid);
if (preview == null) {
preview = new Preview();
}
preview.setName(excelFile.getOriginalFilename());
preview.setReferenceUuid(referenceUuid);
......@@ -182,16 +192,19 @@ public class MetadataServiceImpl implements MetadataService {
excelFile.transferTo(localFile);
// Save
previewService.add(preview);
Preview savedPreview = previewService.add(preview);
// remove old content of the preview
previewService.removeMongoRawdata(savedPreview.getReferenceUuid());
taskExecutor.execute(() -> {
previewService.updateState(preview.getId(), Preview.State.READING, null);
previewService.updateState(savedPreview.getId(), Preview.State.READING, null);
try (InputStream is = new FileInputStream(localFile.toFile()); Workbook workbook = StreamingReader.builder().rowCacheSize(100).bufferSize(4096).open(is)) {
for (Sheet sheet : workbook) {
final int sheetIndex = workbook.getSheetIndex(sheet);
PreviewSheet sheetInfo = new PreviewSheet(sheetIndex, sheet.getSheetName());
previewService.updateState(preview.getId(), Preview.State.READING, "Parsing " + sheet.getSheetName());
previewService.updateState(savedPreview.getId(), Preview.State.READING, "Parsing " + sheet.getSheetName());
AtomicInteger rowNum = new AtomicInteger(0);
for (Row currentRow : sheet) {
......@@ -221,14 +234,14 @@ public class MetadataServiceImpl implements MetadataService {
}
sheetInfo.rowCount = rowNum.get() + 1;
previewService.registerSheet(preview.getId(), sheetInfo);
previewService.registerSheet(savedPreview.getId(), sheetInfo);
}
previewService.updateState(preview.getId(), Preview.State.READY, null);
previewService.updateState(savedPreview.getId(), Preview.State.READY, null);
} catch (final Throwable e) {
LOG.error("Error writing metadata. {}", e.getMessage(), e);
previewService.updateState(preview.getId(), Preview.State.FAILED, e.getMessage());
previewService.updateState(savedPreview.getId(), Preview.State.FAILED, e.getMessage());
} finally {
localFile.toFile().delete();
}
......
......@@ -131,12 +131,12 @@ public class PreviewServiceImpl implements PreviewService, InitializingBean {
}
@Override
public void addRow(UUID refrenceUuid, int sheetIndex, int rowNum, List<Object> values) {
public void addRow(UUID referenceUuid, int sheetIndex, int rowNum, List<Object> values) {
if (values == null || values.isEmpty()) {
return;
}
Document doc = new Document(RAWDATA_DATASET, refrenceUuid).append(RAWDATA_SHEET, sheetIndex).append(RAWDATA_ROW, rowNum);
Document doc = new Document(RAWDATA_DATASET, referenceUuid).append(RAWDATA_SHEET, sheetIndex).append(RAWDATA_ROW, rowNum);
final AtomicLong columnCount = new AtomicLong(0);
values.forEach(value -> {
long col = columnCount.getAndIncrement();
......@@ -193,6 +193,14 @@ public class PreviewServiceImpl implements PreviewService, InitializingBean {
return statistics;
}
public void removeMongoRawdata(UUID referenceUuid) {
Preview preview = previewRepository.findByReferenceUuid(referenceUuid);
if (preview != null) {
DeleteResult res = mongoTemplate.remove(query(where(RAWDATA_DATASET).is(preview.getReferenceUuid())), MONGO_RAWDATA);
LOG.info("Removal of raw data for {}: {}", preview.getReferenceUuid(), res);
}
}
@Scheduled(initialDelay = 1000 * 60 * 1, fixedDelay = 1000 * 60 * 10)
public void removeExpiredPreviews() {
List<Preview> expiredPreviews = previewRepository.findByExpiresLessThan(new Date());
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment