Commit a3ead393 authored by Matija Obreza's avatar Matija Obreza
Browse files

Merge branch 'streaming-support' into 'master'

Added streaming support to read bytes from BytesStorage

See merge request !50
parents d6e03434 2309eda9
/*
* Copyright 2018 Global Crop Diversity Trust
* Copyright 2021 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -17,8 +17,10 @@
package org.genesys.filerepository.service;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.List;
import java.util.function.Consumer;
import org.genesys.filerepository.InvalidRepositoryPathException;
......@@ -55,6 +57,15 @@ public interface BytesStorageService {
*/
byte[] get(Path bytesFile) throws IOException;
/**
* Stream the bytes to the consumer stream
*
* @param bytesFile the bytes file
* @param consumerStream the consumer stream
* @throws IOException Signals that an I/O exception has occurred.
*/
void stream(Path bytesFile, Consumer<InputStream> consumerStream) throws IOException;
/**
* Check if bytes exist at specified location.
*
......
......@@ -22,11 +22,13 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
......@@ -193,6 +195,28 @@ public class FilesystemStorageServiceImpl implements InitializingBean, BytesStor
return data;
}
@Override
public void stream(Path bytesFile, Consumer<InputStream> consumerStream) throws IOException {
final Path normalPath = bytesFile.normalize().toAbsolutePath();
final Path destinationFilePath = getDestPath(normalPath);
LOG.trace("Retrieving bytes of {}", normalPath);
final File destinationFile = destinationFilePath.toFile();
if (!destinationFile.getCanonicalPath().startsWith(repoDir.getCanonicalPath())) {
throw new IOException("Not within repository path: " + destinationFile.getAbsolutePath());
}
if (destinationFile.exists()) {
try (FileInputStream inputStream = new FileInputStream(destinationFile)) {
consumerStream.accept(inputStream);
}
} else {
throw new IOException("Repository bytes not found at " + destinationFile.getAbsolutePath());
}
}
/*
* (non-Javadoc)
* @see org.genesys.filerepository.service.BytesStorageService#exists(java.lang.
......
......@@ -18,6 +18,7 @@ package org.genesys.filerepository.service.impl;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.Charset;
......@@ -34,6 +35,7 @@ import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import javax.crypto.Mac;
......@@ -204,6 +206,26 @@ public class S3StorageServiceImpl implements BytesStorageService, InitializingBe
}
}
@Override
public void stream(Path bytesFile, Consumer<InputStream> consumerStream) throws IOException {
final Path normalPath = bytesFile.normalize().toAbsolutePath();
if (LOG.isDebugEnabled()) {
LOG.debug("Getting bytes path={} filename={}", normalPath.getParent().toString(), normalPath.getFileName().toString());
}
final String url = getAwsUrl(normalPath);
try {
restTemplate.execute(url, HttpMethod.GET, null, (clientHttpResponse) -> {
consumerStream.accept(clientHttpResponse.getBody());
return null;
});
} catch (final HttpClientErrorException e) {
LOG.error("Getting bytes failed with error\n{}", e.getResponseBodyAsString());
throw e;
}
}
/**
* Returns URL for S3 resource.
*
......
......@@ -19,6 +19,7 @@ package org.genesys.filerepository.service;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
......@@ -26,6 +27,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.genesys.filerepository.BytesStorageException;
import org.genesys.filerepository.service.impl.FilesystemStorageServiceImpl;
import org.junit.AfterClass;
......@@ -185,4 +187,35 @@ public class FileSystemStorageTest extends RepositoryServiceTest {
assertThat("File still exists", expectedFile.exists(), is(false));
}
/**
* Test streaming.
*/
@Test
public void testFileStream() throws FileNotFoundException, IOException {
final Path filePath = Paths.get("/foo/bar/", "somecontents");
final File expectedFile = new File(baseDir, "foo/bar/somecontents");
byte[] lotsaBytes = RandomStringUtils.random(1024*1024*5).getBytes();
bytesStorage.upsert(filePath, lotsaBytes);
assertThat("File not created", expectedFile.exists(), is(true));
assertThat("Created file is not a file", expectedFile.isFile(), is(true));
assertThat("File size does not match", expectedFile.length(), is((long) lotsaBytes.length));
try (ByteArrayOutputStream baos = new ByteArrayOutputStream(lotsaBytes.length)) {
bytesStorage.stream(filePath, (inputStreamConsumer) -> {
try {
inputStreamConsumer.transferTo(baos);
} catch (IOException e) {
e.printStackTrace();
}
});
baos.flush();
assertThat("Bytes don't match", baos.toByteArray(), equalTo(lotsaBytes));
}
bytesStorage.remove(filePath);
assertThat("File still exists", expectedFile.exists(), is(false));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment