Commit 3edae7ce authored by Matija Obreza's avatar Matija Obreza
Browse files

WIP: Interfacing with the repositoryService

parent 683a3d08
......@@ -28,6 +28,7 @@
<properties>
<commons.io.version>2.4</commons.io.version>
<tika.version>1.14</tika.version>
</properties>
<build>
......@@ -171,5 +172,10 @@
<version>${querydsl.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>${tika.version}</version>
</dependency>
</dependencies>
</project>
......@@ -28,6 +28,7 @@ import java.util.UUID;
import javax.imageio.ImageIO;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.tika.Tika;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
......@@ -100,11 +101,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
*/
@Override
@Transactional
public RepositoryFile addFile(final String repositoryPath, final String originalFilename, final String contentType, final byte[] bytes, final RepositoryFile metaData)
public RepositoryFile addFile(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryFile metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
PathValidator.checkValidPath(repositoryPath);
if (contentType == null) {
contentType = new Tika().detect(bytes);
} else {
String detectedContentType = new Tika().detect(bytes);
LOG.info("Content-Type provided={} detected={}", contentType, detectedContentType);
}
if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException();
}
......@@ -145,11 +153,18 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
*/
@Override
@Transactional
public RepositoryImage addImage(final String repositoryPath, final String originalFilename, final String contentType, final byte[] bytes, final RepositoryImage metaData)
public RepositoryImage addImage(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryImage metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
PathValidator.checkValidPath(repositoryPath);
if (contentType == null) {
contentType = new Tika().detect(bytes);
} else {
String detectedContentType = new Tika().detect(bytes);
LOG.info("Content-Type provided={} detected={}", contentType, detectedContentType);
}
if (originalFilename == null || contentType == null || bytes == null) {
throw new InvalidRepositoryFileDataException();
}
......@@ -331,6 +346,7 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
// Calculate SHA-1 and MD5 sums
repositoryFile.setSha1Sum(DigestUtils.sha1Hex(bytes));
repositoryFile.setMd5Sum(DigestUtils.md5Hex(bytes));
repositoryFile.setSize(bytes.length);
repositoryFile.setContentType(contentType);
bytesStorageService.upsert(repositoryFile.getPath(), repositoryFile.getFilename(), bytes);
......
......@@ -82,5 +82,10 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
</dependencies>
</project>
......@@ -116,9 +116,7 @@ public abstract class CanBeAnythingFile implements FtpFile {
}
@Override
public OutputStream createOutputStream(long offset) throws IOException {
return null;
}
public abstract OutputStream createOutputStream(long offset) throws IOException;
@Override
public InputStream createInputStream(long offset) throws IOException {
......
......@@ -48,8 +48,11 @@ public class RepositoryFileSystemFactory implements FileSystemFactory, Initializ
@Autowired(required = true)
private RepositoryService repositoryService;
@Autowired(required = true)
private TemporaryBytesManager bytesManager;
private RepositoryFtpFile file(RepositoryFile repositoryFile) {
LOG.debug("Making RepositoryFtpDirectory repositoryFile={}", repositoryFile);
LOG.debug("Making RepositoryFtpFile repositoryFile={}", repositoryFile);
RepositoryFtpFile rff = new RepositoryFtpFile(repositoryFile) {
......@@ -91,13 +94,15 @@ public class RepositoryFileSystemFactory implements FileSystemFactory, Initializ
@Override
public OutputStream createOutputStream(long offset) throws IOException {
LOG.info("Creating output stream for file={} at offset={}", getAbsolutePath(), offset);
// TODO Auto-generated method stub
return null;
}
@Override
public InputStream createInputStream(long offset) throws IOException {
return null;
LOG.info("Creating input stream for file={} at offset={}", getAbsolutePath(), offset);
return bytesManager.createInputStream(Paths.get(getAbsolutePath()), offset);
}
};
......@@ -190,6 +195,7 @@ public class RepositoryFileSystemFactory implements FileSystemFactory, Initializ
try {
return path.endsWith("/") ? directory(path.toString(), this) : file(repositoryService.getFile(path.getParent().toString(), path.getFileName().toString()));
} catch (NoSuchRepositoryFileException e) {
LOG.info("Making new CanBeAnythingFile path={} name={}", path.getParent().toString(), path.getFileName().toString());
return new CanBeAnythingFile(path.getParent().toString(), path.getFileName().toString()) {
@Override
......@@ -199,6 +205,12 @@ public class RepositoryFileSystemFactory implements FileSystemFactory, Initializ
return true;
}
@Override
public OutputStream createOutputStream(long offset) throws IOException {
LOG.info("Creating output stream for new file={} at offset={}", getAbsolutePath(), offset);
assert (offset == 0l);
return bytesManager.newFile(Paths.get(getAbsolutePath()));
}
};
}
}
......
/*
* Copyright 2017 Global Crop Diversity Trust, www.croptrust.org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.filerepository.service.ftp;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.util.UUID;
import org.apache.commons.io.IOUtils;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.service.BytesStorageService;
import org.genesys.filerepository.service.RepositoryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* This little beast manages a local copy of bytes for the FTP service. It makes a local copy of the bytes, allows for
* downloading, updating and synchronizing back to the {@link BytesStorageService}
*
* @author Matija Obreza
*/
@Component
public class TemporaryBytesManager {
private final static Logger LOG = LoggerFactory.getLogger(TemporaryBytesManager.class);
@Autowired(required = true)
private RepositoryService repositoryService;
// private int maxFileSize = 1024 * 1024 * 50; // 50MB
/**
* Create a local file for writing. On stream close push the file to {@link BytesStorageService}
*
* @param absolutePath
* @param name
* @return
* @throws IOException
*/
public OutputStream newFile(Path path) throws IOException {
final String parent = path.getParent().toString();
final String filename = path.getFileName().toString();
LOG.info("Creating new parent={} filename={}", path, filename);
File tempFile = File.createTempFile("ftp-", ".data");
return new BufferedOutputStream(new FileOutputStream(tempFile)) {
@Override
public void close() throws IOException {
try {
super.close();
String repositoryPath = parent;
String originalFilename = filename;
ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
try (InputStream source = new FileInputStream(tempFile)) {
IOUtils.copy(source, baos);
}
RepositoryFile metaData = new RepositoryFile();
metaData.setOriginalFilename(filename);
metaData.setPath(repositoryPath);
LOG.debug("Synchronizing file={} with repository path={} originalFilename={} size={}", tempFile.getAbsolutePath(), repositoryPath, originalFilename, baos.size());
repositoryService.addFile(repositoryPath, originalFilename, null, baos.toByteArray(), metaData);
LOG.info("Synchronized file={} with repository path={} originalFilename={}", tempFile.getAbsolutePath(), repositoryPath, originalFilename);
} catch (InvalidRepositoryPathException | InvalidRepositoryFileDataException e) {
LOG.error("Oh, oh", e);
throw new IOException(e);
} finally {
LOG.info("Removing temporary file={}", tempFile.getAbsolutePath());
tempFile.delete();
}
}
@Override
public void flush() throws IOException {
super.flush();
LOG.debug("Flushing temp file={}", tempFile.getAbsolutePath());
}
};
}
public InputStream createInputStream(Path path, long offset) throws IOException {
LOG.info("Preparing local copy of={} offset={}", path.getFileName().toString(), offset);
UUID uuid = UUID.fromString(path.getFileName().toString());
try {
RepositoryFile repositoryFile = repositoryService.getFile(uuid);
File tempFile = File.createTempFile("ftp-", ".data");
byte[] bytes = repositoryService.getFileBytes(repositoryFile);
LOG.debug("Loaded bytes={} from repository file.size={}", bytes.length, repositoryFile.getSize());
try (FileOutputStream fos = new FileOutputStream(tempFile)) {
IOUtils.write(bytes, fos);
fos.flush();
}
LOG.debug("Shadow file has length={}", tempFile.length());
// we have a local copy, now return the funny stream
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(tempFile)) {
@Override
public void close() throws IOException {
super.close();
try (InputStream tos = new FileInputStream(tempFile)) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
IOUtils.copy(tos, baos);
LOG.info("Stream to temporary upload buffer is closed, synchronizing bytes={}", baos.size());
repositoryService.updateBytes(repositoryFile, null, baos.toByteArray());
LOG.debug("Bytes={}", baos.toByteArray());
} catch (NoSuchRepositoryFileException e) {
LOG.error(e.getMessage());
throw new IOException(e);
}
}
};
if (offset > 0) {
LOG.trace("Skipping {} of output stream", offset);
bis.skip(offset);
}
return bis;
} catch (NoSuchRepositoryFileException e) {
throw new IOException(e);
}
}
}
......@@ -45,6 +45,11 @@ public class ApplicationConfig {
return ftpServer;
}
@Bean
public TemporaryBytesManager temporaryBytesManager() {
return new TemporaryBytesManager();
}
@Bean
public RepositoryFileSystemFactory repositoryFileSystemFactory() {
return new RepositoryFileSystemFactory();
......
......@@ -15,10 +15,12 @@
*/
package org.genesys.filerepository.service.ftp;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.SocketException;
import java.util.Arrays;
import java.util.stream.Collectors;
......@@ -29,6 +31,7 @@ import org.apache.commons.net.ftp.FTPReply;
import org.apache.commons.net.ftp.FTPSClient;
import org.genesys.filerepository.config.DatabaseConfig;
import org.genesys.filerepository.config.ServiceBeanConfig;
import org.hsqldb.lib.StringInputStream;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
......@@ -42,6 +45,8 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@ContextConfiguration(classes = { DatabaseConfig.class, ServiceBeanConfig.class, ApplicationConfig.class })
public class FtpServerTest {
private static final String TEST_FILE_CONTENTS = "This is a test";
private final static Logger LOG = LoggerFactory.getLogger(FtpServerTest.class);
private String username = "user";
......@@ -184,4 +189,44 @@ public class FtpServerTest {
ftp.disconnect();
}
}
@Test
public void simpleUploadTest() throws SocketException, IOException {
final FTPClient ftp = new FTPClient();
try {
ftp.connect("localhost", 8021);
assertThat("FTP server refused connection", FTPReply.isPositiveCompletion(ftp.getReplyCode()), is(true));
ftp.login(username, password);
assertThat("Login failed", FTPReply.isPositiveCompletion(ftp.getReplyCode()), is(true));
assertThat(ftp.printWorkingDirectory(), is("/"));
ftp.setFileTransferMode(FTPClient.BINARY_FILE_TYPE);
try (InputStream local = new StringInputStream(TEST_FILE_CONTENTS)) {
assertThat(ftp.storeFile("file", local), is(true));
}
FTPFile[] files = ftp.listFiles();
assertThat(Arrays.asList(files), hasSize(1));
FTPFile theFile = files[0];
LOG.debug("Uploaded file={}", theFile);
ByteArrayOutputStream local = new ByteArrayOutputStream(1024);
LOG.info("Retrieving file={}", theFile.getName());
ftp.retrieveFile(theFile.getName(), local);
LOG.debug("size={}", local.size());
String contents = new String(local.toByteArray());
LOG.debug("toStr={} test={}", contents, TEST_FILE_CONTENTS);
assertThat(contents, is(TEST_FILE_CONTENTS));
debugListFiles(ftp);
} finally {
ftp.disconnect();
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment