Commit a04cffc0 authored by Matija Obreza's avatar Matija Obreza
Browse files

Implemented AWS V4 Signature algorithm

parent 84f6fbf8
......@@ -187,5 +187,10 @@
<artifactId>jackson-databind</artifactId>
<version>2.8.8</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
</dependencies>
</project>
......@@ -108,7 +108,7 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
* org.genesys.filerepository.model.RepositoryFile)
*/
@Override
@Transactional
@Transactional(rollbackFor=Throwable.class)
public RepositoryFile addFile(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryFile metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
......@@ -166,7 +166,7 @@ public class RepositoryServiceImpl implements RepositoryService, InitializingBea
* org.genesys.filerepository.model.RepositoryImage)
*/
@Override
@Transactional
@Transactional(rollbackFor = Throwable.class)
public RepositoryImage addImage(final String repositoryPath, final String originalFilename, String contentType, final byte[] bytes, final RepositoryImage metaData)
throws InvalidRepositoryPathException, InvalidRepositoryFileDataException, IOException {
......
......@@ -19,51 +19,67 @@ package org.genesys.filerepository.service.impl;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
import java.util.stream.Collectors;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.xml.bind.DatatypeConverter;
import org.apache.commons.lang3.StringUtils;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.service.BytesStorageService;
import org.genesys.filerepository.service.s3.ListBucketResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.HttpRequest;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriUtils;
// TODO: Auto-generated Javadoc
/**
* Amazon S3 storage implementation.
*/
@Service("S3Storage")
public class S3StorageServiceImpl implements BytesStorageService {
public class S3StorageServiceImpl implements BytesStorageService, InitializingBean {
/** The Constant URL_ENCODING. */
private static final String URL_ENCODING = "UTF-8";
private static final Charset CHARSET_UTF8 = Charset.forName("UTF8");
private static final String HTTP_AUTHORIZATION = "Authorization";
private static final String AMZ_CONTENT_SHA256 = "X-Amz-Content-SHA256";
private static final String AMZ_DATE = "X-Amz-Date";
/** The Constant LOG. */
private final static Logger LOG = LoggerFactory.getLogger(S3StorageServiceImpl.class);
/** Algorithm for AWS V4 */
private static final String AWS_SIGN_ALG = "HmacSHA256";
/** The Constant HEADER_DATE_FORMAT. */
private static final SimpleDateFormat HEADER_DATE_FORMAT = new SimpleDateFormat("EEE', 'dd' 'MMM' 'yyyy' 'HH:mm:ss' 'Z", Locale.US);
private static final SimpleDateFormat HEADER_DATE_FORMAT = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'", Locale.US);
private static final SimpleDateFormat YYYYMMDD = new SimpleDateFormat("yyyyMMdd");
{
final TimeZone tz = TimeZone.getTimeZone("UTC");
HEADER_DATE_FORMAT.setTimeZone(tz);
YYYYMMDD.setTimeZone(tz);
}
/** The rest template. */
private final RestTemplate restTemplate = initializeRestTemplate();
......@@ -88,6 +104,11 @@ public class S3StorageServiceImpl implements BytesStorageService {
@Value("${s3.prefix}")
private String prefix;
@Override
public void afterPropertiesSet() throws Exception {
LOG.warn("S3 region={} bucket={} prefix={} dummy={}", region, bucket, prefix, getUrl("/dummy", "filename.txt"));
}
/*
* (non-Javadoc)
* @see org.genesys.filerepository.service.BytesStorageService#upsert
......@@ -113,7 +134,12 @@ public class S3StorageServiceImpl implements BytesStorageService {
}
final String url = getUrl(path, filename);
restTemplate.put(url, data);
try {
restTemplate.put(url, data);
} catch (final HttpClientErrorException e) {
LOG.error("Upserting file failed with error\n{}", e.getResponseBodyAsString());
throw e;
}
}
/*
......@@ -169,7 +195,7 @@ public class S3StorageServiceImpl implements BytesStorageService {
* @return the url
*/
private String getUrl(final String path, final String filename) {
final String url = String.format("https://%s%s", getHost(), getPath(path) + filename);
final String url = String.format("https://%s%s", getHost(), getPath(path) + startWithSlash(filename));
if (LOG.isTraceEnabled()) {
LOG.trace("getUrl path={} filename={} result={}", path, filename, url);
}
......@@ -177,16 +203,31 @@ public class S3StorageServiceImpl implements BytesStorageService {
}
/**
* Gets the path.
* Gets the path. Must end with "/" if not blank.
*
* @param path the path
* @return the path
*/
private String getPath(final String path) {
String result;
if (StringUtils.isEmpty(prefix)) {
return path;
result = startWithSlash(path);
} else {
result = prefix + startWithSlash(path);
}
return startWithSlash(result);
}
private String startWithSlash(String path) {
if (path == null)
return "";
path = path.trim();
if (path.length() == 0) {
return "";
} else {
return path.charAt(0) == '/' ? path : "/" + path;
}
return prefix + path;
}
/**
......@@ -203,66 +244,152 @@ public class S3StorageServiceImpl implements BytesStorageService {
* http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#
* ConstructingTheAuthenticationHeader
*
* @param method the method
* @param date the date
* @param filePath the file path
* @param mediaType the media type
* @return the string to sign
* @param request
* @param body
*
*
* @throws UnsupportedEncodingException the unsupported encoding exception
* @throws NoSuchAlgorithmException
*/
private String getStringToSign(final String method, final String date, final String filePath, final MediaType mediaType) throws UnsupportedEncodingException {
private String buildCanonicalRequest(final HttpRequest request, final byte[] body) throws UnsupportedEncodingException, NoSuchAlgorithmException {
final StringBuilder sb = new StringBuilder();
// Content hash
final byte[] contentSha256 = hashSha256(body == null ? "".getBytes() : body);
// Add header
request.getHeaders().set(AMZ_CONTENT_SHA256, printHex(contentSha256));
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
// HTTP-Verb
final StringBuilder sb = new StringBuilder(method).append("\n");
sb.append(request.getMethod().toString()).append("\n");
// CanonicalURI
sb.append(request.getURI().getPath()).append("\n");
// CanonicalQueryString
sb.append(StringUtils.defaultIfBlank(request.getURI().getQuery(), "")).append("\n");
// sorted headers, lowercase
request.getHeaders().keySet().stream().map(headerName -> headerName.toLowerCase()).sorted()
// remove blanks
.filter(headerName -> !request.getHeaders().getValuesAsList(headerName).isEmpty())
// print values, but how do we print multiples??
.forEach(headerName -> {
sb.append(headerName).append(":").append(request.getHeaders().get(headerName).get(0)).append("\n");
});
sb.append("\n");
// Content MD5
// signed headers
sb.append(request.getHeaders().keySet().stream().map(headerName -> headerName.toLowerCase()).sorted().collect(Collectors.joining(";")));
sb.append("\n");
// Content type
if (mediaType != null) {
sb.append(mediaType.toString());
}
// HashedPayload is the hexadecimal value of the SHA256 hash of the request
// payload.
sb.append(printHex(contentSha256));
sb.append("\n");
LOG.trace("canonicalRequest\n{}", sb.toString());
return sb.toString();
}
// Timestamp
sb.append(date).append("\n");
private static byte[] hashSha256(final byte[] bytes) throws NoSuchAlgorithmException {
final MessageDigest digest = MessageDigest.getInstance("SHA-256");
return digest.digest(bytes);
}
// CanonicalizedResource
sb.append("/").append(UriUtils.encodePath(bucket, URL_ENCODING)).append(UriUtils.encodePath(filePath, URL_ENCODING));
private static String printHex(final byte[] bytes) {
return DatatypeConverter.printHexBinary(bytes).toLowerCase();
}
private static String buildStringToSign(final String canonicalRequest, final Date date, final String region, final String awsService) throws NoSuchAlgorithmException {
final StringBuilder sb = new StringBuilder();
// "AWS4-HMAC-SHA256" + "\n" +
// timeStampISO8601Format + "\n" +
// <Scope> + "\n" +
// Hex(SHA256Hash(<CanonicalRequest>))
sb.append("AWS4-HMAC-SHA256\n");
sb.append(HEADER_DATE_FORMAT.format(date)).append("\n");
// 20130606/us-east-1/s3/aws4_request
sb.append(YYYYMMDD.format(date)).append("/").append(region).append("/").append(awsService).append("/aws4_request").append("\n");
// Hex(SHA256Hash(<CanonicalRequest>))
sb.append(printHex(hashSha256(canonicalRequest.getBytes())));
LOG.trace("stringToSign\n{}", sb.toString());
return sb.toString();
}
private static byte[] calculateSigningKey(final String secretKey, final String date, final String region, final String service) throws InvalidKeyException,
NoSuchAlgorithmException {
LOG.trace("sign date={} region={} service={}", date, region, service);
return
// SigningKey = HMAC-SHA256(<DateRegionServiceKey>, "aws4_request")
hmacSha256(
// DateRegionServiceKey = HMAC-SHA256(<DateRegionKey>, "<aws-service>")
hmacSha256(
// DateRegionKey = HMAC-SHA256(<DateKey>, "<aws-region>")
hmacSha256(
// DateKey = HMAC-SHA256("AWS4"+"<SecretAccessKey>", "<YYYYMMDD>")
hmacSha256(("AWS4" + secretKey).getBytes(), date), region), service), "aws4_request");
}
private static byte[] hmacSha256(final byte[] key, final String data) throws InvalidKeyException, NoSuchAlgorithmException {
return hmacSha256(key, data.getBytes(CHARSET_UTF8));
}
private static byte[] hmacSha256(final byte[] key, final byte[] data) throws NoSuchAlgorithmException, InvalidKeyException {
final Mac mac = Mac.getInstance(AWS_SIGN_ALG);
mac.init(new SecretKeySpec(key, AWS_SIGN_ALG));
return mac.doFinal(data);
}
/**
* Returns AWS authorization HTTP Header.
*
* http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
* http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
*
* @param request
* @param finalSignature
*
* @param signature the signature
* @param date
* @return the authorization header
*/
private String getAuthorizationHeader(final String signature) {
return new StringBuilder("AWS").append(" ").append(accessKey).append(":").append(signature).toString();
private String getAuthorizationHeader(final byte[] finalSignature, final HttpRequest request, final Date date) {
final StringBuilder sb = new StringBuilder("AWS4-HMAC-SHA256").append(" Credential=")
// credential
.append(getAWSCredential(date))
// signed headers
.append(",SignedHeaders=");
// signed headers
sb.append(request.getHeaders().keySet().stream().map(headerName -> headerName.toLowerCase()).sorted().collect(Collectors.joining(";")));
// request signature
sb.append(",Signature=").append(printHex(finalSignature));
LOG.trace("authorizationHeader=\n{}", sb.toString());
return sb.toString();
}
/**
* Returns encrypted signature.
*
* @param stringToSign the string to sign
* @return the byte[]
* @throws NoSuchAlgorithmException the no such algorithm exception
* @throws InvalidKeyException the invalid key exception
* @throws UnsupportedEncodingException the unsupported encoding exception
*/
private byte[] encryptHmacSHA1(final String stringToSign) throws NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
final String algorithm = "HmacSHA1";
final Mac mac = Mac.getInstance(algorithm);
mac.init(new SecretKeySpec(secretKey.getBytes("UTF8"), algorithm));
return mac.doFinal(stringToSign.getBytes("UTF8"));
private String getAWSCredential(final Date date) {
return String.format("%s/%s/%s/%s/aws4_request", accessKey, (YYYYMMDD.format(date)), (region), ("s3"));
}
/**
* Initializes RestTemplate with the interceptor that signs the HTTP requests to
* AWS.
* AWS using V4 signature method.
*
* @return the rest template
*/
......@@ -272,49 +399,23 @@ public class S3StorageServiceImpl implements BytesStorageService {
final List<ClientHttpRequestInterceptor> interceptors = new ArrayList<>();
interceptors.add((request, body, execution) -> {
final String date = HEADER_DATE_FORMAT.format(new Date());
request.getHeaders().set("Date", date);
final Date date = new Date();
request.getHeaders().set("Host", getHost());
// This avoids date formatting problems
request.getHeaders().add(AMZ_DATE, HEADER_DATE_FORMAT.format(date));
final String stringToSign = getStringToSign(request.getMethod().toString(), date, request.getURI().getPath(), request.getHeaders().getContentType());
String authorizationHeader = null;
try {
final byte[] signatureBytes = encryptHmacSHA1(stringToSign);
final String signature = new String(Base64.getEncoder().encode(signatureBytes));
authorizationHeader = getAuthorizationHeader(signature);
final String canonicalRequest = buildCanonicalRequest(request, body);
final String stringToSign = buildStringToSign(canonicalRequest, date, region, "s3");
final byte[] signingKey = calculateSigningKey(secretKey, YYYYMMDD.format(date), region, "s3");
final byte[] finalSignature = hmacSha256(signingKey, stringToSign);
request.getHeaders().set(HTTP_AUTHORIZATION, getAuthorizationHeader(finalSignature, request, date));
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
throw new IOException("Signature for request can't be created");
LOG.error("Could not sign AWS request.", e);
}
request.getHeaders().set("Authorization", authorizationHeader);
// LOG.debug("===========================request
// begin================================================");
// LOG.debug("URI : " + request.getURI());
// LOG.debug("Method : " + request.getMethod());
// LOG.debug("Request Body : " + new String(body, "UTF-8"));
// LOG.debug("==========================request
// end================================================");
final ClientHttpResponse response = execution.execute(request, body);
//
// StringBuilder inputStringBuilder = new StringBuilder();
// BufferedReader bufferedReader = new BufferedReader(new
// InputStreamReader(response.getBody(), "UTF-8"));
// String line = bufferedReader.readLine();
// while (line != null) {
// inputStringBuilder.append(line);
// inputStringBuilder.append('\n');
// line = bufferedReader.readLine();
// }
// LOG.debug("============================response
// begin==========================================");
// LOG.debug("status code: " + response.getStatusCode());
// LOG.debug("status text: " + response.getStatusText());
// LOG.debug("Response Body : " +
// inputStringBuilder.toString());
// LOG.debug("=======================response
// end=================================================");
return response;
});
......@@ -402,4 +503,5 @@ public class S3StorageServiceImpl implements BytesStorageService {
return listBucketResult.getContents().stream().map(content -> content.getKey().substring(s3prefix.length())).collect(Collectors.toList());
}
}
}
/*
* Copyright 2017 Global Crop Diversity Trust
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.genesys.filerepository.service;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
import java.nio.charset.Charset;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.TimeZone;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.xml.bind.DatatypeConverter;
import org.junit.Test;
public class S3SignatureTest {
private static final Charset CHARSET_UTF8 = Charset.forName("UTF8");
private static final String AWS_SIGN_ALG = "HmacSHA256";
private static final SimpleDateFormat HEADER_DATE_FORMAT = new SimpleDateFormat("EEE', 'dd' 'MMM' 'yyyy' 'HH:mm:ss' UTC'", Locale.US);
private static final SimpleDateFormat YYYYMMDD = new SimpleDateFormat("yyyyMMdd");
{
final TimeZone tz = TimeZone.getTimeZone("UTC");
HEADER_DATE_FORMAT.setTimeZone(tz);
YYYYMMDD.setTimeZone(tz);
}
private byte[] hmacSha256(final byte[] key, final String data) throws InvalidKeyException, NoSuchAlgorithmException {
return hmacSha256(key, data.getBytes(CHARSET_UTF8));
}
private byte[] hmacSha256(final byte[] key, final byte[] data) throws NoSuchAlgorithmException, InvalidKeyException {
final Mac mac = Mac.getInstance(AWS_SIGN_ALG);
mac.init(new SecretKeySpec(key, AWS_SIGN_ALG));
return mac.doFinal(data);
}
public String printHex(final byte[] bytes) {
return DatatypeConverter.printHexBinary(bytes).toLowerCase();
}
private byte[] calculateSigningKey(final String secretKey, final String date, final String region, final String service) throws InvalidKeyException, NoSuchAlgorithmException {
System.err.println("sign date" + date + " region=" + region + " service=" + service);
return
// SigningKey = HMAC-SHA256(<DateRegionServiceKey>, "aws4_request")
hmacSha256(
// DateRegionServiceKey = HMAC-SHA256(<DateRegionKey>, "<aws-service>")
hmacSha256(
// DateRegionKey = HMAC-SHA256(<DateKey>, "<aws-region>")
hmacSha256(
// DateKey = HMAC-SHA256("AWS4"+"<SecretAccessKey>", "<YYYYMMDD>")
hmacSha256(("AWS4" + secretKey).getBytes(), date), region), service), "aws4_request");
}
// http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
@Test
public void test0() throws InvalidKeyException, NoSuchAlgorithmException {
final byte[] signingKey = calculateSigningKey("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20150830", "us-east-1", "iam");
assertThat("c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9", is(printHex(signingKey)));
final byte[] signature = hmacSha256(signingKey, "AWS4-HMAC-SHA256\n" + "20150830T123600Z\n" + "20150830/us-east-1/iam/aws4_request\n"
+ "f536975d06c0309214f805bb90ccff089219ecd68b2577efef23edd43b7e1a59");
assertThat("5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400e06b5924a6f2b5d7", is(printHex(signature)));
}
@Test
public void test1() throws InvalidKeyException, NoSuchAlgorithmException {
// AKIDEXAMPLE/20150830/us-east-1/service/aws4_request
final String accessKey = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY";
final byte[] signingKey = calculateSigningKey(accessKey, "20150830", "us-east-1", "service");
System.err.println(printHex(signingKey));
final byte[] signature = hmacSha256(signingKey, "Copy\n" + "AWS4-HMAC-SHA256\n" + "20150830T123600Z\n" + "20150830/us-east-1/service/aws4_request\n"
+ "816cd5b414d056048ba4f7c5386d6e0533120fb1fcfa93762cf0fc39e2cf19e0");
System.err.println(printHex(signature));
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment