Added support for OpenSearch Serverless.

dblock · acm19 · commit f0ace05d27d6 · 2023-01-25T23:23:03.000+01:00
Signed-off-by: dblock &lt;dblock@amazon.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
-### 2.2.1 (Next)
+### 2.3.0 (Next)
 
+* [#94](https://github.com/acm19/aws-request-signing-apache-interceptor/pull/94): Add support for OpenSearch Serverless - [@dblock](https://github.com/dblock).
 * [#92](https://github.com/acm19/aws-request-signing-apache-interceptor/pull/92): Make samples compatible with OpenSearch 2.x - [@dblock](https://github.com/dblock).
 
 ### 2.2.0 (2022/10/02)
diff --git a/Makefile b/Makefile
@@ -1,5 +1,6 @@
 REGION?=us-east-1
 ENDPOINT?=http://my-domain.us-east-1.opensearch.localhost.localstack.cloud:4566
+SERVICE?=es
 
 .PHONY: verify
 .SILENT: verify
@@ -12,21 +13,21 @@ run_sample:
 	mvn test-compile exec:java \
 		-Dexec.classpathScope=test \
 		-Dexec.mainClass="io.github.acm19.aws.interceptor.test.AmazonOpenSearchServiceSample" \
-		-Dexec.args="--endpoint=$(ENDPOINT) --region=$(REGION)"
+		-Dexec.args="--endpoint=$(ENDPOINT) --region=$(REGION) --service=$(SERVICE)"
 
 .PHONY: run_v5_sample
 .SILENT: run_v5_sample
 run_v5_sample:
 	mvn test-compile exec:java \
 		-Dexec.classpathScope=test \
 		-Dexec.mainClass="io.github.acm19.aws.interceptorv5.test.AmazonOpenSearchServiceSample" \
-		-Dexec.args="--endpoint=$(ENDPOINT) --region=$(REGION)"
+		-Dexec.args="--endpoint=$(ENDPOINT) --region=$(REGION) --service=$(SERVICE)"
 
 debug_v5_sample:
 	mvn exec:exec -Dexec.executable="java" -Dexec.classpathScope=test \
 		-Dexec.args="-classpath %classpath -Xdebug \
 				-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005 \
-				io.github.acm19.aws.interceptorv5.test.AmazonOpenSearchServiceSample --endpoint=$(ENDPOINT) --region=$(REGION)"
+				io.github.acm19.aws.interceptorv5.test.AmazonOpenSearchServiceSample --endpoint=$(ENDPOINT) --region=$(REGION) --service=$(SERVICE)"
 
 .PHONY: validate_renovate_config
 .SILENT: validate_renovate_config
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 
 An AWS request signing interceptor for arbitrary HTTP requests. It supports both [Apache HTTP Client](https://search.maven.org/artifact/org.apache.httpcomponents/httpclient) and [Apache HTTP Client V5](https://search.maven.org/artifact/org.apache.httpcomponents.client5/httpclient5).
 
-This library enables you to sign requests to any service that leverages SigV4, and thus access any AWS Service or APIG-backed service.
+This library enables you to sign requests to any service that leverages SigV4, and thus access any AWS Service or APIG-backed service, including Amazon managed OpenSearch and OpenSearch Serverless.
 
 This library is based on [AWS Interceptor](https://github.com/awslabs/aws-request-signing-apache-interceptor), but using AWS SDK 2.x.
 
@@ -110,13 +110,13 @@ export AWS_ACCESS_KEY_ID=
 export AWS_SECRET_ACCESS_KEY=
 export AWS_SESSION_TOKEN=
 
-mvn test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass="io.github.acm19.aws.interceptor.test.AmazonOpenSearchServiceSample" -Dexec.args="--endpoint=https://...us-west-2.es.amazonaws.com --region=us-west-2"
+mvn test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass="io.github.acm19.aws.interceptor.test.AmazonOpenSearchServiceSample" -Dexec.args="--endpoint=https://...us-west-2.es.amazonaws.com --region=us-west-2 --service=es"
 ```
 
 Alternatively use `make` as follows:
 
 ```
-ENDPOINT=<your-endpoint> REGION=<your-region> make run_sample
+ENDPOINT=<your-endpoint> SERVICE=es REGION=<your-region> SERVICE=es make run_sample
 ```
 
 See [examples](src/test/java/io/github/acm19/aws/interceptor/test) for more valid requests.
@@ -130,13 +130,13 @@ export AWS_ACCESS_KEY_ID=
 export AWS_SECRET_ACCESS_KEY=
 export AWS_SESSION_TOKEN=
 
-mvn test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass="io.github.acm19.aws.interceptorv5.test.AmazonOpenSearchServiceSample" -Dexec.args="--endpoint=https://...us-west-2.es.amazonaws.com --region=us-west-2"
+mvn test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass="io.github.acm19.aws.interceptorv5.test.AmazonOpenSearchServiceSample" -Dexec.args="--endpoint=https://...us-west-2.aoss.amazonaws.com --region=us-west-2 --service=aoss"
 ```
 
 Alternatively use `make` as follows:
 
 ```
-ENDPOINT=<your-endpoint> REGION=<your-region> make run_v5_sample
+ENDPOINT=<your-endpoint> REGION=<your-region> SERVICE=aoss make run_v5_sample
 ```
 
 See [examples](src/test/java/io/github/acm19/aws/interceptorv5/test) for more valid requests.
diff --git a/infra/README.md b/infra/README.md
@@ -6,13 +6,13 @@ This project creates an OpenSearch cluster on AWS for development purposes. Its
 To manually create a virtualenv and install the dependencies on MacOS and Linux:
 
 ```
-make .venv
+make venv
 ```
 
 After the init process completes and the virtualenv is created, you can use the following step to activate your virtualenv.
 
 ```
-source .venv/bin/activate
+source ./venv/bin/activate
 ```
 
 At this point you can now synthesise the CloudFormation template for this code.
diff --git a/infra/opensearch_cluster/opensearch_cluster.py b/infra/opensearch_cluster/opensearch_cluster.py
@@ -10,7 +10,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
 
         open_search_cluster = aws_opensearchservice.Domain(self,
                                                            "InterceptorTest",
-                                                           version=EngineVersion.OPENSEARCH_1_2,
+                                                           version=EngineVersion.OPENSEARCH_2_3,
                                                            capacity=aws_opensearchservice.CapacityConfig(
                                                                data_node_instance_type="t3.small.search"
                                                            ),
diff --git a/infra/package-lock.json b/infra/package-lock.json
diff --git a/infra/package.json b/infra/package.json
@@ -2,7 +2,7 @@
   "name": "aws-request-signing-apache-interceptor",
   "version": "1.0.0",
   "devDependencies": {
-    "aws-cdk": "2.58.1"
+    "aws-cdk": "2.61.1"
   },
   "author": "acm19"
 }
diff --git a/infra/requirements.txt b/infra/requirements.txt
@@ -1,3 +1,3 @@
-aws-cdk-lib==2.58.1
+aws-cdk-lib==2.61.1
 constructs==10.1.206
 types-setuptools==65.6.0.2
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 
   <groupId>io.github.acm19</groupId>
   <artifactId>aws-request-signing-apache-interceptor</artifactId>
-  <version>2.2.1-SNAPSHOT</version>
+  <version>2.3.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <name>AWS Request Signing Apache Interceptor</name>
diff --git a/src/main/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java b/src/main/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java
@@ -98,7 +98,11 @@ public void process(final HttpRequest request, final HttpContext context)
                 requestBuilder.contentStreamProvider(() -> new ByteArrayInputStream(outputStream.toByteArray()));
             }
         }
-        requestBuilder.headers(headerArrayToMap(request.getAllHeaders()));
+
+        Map<String, List<String>> headers = headerArrayToMap(request.getAllHeaders());
+        // adds a hash of the request payload when signing
+        headers.put("x-amz-content-sha256", Collections.singletonList("required"));
+        requestBuilder.headers(headers);
         SdkHttpFullRequest signedRequest = signer.signRequest(requestBuilder.build());
 
         // copy everything back
diff --git a/src/main/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheV5Interceptor.java b/src/main/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheV5Interceptor.java
@@ -82,7 +82,10 @@ public void process(final HttpRequest request, final EntityDetails entityDetails
             }
         }
 
-        requestBuilder.headers(headerArrayToMap(request.getHeaders()));
+        Map<String, List<String>> headers = headerArrayToMap(request.getHeaders());
+        // adds a hash of the request payload when signing
+        headers.put("x-amz-content-sha256", Collections.singletonList("required"));
+        requestBuilder.headers(headers);
         SdkHttpFullRequest signedRequest = signer.signRequest(requestBuilder.build());
 
         // copy everything back
diff --git a/src/test/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java b/src/test/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java
@@ -71,6 +71,7 @@ void signGetRequest() throws Exception {
         assertEquals("bar", request.getFirstHeader("foo").getValue());
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
         assertNull(request.getFirstHeader("content-length"));
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
     }
 
     @Test
@@ -96,6 +97,7 @@ void signPostRequest() throws Exception {
 
         assertEquals("bar", request.getFirstHeader("foo").getValue());
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
 
         assertEquals(Long.toString(httpEntity.getContentLength()),
                 request.getFirstHeader("signedContentLength").getValue());
@@ -143,6 +145,7 @@ void testEncodedUriSigner() throws Exception {
 
         assertEquals("bar", request.getFirstHeader("foo").getValue());
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
         assertNull(request.getFirstHeader("content-length"));
         assertEquals("/foo-2017-02-25%2Cfoo-2017-02-26/_search", request.getFirstHeader("resourcePath").getValue());
         assertEquals(Long.toString(data.length()), request.getFirstHeader("signedContentLength").getValue());
@@ -172,6 +175,7 @@ void testGzipCompressedContent() throws Exception {
         interceptor.process(request, context);
 
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
 
         assertEquals(Long.toString(entity.getContentLength()),
                 request.getFirstHeader("signedContentLength").getValue());
diff --git a/src/test/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheV5InterceptorTest.java b/src/test/java/io/github/acm19/aws/interceptor/http/AwsRequestSigningApacheV5InterceptorTest.java
@@ -65,6 +65,7 @@ void signGetRequest() throws Exception {
 
         assertEquals("bar", request.getFirstHeader("foo").getValue());
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
         assertNull(request.getFirstHeader("content-length"));
     }
 
@@ -83,6 +84,7 @@ void signPostRequest() throws Exception {
 
         assertEquals("bar", request.getFirstHeader("foo").getValue());
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
 
         assertEquals(Long.toString(httpEntity.getContentLength()),
                 request.getFirstHeader("signedContentLength").getValue());
@@ -114,6 +116,7 @@ void testEncodedUriSigner() throws Exception {
 
         assertEquals("bar", request.getFirstHeader("foo").getValue());
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
         assertNull(request.getFirstHeader("content-length"));
         assertEquals("/foo-2017-02-25%2Cfoo-2017-02-26/_search", request.getFirstHeader("resourcePath").getValue());
         assertEquals(Long.toString(data.length()), request.getFirstHeader("signedContentLength").getValue());
@@ -135,6 +138,7 @@ void testGzipCompressedContent() throws Exception {
         interceptor.process(request, ENTITY_DETAILS, null);
 
         assertEquals("wuzzle", request.getFirstHeader("Signature").getValue());
+        assertEquals("required", request.getFirstHeader("x-amz-content-sha256").getValue());
 
         assertEquals(Long.toString(entity.getContentLength()),
                 request.getFirstHeader("signedContentLength").getValue());
diff --git a/src/test/java/io/github/acm19/aws/interceptor/test/APIGatewaySample.java b/src/test/java/io/github/acm19/aws/interceptor/test/APIGatewaySample.java
@@ -14,7 +14,7 @@
 
 public class APIGatewaySample extends Sample {
     APIGatewaySample(final String[] args) throws ParseException {
-        super("execute-api", args);
+        super(args);
     }
 
     /**
diff --git a/src/test/java/io/github/acm19/aws/interceptor/test/AmazonOpenSearchServiceSample.java b/src/test/java/io/github/acm19/aws/interceptor/test/AmazonOpenSearchServiceSample.java
@@ -64,7 +64,7 @@
  */
 public class AmazonOpenSearchServiceSample extends Sample {
     AmazonOpenSearchServiceSample(final String[] args) throws ParseException {
-        super("es", args);
+        super(args);
     }
 
     /**
@@ -89,7 +89,10 @@ public static void main(final String[] args) throws IOException, ParseException
     }
 
     private void makeRequest() throws IOException {
-        logRequest(new HttpGet(endpoint));
+        // todo: https://github.com/acm19/aws-request-signing-apache-interceptor/issues/95
+        if (!service.equals("aoss")) {
+            logRequest(new HttpGet(endpoint));
+        }
     }
 
     private void createIndex() throws IOException {
@@ -107,15 +110,15 @@ private void deleteIndex() throws IOException {
 
     private void indexDocument() throws IOException {
         String payload = "{\"test\": \"val\"}";
-        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc/document_id");
+        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc");
         httpPost.setEntity(new StringEntity(payload));
         httpPost.addHeader("Content-Type", "application/json");
         logRequest(httpPost);
     }
 
     private void indexDocumentWithChunkedTransferEncoding() throws IOException {
         String payload = "{\"test\": \"val\"}";
-        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc/document_id");
+        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc");
         StringEntity entity = new StringEntity(payload);
         entity.setChunked(true);
         httpPost.setEntity(entity);
@@ -124,7 +127,7 @@ private void indexDocumentWithChunkedTransferEncoding() throws IOException {
     }
 
     private void indexDocumentWithCompressionEnabled() throws IOException {
-        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc/document_id");
+        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc");
         httpPost.setHeader(HttpHeaders.CONTENT_ENCODING, "gzip");
         httpPost.addHeader("Content-Type", "application/json");
         String payload = "{\"test\": \"val\"}";
@@ -140,7 +143,7 @@ private void indexDocumentWithCompressionEnabled() throws IOException {
     }
 
     private void indexDocumentWithChunkedTransferEncodingCompressionEnabled() throws IOException {
-        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc/document_id");
+        HttpPost httpPost = new HttpPost(endpoint + "/index_name/_doc");
         httpPost.setHeader(HttpHeaders.CONTENT_ENCODING, "gzip");
         httpPost.addHeader("Content-Type", "application/json");
         String payload = "{\"test\": \"val\"}";
diff --git a/src/test/java/io/github/acm19/aws/interceptor/test/Sample.java b/src/test/java/io/github/acm19/aws/interceptor/test/Sample.java
@@ -40,8 +40,7 @@ class Sample {
     protected Region region;
     protected String service;
 
-    Sample(final String service, final String[] args) throws ParseException {
-        this.service = service;
+    Sample(final String[] args) throws ParseException {
         parseOptions(args);
     }
 
@@ -52,20 +51,22 @@ class Sample {
     }
 
     public static void main(final String[] args) throws IOException, ParseException {
-        Sample sampleClass = new Sample("", args);
+        Sample sampleClass = new Sample(args);
         sampleClass.makeGetRequest();
         sampleClass.makePostRequest();
     }
 
     private void parseOptions(final String[] args) throws ParseException {
         Options options = new Options()
                 .addRequiredOption(null, "endpoint", true, "OpenSearch endpoint")
-                .addRequiredOption(null, "region", true, "AWS signing region");
+                .addRequiredOption(null, "region", true, "AWS signing region")
+                .addOption(null, "service", true, "AWS signing service, default is 'es'");
 
         try {
             CommandLine cmd = new DefaultParser().parse(options, args);
             this.endpoint = cmd.getOptionValue("endpoint");
             this.region = Region.of(cmd.getOptionValue("region"));
+            this.service = cmd.getOptionValue("service", "es");
         } catch (ParseException e) {
             System.err.println(e.getMessage());
             new HelpFormatter().printHelp(
diff --git a/src/test/java/io/github/acm19/aws/interceptorv5/test/AmazonOpenSearchServiceSample.java b/src/test/java/io/github/acm19/aws/interceptorv5/test/AmazonOpenSearchServiceSample.java
diff --git a/src/test/java/io/github/acm19/aws/interceptorv5/test/Sample.java b/src/test/java/io/github/acm19/aws/interceptorv5/test/Sample.java

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`"name": "aws-request-signing-apache-interceptor",`
`3`	`3`	`"version": "1.0.0",`
`4`	`4`	`"devDependencies": {`
`5`		`- "aws-cdk": "2.58.1"`
	`5`	`+ "aws-cdk": "2.61.1"`
`6`	`6`	`},`
`7`	`7`	`"author": "acm19"`
`8`	`8`	`}`
Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,11 @@ public void process(final HttpRequest request, final HttpContext context)`
`98`	`98`	`requestBuilder.contentStreamProvider(() -> new ByteArrayInputStream(outputStream.toByteArray()));`
`99`	`99`	`}`
`100`	`100`	`}`
`101`		`- requestBuilder.headers(headerArrayToMap(request.getAllHeaders()));`
	`101`	`+`
	`102`	`+ Map<String, List<String>> headers = headerArrayToMap(request.getAllHeaders());`
	`103`	`+ // adds a hash of the request payload when signing`
	`104`	`+ headers.put("x-amz-content-sha256", Collections.singletonList("required"));`
	`105`	`+ requestBuilder.headers(headers);`
`102`	`106`	`SdkHttpFullRequest signedRequest = signer.signRequest(requestBuilder.build());`
`103`	`107`
`104`	`108`	`// copy everything back`