From 4cdba069b781a8ba34bc3751bd9ac2f6b4c86db3 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 2 Dec 2024 16:19:05 +0530 Subject: [PATCH 1/7] Bump cdap and hadoop version --- pom.xml | 16 ++++++++++------ .../http/etl/HttpStreamingSourceETLTest.java | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 2fa12018..6c9bab2f 100644 --- a/pom.xml +++ b/pom.xml @@ -76,14 +76,14 @@ 3.1.6 - 6.8.0-SNAPSHOT + 6.11.0-SNAPSHOT 3.9 1.12 2.8.5 0.4.0 - 2.3.0 + 3.3.6 4.5.9 - 2.10.0-SNAPSHOT + 2.13.0-SNAPSHOT 2.12.0 2.9.9 4.11 @@ -150,6 +150,10 @@ org.slf4j slf4j-log4j12 + + org.slf4j + slf4j-reload4j + org.apache.avro avro @@ -410,19 +414,19 @@ io.cdap.cdap - cdap-data-streams2_2.11 + cdap-data-streams3_2.12 ${cdap.version} test io.cdap.cdap - cdap-data-pipeline2_2.11 + cdap-data-pipeline3_2.12 ${cdap.version} test io.cdap.cdap - cdap-spark-core2_2.11 + cdap-spark-core3_2.12 ${cdap.version} test diff --git a/src/test/java/io/cdap/plugin/http/etl/HttpStreamingSourceETLTest.java b/src/test/java/io/cdap/plugin/http/etl/HttpStreamingSourceETLTest.java index 9488a523..3e4c0534 100644 --- a/src/test/java/io/cdap/plugin/http/etl/HttpStreamingSourceETLTest.java +++ b/src/test/java/io/cdap/plugin/http/etl/HttpStreamingSourceETLTest.java @@ -56,11 +56,11 @@ public class HttpStreamingSourceETLTest extends HttpSourceETLTest { private static final ArtifactSummary APP_ARTIFACT = new ArtifactSummary("data-streams", "1.0.0"); private static final int WAIT_FOR_RECORDS_TIMEOUT_SECONDS = 60; private static final long WAIT_FOR_RECORDS_POLLING_INTERVAL_MS = 100; + public static final String EXPLORE_ENABLED = "explore.enabled"; @ClassRule public static final TestConfiguration CONFIG = - new TestConfiguration(Constants.Explore.EXPLORE_ENABLED, false, - Constants.AppFabric.SPARK_COMPAT, Compat.SPARK_COMPAT); + new TestConfiguration(EXPLORE_ENABLED, false, Constants.AppFabric.SPARK_COMPAT, Compat.SPARK_COMPAT); @BeforeClass public static void setupTest() throws Exception { From 1484037d58a1ce21fe59166c941c1687de1d0f8d Mon Sep 17 00:00:00 2001 From: Amit Kumar Singh Date: Wed, 4 Dec 2024 12:07:37 +0000 Subject: [PATCH 2/7] Added HTTP error detail provider and refactored Http-sink package to handle error provider and fix sonar issues --- .../http/common/HttpErrorDetailsProvider.java | 133 +++ .../http/sink/batch/HTTPOutputFormat.java | 101 +- .../http/sink/batch/HTTPRecordWriter.java | 559 ++++----- .../cdap/plugin/http/sink/batch/HTTPSink.java | 115 +- .../http/sink/batch/HTTPSinkConfig.java | 1028 ++++++++--------- .../plugin/http/sink/batch/MessageBuffer.java | 360 +++--- .../http/sink/batch/PlaceholderBean.java | 39 +- 7 files changed, 1253 insertions(+), 1082 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java diff --git a/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java new file mode 100644 index 00000000..293affaa --- /dev/null +++ b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java @@ -0,0 +1,133 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.http.common; + +import io.cdap.cdap.api.exception.*; +import io.cdap.cdap.etl.api.exception.ErrorContext; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; +import com.google.common.base.Throwables; +import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; +import io.cdap.cdap.api.exception.ProgramFailureException; +import io.cdap.cdap.etl.api.validation.InvalidConfigPropertyException; + +import java.io.*; +import java.util.List; + +import java.util.NoSuchElementException; + +public class HttpErrorDetailsProvider implements ErrorDetailsProvider { + @Override + public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { + List causalChain = Throwables.getCausalChain(e); + for (Throwable t : causalChain) { + //, UnsupportedOperationException, + if (t instanceof ProgramFailureException) { + // if causal chain already has program failure exception, return null to avoid double wrap. + return null; + } + if (t instanceof IllegalArgumentException) { + return getProgramFailureException((IllegalArgumentException) t, errorContext); + } + if (t instanceof IllegalStateException) { + return getProgramFailureException((IllegalStateException) t, errorContext); + } + if (t instanceof InvalidConfigPropertyException) { + return getProgramFailureException((InvalidConfigPropertyException) t, errorContext); + } + if (t instanceof NoSuchElementException) { + return getProgramFailureException((NoSuchElementException) t, errorContext); + } + if (t instanceof UnsupportedEncodingException) { + return getProgramFailureException((UnsupportedEncodingException) t, errorContext); + } + } + return null; + } + + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalArgumentException}. + * + * @param e The IllegalArgumentException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); + } + + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalStateException}. + * + * @param e The IllegalStateException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } + + /** + * Get a ProgramFailureException with the given error + * information from {@link InvalidConfigPropertyException}. + * + * @param e The InvalidConfigPropertyException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(InvalidConfigPropertyException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } + + /** + * Get a ProgramFailureException with the given error + * information from {@link NoSuchElementException}. + * + * @param e The NoSuchElementException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(NoSuchElementException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } + + + /** + * Get a ProgramFailureException with the given error + * information from {@link UnsupportedEncodingException}. + * + * @param e The UnsupportedEncodingException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(UnsupportedEncodingException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } + + +} diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java index 867d46be..ab3c2efc 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java @@ -19,6 +19,7 @@ import com.google.gson.Gson; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.OutputCommitter; @@ -32,51 +33,57 @@ * OutputFormat for HTTP writing */ public class HTTPOutputFormat extends OutputFormat { - private static final Gson GSON = new Gson(); - static final String CONFIG_KEY = "http.sink.config"; - static final String INPUT_SCHEMA_KEY = "http.sink.input.schema"; - - @Override - public RecordWriter getRecordWriter(TaskAttemptContext context) - throws IOException { - Configuration hConf = context.getConfiguration(); - HTTPSinkConfig config = GSON.fromJson(hConf.get(CONFIG_KEY), HTTPSinkConfig.class); - Schema inputSchema = Schema.parseJson(hConf.get(INPUT_SCHEMA_KEY)); - return new HTTPRecordWriter(config, inputSchema); - } - - @Override - public void checkOutputSpecs(JobContext jobContext) { - - } - - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext context) { - return new OutputCommitter() { - @Override - public void setupJob(JobContext jobContext) { - - } - - @Override - public void setupTask(TaskAttemptContext taskAttemptContext) { - - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) { - return false; - } - - @Override - public void commitTask(TaskAttemptContext taskAttemptContext) { - - } - - @Override - public void abortTask(TaskAttemptContext taskAttemptContext) { - - } - }; - } + private static final Gson GSON = new Gson(); + static final String CONFIG_KEY = "http.sink.config"; + static final String INPUT_SCHEMA_KEY = "http.sink.input.schema"; + + @Override + public RecordWriter getRecordWriter(TaskAttemptContext context) { + Configuration hConf = context.getConfiguration(); + HTTPSinkConfig config = GSON.fromJson(hConf.get(CONFIG_KEY), HTTPSinkConfig.class); + Schema inputSchema; + try { + inputSchema = Schema.parseJson(hConf.get(INPUT_SCHEMA_KEY)); + return new HTTPRecordWriter(config, inputSchema); + } catch (IOException e) { + String errorMessage = "Unable to parse and write the record"; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); + } + } + + @Override + public void checkOutputSpecs(JobContext jobContext) { + + } + + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) { + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) { + + } + + @Override + public void setupTask(TaskAttemptContext taskAttemptContext) { + + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) { + return false; + } + + @Override + public void commitTask(TaskAttemptContext taskAttemptContext) { + + } + + @Override + public void abortTask(TaskAttemptContext taskAttemptContext) { + + } + }; + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java index 2180737b..7f1c3d6a 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java @@ -21,6 +21,8 @@ import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.*; +import io.cdap.cdap.etl.api.exception.*; import io.cdap.plugin.http.common.RetryPolicy; import io.cdap.plugin.http.common.error.ErrorHandling; import io.cdap.plugin.http.common.error.HttpErrorHandler; @@ -59,6 +61,7 @@ import java.net.URI; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.X509Certificate; @@ -80,305 +83,323 @@ * RecordWriter for HTTP. */ public class HTTPRecordWriter extends RecordWriter { - private static final Logger LOG = LoggerFactory.getLogger(HTTPRecordWriter.class); - private static final String REGEX_HASHED_VAR = "#(\\w+)"; - public static final String REQUEST_METHOD_POST = "POST"; - public static final String REQUEST_METHOD_PUT = "PUT"; - public static final String REQUEST_METHOD_DELETE = "DELETE"; - public static final String REQUEST_METHOD_PATCH = "PATCH"; - - private final HTTPSinkConfig config; - private final MessageBuffer messageBuffer; - private String contentType; - private String url; - private String configURL; - private List placeHolderList; - private final Map headers; - - private AccessToken accessToken; - private final HttpErrorHandler httpErrorHandler; - private final PollInterval pollInterval; - private int httpStatusCode; - private String httpResponseBody; - private static int retryCount; - - HTTPRecordWriter(HTTPSinkConfig config, Schema inputSchema) { - this.headers = config.getRequestHeadersMap(); - this.config = config; - this.accessToken = null; - this.messageBuffer = new MessageBuffer( - config.getMessageFormat(), config.getJsonBatchKey(), config.shouldWriteJsonAsArray(), - config.getDelimiterForMessages(), config.getCharset(), config.getBody(), inputSchema - ); - this.httpErrorHandler = new HttpErrorHandler(config); - if (config.getRetryPolicy().equals(RetryPolicy.LINEAR)) { - pollInterval = FixedPollInterval.fixed(config.getLinearRetryInterval(), TimeUnit.SECONDS); - } else { - pollInterval = IterativePollInterval.iterative(duration -> duration.multiply(2), - Duration.FIVE_HUNDRED_MILLISECONDS); - } - url = config.getUrl(); - placeHolderList = getPlaceholderListFromURL(); - } - - @Override - public void write(StructuredRecord input, StructuredRecord unused) throws IOException { - configURL = url; - if (config.getMethod().equals(REQUEST_METHOD_POST) || config.getMethod().equals(REQUEST_METHOD_PUT) || - config.getMethod().equals(REQUEST_METHOD_PATCH)) { - messageBuffer.add(input); + private static final Logger LOG = LoggerFactory.getLogger(HTTPRecordWriter.class); + private static final String REGEX_HASHED_VAR = "#(\\w+)"; + public static final String REQUEST_METHOD_POST = "POST"; + public static final String REQUEST_METHOD_PUT = "PUT"; + public static final String REQUEST_METHOD_DELETE = "DELETE"; + public static final String REQUEST_METHOD_PATCH = "PATCH"; + + private final HTTPSinkConfig config; + private final MessageBuffer messageBuffer; + private String contentType; + private final String url; + private String configURL; + private final List placeHolderList; + private final Map headers; + + private AccessToken accessToken; + private final HttpErrorHandler httpErrorHandler; + private final PollInterval pollInterval; + private int httpStatusCode; + private String httpResponseBody; + private static int retryCount; + + HTTPRecordWriter(HTTPSinkConfig config, Schema inputSchema) { + this.headers = config.getRequestHeadersMap(); + this.config = config; + this.accessToken = null; + this.messageBuffer = new MessageBuffer( + config.getMessageFormat(), config.getJsonBatchKey(), config.shouldWriteJsonAsArray(), + config.getDelimiterForMessages(), config.getCharset(), config.getBody(), inputSchema + ); + this.httpErrorHandler = new HttpErrorHandler(config); + if (config.getRetryPolicy().equals(RetryPolicy.LINEAR)) { + pollInterval = FixedPollInterval.fixed(config.getLinearRetryInterval(), TimeUnit.SECONDS); + } else { + pollInterval = IterativePollInterval.iterative(duration -> duration.multiply(2), + Duration.FIVE_HUNDRED_MILLISECONDS); + } + url = config.getUrl(); + placeHolderList = getPlaceholderListFromURL(); } - if (config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || - config.getMethod().equals(REQUEST_METHOD_DELETE) - && !placeHolderList.isEmpty()) { - configURL = updateURLWithPlaceholderValue(input); - } + @Override + public void write(StructuredRecord input, StructuredRecord unused) { + configURL = url; + if (config.getMethod().equals(REQUEST_METHOD_POST) || config.getMethod().equals(REQUEST_METHOD_PUT) || + config.getMethod().equals(REQUEST_METHOD_PATCH)) { + messageBuffer.add(input); + } + + if (config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || + config.getMethod().equals(REQUEST_METHOD_DELETE) + && !placeHolderList.isEmpty()) { + configURL = updateURLWithPlaceholderValue(input); + } - if (config.getBatchSize() == messageBuffer.size() || config.getMethod().equals(REQUEST_METHOD_DELETE)) { - flushMessageBuffer(); + if (config.getBatchSize() == messageBuffer.size() || config.getMethod().equals(REQUEST_METHOD_DELETE)) { + flushMessageBuffer(); + } } - } - @Override - public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - // Process remaining messages after batch executions. - if (!config.getMethod().equals(REQUEST_METHOD_DELETE)) { - flushMessageBuffer(); + @Override + public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { + // Process remaining messages after batch executions. + if (!config.getMethod().equals(REQUEST_METHOD_DELETE)) { + flushMessageBuffer(); + } } - } - private void disableSSLValidation() { - TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } + private void disableSSLValidation() { + TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } - public void checkClientTrusted(X509Certificate[] certs, String authType) { - } + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } - public void checkServerTrusted(X509Certificate[] certs, String authType) { - } - } - }; - SSLContext sslContext = null; - try { - sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); - } catch (KeyManagementException | NoSuchAlgorithmException e) { - throw new IllegalStateException("Error while installing the trust manager: " + e.getMessage(), e); - } - HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); - HostnameVerifier allHostsValid = new HostnameVerifier() { - public boolean verify(String hostname, SSLSession session) { - return true; - } - }; - HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); - } - - private boolean executeHTTPServiceAndCheckStatusCode() throws IOException { - LOG.debug("HTTP Request Attempt No. : {}", ++retryCount); - CloseableHttpClient httpClient = createHttpClient(configURL); - - CloseableHttpResponse response = null; - try { - URL url = new URL(configURL); - HttpEntityEnclosingRequestBase request = new HttpRequest(URI.create(String.valueOf(url)), - config.getMethod()); - - if (url.getProtocol().equalsIgnoreCase("https")) { - // Disable SSLv3 - System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2"); - if (config.getDisableSSLValidation()) { - disableSSLValidation(); + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } } - } - - if (!messageBuffer.isEmpty()) { - String requestBodyString = messageBuffer.getMessage(); - if (requestBodyString != null) { - StringEntity requestBody = new StringEntity(requestBodyString, Charsets.UTF_8.toString()); - request.setEntity(requestBody); + }; + SSLContext sslContext = null; + try { + sslContext = SSLContext.getInstance("SSL"); + sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); + } catch (KeyManagementException | NoSuchAlgorithmException e) { + throw new IllegalStateException("Error while installing the trust manager: " + e.getMessage(), e); } - } - - request.setHeaders(getRequestHeaders()); - - response = httpClient.execute(request); - httpStatusCode = response.getStatusLine().getStatusCode(); - LOG.debug("Response HTTP Status code: {}", httpStatusCode); - httpResponseBody = new HttpResponse(response).getBody(); - - } catch (MalformedURLException | ProtocolException e) { - throw new IllegalStateException("Error opening url connection. Reason: " + e.getMessage(), e); - } catch (IOException e) { - LOG.warn("Error making {} request to url {}.", config.getMethod(), config.getUrl()); - } finally { - if (response != null) { - response.close(); - } + HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); + HostnameVerifier allHostsValid = (hostname, session) -> true; + HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); } - RetryableErrorHandling errorHandlingStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode); - boolean shouldRetry = errorHandlingStrategy.shouldRetry(); - if (!shouldRetry) { - messageBuffer.clear(); - retryCount = 0; + + private boolean executeHTTPServiceAndCheckStatusCode() { + LOG.debug("HTTP Request Attempt No. : {}", ++retryCount); + + // Try-with-resources ensures proper resource management + try (CloseableHttpClient httpClient = createHttpClient(configURL)) { + URL url = new URL(configURL); + + // Use try-with-resources to ensure response is closed + try (CloseableHttpResponse response = executeHttpRequest(httpClient, url)) { + httpStatusCode = response.getStatusLine().getStatusCode(); + LOG.debug("Response HTTP Status code: {}", httpStatusCode); + httpResponseBody = new HttpResponse(response).getBody(); + } + + RetryableErrorHandling errorHandlingStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode); + boolean shouldRetry = errorHandlingStrategy.shouldRetry(); + + if (!shouldRetry) { + messageBuffer.clear(); + retryCount = 0; + } + return !shouldRetry; + + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Invalid URL: " + configURL, e); + } catch (IOException e) { + LOG.warn("Error making {} request to URL {}.", config.getMethod(), config.getUrl()); + String errorMessage = "Unable to make request. "; + throw ErrorUtils.getProgramFailureException( + new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, e); + } } - return !shouldRetry; - } - - - public CloseableHttpClient createHttpClient(String pageUriStr) throws IOException { - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); - - // set timeouts - Long connectTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getConnectTimeout()); - Long readTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getReadTimeout()); - RequestConfig.Builder requestBuilder = RequestConfig.custom(); - requestBuilder.setSocketTimeout(readTimeoutMillis.intValue()); - requestBuilder.setConnectTimeout(connectTimeoutMillis.intValue()); - requestBuilder.setConnectionRequestTimeout(connectTimeoutMillis.intValue()); - httpClientBuilder.setDefaultRequestConfig(requestBuilder.build()); - - // basic auth - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - if (!Strings.isNullOrEmpty(config.getUsername()) && !Strings.isNullOrEmpty(config.getPassword())) { - URI uri = URI.create(pageUriStr); - AuthScope authScope = new AuthScope(new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme())); - credentialsProvider.setCredentials(authScope, - new UsernamePasswordCredentials(config.getUsername(), config.getPassword())); + + private CloseableHttpResponse executeHttpRequest(CloseableHttpClient httpClient, URL url) { + try { + HttpEntityEnclosingRequestBase request = new HttpRequest(URI.create(url.toString()), config.getMethod()); + + if ("https".equalsIgnoreCase(url.getProtocol())) { + configureHttpsSettings(); + } + + if (!messageBuffer.isEmpty()) { + String requestBodyString = messageBuffer.getMessage(); + if (requestBodyString != null) { + StringEntity requestBody = new StringEntity(requestBodyString, StandardCharsets.UTF_8.name()); + request.setEntity(requestBody); + } + } + + request.setHeaders(getRequestHeaders()); + + // Execute the request and return the response + return httpClient.execute(request); + + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Error encoding the request Reason: " + e.getMessage(), e); + } catch (IOException e) { + String errorMessage = String.format("Unable to execute HTTP request to %s.", url); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); + } catch (Exception e) { + String errorMessage = String.format("Unexpected error occurred while executing HTTP request to URL: %s", url); + throw ErrorUtils.getProgramFailureException( + new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), errorMessage, + errorMessage, ErrorType.UNKNOWN, true, e); + } } - // proxy and proxy auth - if (!Strings.isNullOrEmpty(config.getProxyUrl())) { - HttpHost proxyHost = HttpHost.create(config.getProxyUrl()); - if (!Strings.isNullOrEmpty(config.getProxyUsername()) && !Strings.isNullOrEmpty(config.getProxyPassword())) { - credentialsProvider.setCredentials(new AuthScope(proxyHost), - new UsernamePasswordCredentials( - config.getProxyUsername(), config.getProxyPassword())); - } - httpClientBuilder.setProxy(proxyHost); + private void configureHttpsSettings() { + System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2"); + if (Boolean.TRUE.equals(config.getDisableSSLValidation())) { + disableSSLValidation(); + } } - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - return httpClientBuilder.build(); - } + public CloseableHttpClient createHttpClient(String pageUriStr) { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + + // set timeouts + long connectTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getConnectTimeout()); + long readTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getReadTimeout()); + RequestConfig.Builder requestBuilder = RequestConfig.custom(); + requestBuilder.setSocketTimeout((int) readTimeoutMillis); + requestBuilder.setConnectTimeout((int) connectTimeoutMillis); + requestBuilder.setConnectionRequestTimeout((int) connectTimeoutMillis); + httpClientBuilder.setDefaultRequestConfig(requestBuilder.build()); + + // basic auth + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + if (!Strings.isNullOrEmpty(config.getUsername()) && !Strings.isNullOrEmpty(config.getPassword())) { + URI uri = URI.create(pageUriStr); + AuthScope authScope = new AuthScope(new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme())); + credentialsProvider.setCredentials(authScope, + new UsernamePasswordCredentials(config.getUsername(), config.getPassword())); + } - private Header[] getRequestHeaders() throws IOException { - ArrayList
clientHeaders = new ArrayList<>(); + // proxy and proxy auth + if (!Strings.isNullOrEmpty(config.getProxyUrl())) { + HttpHost proxyHost = HttpHost.create(config.getProxyUrl()); + if (!Strings.isNullOrEmpty(config.getProxyUsername()) && !Strings.isNullOrEmpty(config.getProxyPassword())) { + credentialsProvider.setCredentials(new AuthScope(proxyHost), + new UsernamePasswordCredentials( + config.getProxyUsername(), config.getProxyPassword())); + } + httpClientBuilder.setProxy(proxyHost); + } + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - if (accessToken == null || OAuthUtil.tokenExpired(accessToken)) { - accessToken = OAuthUtil.getAccessToken(config); + return httpClientBuilder.build(); } - if (accessToken != null) { - Header authorizationHeader = getAuthorizationHeader(accessToken); - if (authorizationHeader != null) { - clientHeaders.add(authorizationHeader); - } - } + private Header[] getRequestHeaders() throws IOException { + ArrayList
clientHeaders = new ArrayList<>(); - headers.put("Request-Method", config.getMethod().toUpperCase()); - headers.put("Instance-Follow-Redirects", String.valueOf(config.getFollowRedirects())); - headers.put("charset", config.getCharset()); + if (accessToken == null || OAuthUtil.tokenExpired(accessToken)) { + accessToken = OAuthUtil.getAccessToken(config); + } - if (config.getMethod().equals(REQUEST_METHOD_POST) - || config.getMethod().equals(REQUEST_METHOD_PATCH) - || config.getMethod().equals(REQUEST_METHOD_PUT)) { - if (!headers.containsKey("Content-Type")) { - headers.put("Content-Type", contentType); - } - } + if (accessToken != null) { + Header authorizationHeader = getAuthorizationHeader(accessToken); + clientHeaders.add(authorizationHeader); + } - // set default headers - if (headers != null) { - for (Map.Entry headerEntry : this.headers.entrySet()) { - clientHeaders.add(new BasicHeader(headerEntry.getKey(), headerEntry.getValue())); - } - } + headers.put("Request-Method", config.getMethod().toUpperCase()); + headers.put("Instance-Follow-Redirects", String.valueOf(config.getFollowRedirects())); + headers.put("charset", config.getCharset()); - return clientHeaders.toArray(new Header[clientHeaders.size()]); - } - - private Header getAuthorizationHeader(AccessToken accessToken) { - return new BasicHeader("Authorization", String.format("Bearer %s", accessToken.getTokenValue())); - } - - /** - * @return List of placeholders which should be replaced by actual value in the URL. - */ - private List getPlaceholderListFromURL() { - List placeholderList = new ArrayList<>(); - if (!(config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || - config.getMethod().equals(REQUEST_METHOD_DELETE))) { - return placeholderList; - } - Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); - Matcher matcher = pattern.matcher(url); - while (matcher.find()) { - placeholderList.add(new PlaceholderBean(url, matcher.group(1))); - } - return placeholderList; // Return blank list if no match found - } - - private String updateURLWithPlaceholderValue(StructuredRecord inputRecord) { - try { - StringBuilder finalURLBuilder = new StringBuilder(url); - //Running a loop backwards so that it does not impact the start and end index for next record. - for (int i = placeHolderList.size() - 1; i >= 0; i--) { - PlaceholderBean key = placeHolderList.get(i); - String replacement = inputRecord.get(key.getPlaceHolderKey()); - if (replacement != null) { - String encodedReplacement = URLEncoder.encode(replacement, config.getCharset()); - finalURLBuilder.replace(key.getStartIndex(), key.getEndIndex(), encodedReplacement); + if ((config.getMethod().equals(REQUEST_METHOD_POST) + || config.getMethod().equals(REQUEST_METHOD_PATCH) + || config.getMethod().equals(REQUEST_METHOD_PUT)) && !headers.containsKey("Content-Type")) { + headers.put("Content-Type", contentType); } - } - return finalURLBuilder.toString(); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Error encoding URL with placeholder value. Reason: " + e.getMessage(), e); + + + // set default headers + if (headers != null) { + for (Map.Entry headerEntry : this.headers.entrySet()) { + clientHeaders.add(new BasicHeader(headerEntry.getKey(), headerEntry.getValue())); + } + } + + return clientHeaders.toArray(new Header[clientHeaders.size()]); } - } - - /** - * Clears the message buffer if it is empty and the HTTP method is not 'DELETE'. - */ - private void flushMessageBuffer() { - if (messageBuffer.isEmpty() && !config.getMethod().equals(REQUEST_METHOD_DELETE)) { - return; + + private Header getAuthorizationHeader(AccessToken accessToken) { + return new BasicHeader("Authorization", String.format("Bearer %s", accessToken.getTokenValue())); } - contentType = messageBuffer.getContentType(); - try { - Awaitility - .await().with() - .pollInterval(pollInterval) - .pollDelay(config.getWaitTimeBetweenPages(), TimeUnit.MILLISECONDS) - .timeout(config.getMaxRetryDuration(), TimeUnit.SECONDS) - .until(this::executeHTTPServiceAndCheckStatusCode); - } catch (Exception e) { - throw new RuntimeException("Error while executing http request for remaining input messages " + - "after the batch execution. " + e); + + /** + * @return List of placeholders which should be replaced by actual value in the URL. + */ + private List getPlaceholderListFromURL() { + List placeholderList = new ArrayList<>(); + if (!(config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || + config.getMethod().equals(REQUEST_METHOD_DELETE))) { + return placeholderList; + } + Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); + Matcher matcher = pattern.matcher(url); + while (matcher.find()) { + placeholderList.add(new PlaceholderBean(url, matcher.group(1))); + } + return placeholderList; // Return blank list if no match found } - messageBuffer.clear(); - - ErrorHandling postRetryStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode) - .getAfterRetryStrategy(); - - switch (postRetryStrategy) { - case SUCCESS: - break; - case STOP: - throw new IllegalStateException(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", - config.getUrl(), httpStatusCode, httpResponseBody)); - case SKIP: - case SEND: - LOG.warn(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", - config.getUrl(), httpStatusCode, httpResponseBody)); - break; - default: - throw new IllegalArgumentException(String.format("Unexpected http error handling: '%s'", postRetryStrategy)); + + private String updateURLWithPlaceholderValue(StructuredRecord inputRecord) { + try { + StringBuilder finalURLBuilder = new StringBuilder(url); + //Running a loop backwards so that it does not impact the start and end index for next record. + for (int i = placeHolderList.size() - 1; i >= 0; i--) { + PlaceholderBean key = placeHolderList.get(i); + String replacement = inputRecord.get(key.getPlaceHolderKey()); + if (replacement != null) { + String encodedReplacement = URLEncoder.encode(replacement, config.getCharset()); + finalURLBuilder.replace(key.getStartIndex(), key.getEndIndex(), encodedReplacement); + } + } + return finalURLBuilder.toString(); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Error encoding URL with placeholder value. Reason: " + e.getMessage(), e); + } } - } + /** + * Clears the message buffer if it is empty and the HTTP method is not 'DELETE'. + */ + private void flushMessageBuffer() { + if (messageBuffer.isEmpty() && !config.getMethod().equals(REQUEST_METHOD_DELETE)) { + return; + } + contentType = messageBuffer.getContentType(); + try { + Awaitility + .await().with() + .pollInterval(pollInterval) + .pollDelay(config.getWaitTimeBetweenPages(), TimeUnit.MILLISECONDS) + .timeout(config.getMaxRetryDuration(), TimeUnit.SECONDS) + .until(this::executeHTTPServiceAndCheckStatusCode); + } catch (Exception e) { + String errorMessage = "Error while executing http request for remaining input messages after the batch execution."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new RuntimeException(errorMessage)); + } + messageBuffer.clear(); + + ErrorHandling postRetryStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode) + .getAfterRetryStrategy(); + + switch (postRetryStrategy) { + case SUCCESS: + break; + case STOP: + throw new IllegalStateException(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", + config.getUrl(), httpStatusCode, httpResponseBody)); + case SKIP: + case SEND: + LOG.warn(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", + config.getUrl(), httpStatusCode, httpResponseBody)); + break; + default: + throw new IllegalArgumentException(String.format("Unexpected http error handling: '%s'", postRetryStrategy)); + } + + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java index 7f45b73d..8d2aab31 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java @@ -32,6 +32,8 @@ import io.cdap.cdap.etl.api.batch.BatchSinkContext; import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; +import io.cdap.plugin.http.common.HttpErrorDetailsProvider; import java.util.Collections; import java.util.List; @@ -45,66 +47,75 @@ @Name("HTTP") @Description("Sink plugin to send the messages from the pipeline to an external http endpoint.") public class HTTPSink extends BatchSink { - private HTTPSinkConfig config; - - public HTTPSink(HTTPSinkConfig config) { - this.config = config; - } - - @Override - public void configurePipeline(PipelineConfigurer pipelineConfigurer) { - super.configurePipeline(pipelineConfigurer); - StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); - FailureCollector collector = stageConfigurer.getFailureCollector(); - config.validate(collector); - config.validateSchema(stageConfigurer.getInputSchema(), collector); - collector.getOrThrowException(); - } - - @Override - public void prepareRun(BatchSinkContext context) { - FailureCollector collector = context.getFailureCollector(); - config.validate(collector); - config.validateSchema(context.getInputSchema(), collector); - collector.getOrThrowException(); - - Schema inputSchema = context.getInputSchema(); - Asset asset = Asset.builder(config.getReferenceNameOrNormalizedFQN()) - .setFqn(config.getUrl()).build(); - LineageRecorder lineageRecorder = new LineageRecorder(context, asset); - lineageRecorder.createExternalDataset(context.getInputSchema()); - List fields = inputSchema == null ? - Collections.emptyList() : - inputSchema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList()); - lineageRecorder.recordWrite("Write", String.format("Wrote to HTTP '%s'", config.getUrl()), fields); - - context.addOutput(Output.of(config.getReferenceNameOrNormalizedFQN(), - new HTTPSink.HTTPOutputFormatProvider(config, inputSchema))); - } - - /** - * Output format provider for HTTP Sink. - */ - private static class HTTPOutputFormatProvider implements OutputFormatProvider { - private static final Gson GSON = new Gson(); private final HTTPSinkConfig config; - private final Schema inputSchema; - HTTPOutputFormatProvider(HTTPSinkConfig config, Schema inputSchema) { - this.config = config; - this.inputSchema = inputSchema; + public HTTPSink(HTTPSinkConfig config) { + this.config = config; } @Override - public String getOutputFormatClassName() { - return HTTPOutputFormat.class.getName(); + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); + FailureCollector collector = stageConfigurer.getFailureCollector(); + config.validate(collector); + config.validateSchema(stageConfigurer.getInputSchema(), collector); + collector.getOrThrowException(); } @Override - public Map getOutputFormatConfiguration() { - return ImmutableMap.of("http.sink.config", GSON.toJson(config), - "http.sink.input.schema", inputSchema == null ? "" : inputSchema.toString()); + public void prepareRun(BatchSinkContext context) { + FailureCollector collector = context.getFailureCollector(); + config.validate(collector); + config.validateSchema(context.getInputSchema(), collector); + collector.getOrThrowException(); + + Schema inputSchema = context.getInputSchema(); + Asset asset = Asset.builder(config.getReferenceNameOrNormalizedFQN()) + .setFqn(config.getUrl()).build(); + LineageRecorder lineageRecorder = new LineageRecorder(context, asset); + lineageRecorder.createExternalDataset(context.getInputSchema()); + List fields; + if (inputSchema == null) { + fields = Collections.emptyList(); + } else { + assert inputSchema.getFields() != null; + fields = inputSchema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList()); + } + lineageRecorder.recordWrite("Write", String.format("Wrote to HTTP '%s'", config.getUrl()), fields); + + context.addOutput(Output.of(config.getReferenceNameOrNormalizedFQN(), + new HTTPSink.HTTPOutputFormatProvider(config, inputSchema))); + + // set error details provider + context.setErrorDetailsProvider( + new ErrorDetailsProviderSpec(HttpErrorDetailsProvider.class.getName())); + + } + + /** + * Output format provider for HTTP Sink. + */ + private static class HTTPOutputFormatProvider implements OutputFormatProvider { + private static final Gson GSON = new Gson(); + private final HTTPSinkConfig config; + private final Schema inputSchema; + + HTTPOutputFormatProvider(HTTPSinkConfig config, Schema inputSchema) { + this.config = config; + this.inputSchema = inputSchema; + } + + @Override + public String getOutputFormatClassName() { + return HTTPOutputFormat.class.getName(); + } + + @Override + public Map getOutputFormatConfiguration() { + return ImmutableMap.of("http.sink.config", GSON.toJson(config), + "http.sink.input.schema", inputSchema == null ? "" : inputSchema.toString()); + } } - } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java index cb3cc9ff..c746a2b9 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java @@ -56,573 +56,573 @@ * Config class for {@link HTTPSink}. */ public class HTTPSinkConfig extends BaseHttpConfig { - public static final String URL = "url"; - public static final String METHOD = "method"; - public static final String BATCH_SIZE = "batchSize"; - public static final String WRITE_JSON_AS_ARRAY = "writeJsonAsArray"; - public static final String JSON_BATCH_KEY = "jsonBatchKey"; - public static final String DELIMETER_FOR_MESSAGE = "delimiterForMessages"; - public static final String MESSAGE_FORMAT = "messageFormat"; - public static final String BODY = "body"; - public static final String REQUEST_HEADERS = "requestHeaders"; - public static final String CHARSET = "charset"; - public static final String FOLLOW_REDIRECTS = "followRedirects"; - public static final String DISABLE_SSL_VALIDATION = "disableSSLValidation"; - public static final String PROPERTY_HTTP_ERROR_HANDLING = "httpErrorsHandling"; - public static final String PROPERTY_ERROR_HANDLING = "errorHandling"; - public static final String PROPERTY_RETRY_POLICY = "retryPolicy"; - public static final String PROPERTY_LINEAR_RETRY_INTERVAL = "linearRetryInterval"; - public static final String PROPERTY_MAX_RETRY_DURATION = "maxRetryDuration"; - public static final String CONNECTION_TIMEOUT = "connectTimeout"; - public static final String READ_TIMEOUT = "readTimeout"; - private static final String KV_DELIMITER = ":"; - private static final String DELIMITER = "\n"; - private static final String REGEX_HASHED_VAR = "#(\\w+)"; - private static final String PLACEHOLDER = "#"; - private static final Set METHODS = ImmutableSet.of(HttpMethod.GET, HttpMethod.POST, - HttpMethod.PUT, HttpMethod.DELETE, "PATCH"); - - @Name(URL) - @Description("The URL to post data to. Additionally, a placeholder like #columnName can be added to the URL that " + - "can be substituted with column value at the runtime. E.g. https://customer-url/user/#user_id. Here user_id " + - "column should exist in input schema. (Macro Enabled)") - @Macro - private final String url; - - @Name(METHOD) - @Description("The http request method. Defaults to POST. (Macro Enabled)") - @Macro - private final String method; - - @Name(BATCH_SIZE) - @Description("Batch size. Defaults to 1. (Macro Enabled)") - @Macro - private final Integer batchSize; - - @Name(WRITE_JSON_AS_ARRAY) - @Nullable - @Description("Whether to write json as array. Defaults to false. (Macro Enabled)") - @Macro - private final Boolean writeJsonAsArray; - - @Name(JSON_BATCH_KEY) - @Nullable - @Description("Optional key to be used for wrapping json array as object. " + - "Leave empty for no wrapping of the array (Macro Enabled)") - @Macro - private final String jsonBatchKey; - - @Name(DELIMETER_FOR_MESSAGE) - @Nullable - @Description("Delimiter for messages to be used while batching. Defaults to \"\\n\". (Macro Enabled)") - @Macro - private final String delimiterForMessages; - - @Name(MESSAGE_FORMAT) - @Description("Format to send messsage in. (Macro Enabled)") - @Macro - private final String messageFormat; - - @Name(BODY) - @Nullable - @Description("Optional custom message. This is required if the message format is set to 'Custom'." + - "User can leverage incoming message fields in the post payload. For example-" + - "User has defined payload as \\{ \"messageType\" : \"update\", \"name\" : \"#firstName\" \\}" + - "where #firstName will be substituted for the value that is in firstName in the incoming message. " + - "(Macro enabled)") - @Macro - private final String body; - - @Name(REQUEST_HEADERS) - @Nullable - @Description("Request headers to set when performing the http request. (Macro enabled)") - @Macro - private final String requestHeaders; - - @Name(CHARSET) - @Description("Charset. Defaults to UTF-8. (Macro enabled)") - @Macro - private final String charset; - - @Name(FOLLOW_REDIRECTS) - @Description("Whether to automatically follow redirects. Defaults to true. (Macro enabled)") - @Macro - private final Boolean followRedirects; - - @Name(DISABLE_SSL_VALIDATION) - @Description("If user enables SSL validation, they will be expected to add the certificate to the trustStore" + - " on each machine. Defaults to true. (Macro enabled)") - @Macro - private final Boolean disableSSLValidation; - - @Nullable - @Name(PROPERTY_HTTP_ERROR_HANDLING) - @Description("Defines the error handling strategy to use for certain HTTP response codes." + - "The left column contains a regular expression for HTTP status code. The right column contains an action which" + - "is done in case of match. If HTTP status code matches multiple regular expressions, " + - "the first specified in mapping is matched.") - protected String httpErrorsHandling; - - @Nullable - @Name(PROPERTY_ERROR_HANDLING) - @Description("Error handling strategy to use when the HTTP response cannot be transformed to an output record.") - protected String errorHandling; - - @Nullable - @Name(PROPERTY_RETRY_POLICY) - @Description("Policy used to calculate delay between retries. Default Retry Policy is Exponential.") - protected String retryPolicy; - - @Nullable - @Name(PROPERTY_LINEAR_RETRY_INTERVAL) - @Description("Interval in seconds between retries. Is only used if retry policy is \"linear\".") - @Macro - protected Long linearRetryInterval; - - @Nullable - @Name(PROPERTY_MAX_RETRY_DURATION) - @Description("Maximum time in seconds retries can take. Default value is 600 seconds (10 minute).") - @Macro - protected Long maxRetryDuration; - - @Name(CONNECTION_TIMEOUT) - @Description("Sets the connection timeout in milliseconds. Set to 0 for infinite. Default is 60000 (1 minute). " + - "(Macro enabled)") - @Nullable - @Macro - private final Integer connectTimeout; - - @Name(READ_TIMEOUT) - @Description("The time in milliseconds to wait for a read. Set to 0 for infinite. Defaults to 60000 (1 minute). " + - "(Macro enabled)") - @Nullable - @Macro - private final Integer readTimeout; - - public HTTPSinkConfig(String referenceName, String url, String method, Integer batchSize, - @Nullable String delimiterForMessages, String messageFormat, @Nullable String body, - @Nullable String requestHeaders, String charset, - boolean followRedirects, boolean disableSSLValidation, @Nullable String httpErrorsHandling, - String errorHandling, String retryPolicy, @Nullable Long linearRetryInterval, - Long maxRetryDuration, @Nullable int readTimeout, @Nullable int connectTimeout, - String oauth2Enabled, String authType, @Nullable String jsonBatchKey, - Boolean writeJsonAsArray) { - super(referenceName); - this.url = url; - this.method = method; - this.batchSize = batchSize; - this.delimiterForMessages = delimiterForMessages; - this.messageFormat = messageFormat; - this.body = body; - this.requestHeaders = requestHeaders; - this.charset = charset; - this.followRedirects = followRedirects; - this.disableSSLValidation = disableSSLValidation; - this.httpErrorsHandling = httpErrorsHandling; - this.errorHandling = errorHandling; - this.retryPolicy = retryPolicy; - this.linearRetryInterval = linearRetryInterval; - this.maxRetryDuration = maxRetryDuration; - this.readTimeout = readTimeout; - this.connectTimeout = connectTimeout; - this.jsonBatchKey = jsonBatchKey; - this.writeJsonAsArray = writeJsonAsArray; - this.oauth2Enabled = oauth2Enabled; - this.authType = authType; - } - - private HTTPSinkConfig(Builder builder) { - super(builder.referenceName); - url = builder.url; - method = builder.method; - batchSize = builder.batchSize; - delimiterForMessages = builder.delimiterForMessages; - messageFormat = builder.messageFormat; - body = builder.body; - requestHeaders = builder.requestHeaders; - charset = builder.charset; - followRedirects = builder.followRedirects; - disableSSLValidation = builder.disableSSLValidation; - connectTimeout = builder.connectTimeout; - readTimeout = builder.readTimeout; - jsonBatchKey = builder.jsonBatchKey; - writeJsonAsArray = builder.writeJsonAsArray; - oauth2Enabled = builder.oauth2Enabled; - authType = builder.authType; - } - - public static Builder newBuilder() { - return new Builder(); - } - - public static Builder newBuilder(HTTPSinkConfig copy) { - Builder builder = new Builder(); - builder.referenceName = copy.referenceName; - builder.url = copy.getUrl(); - builder.method = copy.getMethod(); - builder.batchSize = copy.getBatchSize(); - builder.delimiterForMessages = copy.getDelimiterForMessages(); - builder.messageFormat = copy.getMessageFormat().getValue(); - builder.body = copy.getBody(); - builder.requestHeaders = copy.getRequestHeaders(); - builder.charset = copy.getCharset(); - builder.followRedirects = copy.getFollowRedirects(); - builder.disableSSLValidation = copy.getDisableSSLValidation(); - builder.connectTimeout = copy.getConnectTimeout(); - builder.readTimeout = copy.getReadTimeout(); - builder.oauth2Enabled = copy.getOAuth2Enabled(); - builder.authType = copy.getAuthTypeString(); - return builder; - } - - public String getUrl() { - return url; - } - - public String getMethod() { - return method; - } - - public Integer getBatchSize() { - return batchSize; - } - - public boolean shouldWriteJsonAsArray() { - return writeJsonAsArray != null && writeJsonAsArray; - } - - public String getJsonBatchKey() { - return jsonBatchKey; - } - - @Nullable - public String getDelimiterForMessages() { - return Strings.isNullOrEmpty(delimiterForMessages) ? "\n" : delimiterForMessages; - } - - public MessageFormatType getMessageFormat() { - return MessageFormatType.valueOf(messageFormat.toUpperCase()); - } - - @Nullable - public String getBody() { - return body; - } - - @Nullable - public String getRequestHeaders() { - return requestHeaders; - } - - public String getCharset() { - return charset; - } - - public Boolean getFollowRedirects() { - return followRedirects; - } - - public Boolean getDisableSSLValidation() { - return disableSSLValidation; - } - - @Nullable - public String getHttpErrorsHandling() { - return httpErrorsHandling; - } - - public ErrorHandling getErrorHandling() { - return getEnumValueByString(ErrorHandling.class, errorHandling, PROPERTY_ERROR_HANDLING); - } - - public RetryPolicy getRetryPolicy() { - if (retryPolicy == null) { - return RetryPolicy.EXPONENTIAL; - } - return getEnumValueByString(RetryPolicy.class, retryPolicy, PROPERTY_RETRY_POLICY); - } - - private static T - getEnumValueByString(Class enumClass, String stringValue, String propertyName) { - return Stream.of(enumClass.getEnumConstants()) - .filter(keyType -> keyType.getValue().equalsIgnoreCase(stringValue)) - .findAny() - .orElseThrow(() -> new InvalidConfigPropertyException( - String.format("Unsupported value for '%s': '%s'", propertyName, stringValue), propertyName)); - } - - @Nullable - public Long getLinearRetryInterval() { - return linearRetryInterval; - } - - public Long getMaxRetryDuration() { - if (maxRetryDuration == null) { - return 600L; - } - return maxRetryDuration; - } - - @Nullable - public Integer getConnectTimeout() { - return connectTimeout; - } - - @Nullable - public Integer getReadTimeout() { - return readTimeout; - } - - public Map getRequestHeadersMap() { - return convertHeadersToMap(requestHeaders); - } - - public Map getHeadersMap(String header) { - return convertHeadersToMap(header); - } - - public String getReferenceNameOrNormalizedFQN() { - return Strings.isNullOrEmpty(referenceName) ? ReferenceNames.normalizeFqn(url) : referenceName; - } - - public List getHttpErrorHandlingEntries() { - Map httpErrorsHandlingMap = getMapFromKeyValueString(httpErrorsHandling); - List results = new ArrayList<>(httpErrorsHandlingMap.size()); - - for (Map.Entry entry : httpErrorsHandlingMap.entrySet()) { - String regex = entry.getKey(); - try { - results.add(new HttpErrorHandlerEntity(Pattern.compile(regex), - getEnumValueByString(RetryableErrorHandling.class, - entry.getValue(), PROPERTY_HTTP_ERROR_HANDLING))); - } catch (PatternSyntaxException e) { - // We embed causing exception message into this one. Since this message is shown on UI when validation fails. - throw new InvalidConfigPropertyException( - String.format( - "Error handling regex '%s' is not valid. %s", regex, e.getMessage()), PROPERTY_HTTP_ERROR_HANDLING); - } + public static final String URL = "url"; + public static final String METHOD = "method"; + public static final String BATCH_SIZE = "batchSize"; + public static final String WRITE_JSON_AS_ARRAY = "writeJsonAsArray"; + public static final String JSON_BATCH_KEY = "jsonBatchKey"; + public static final String DELIMETER_FOR_MESSAGE = "delimiterForMessages"; + public static final String MESSAGE_FORMAT = "messageFormat"; + public static final String BODY = "body"; + public static final String REQUEST_HEADERS = "requestHeaders"; + public static final String CHARSET = "charset"; + public static final String FOLLOW_REDIRECTS = "followRedirects"; + public static final String DISABLE_SSL_VALIDATION = "disableSSLValidation"; + public static final String PROPERTY_HTTP_ERROR_HANDLING = "httpErrorsHandling"; + public static final String PROPERTY_ERROR_HANDLING = "errorHandling"; + public static final String PROPERTY_RETRY_POLICY = "retryPolicy"; + public static final String PROPERTY_LINEAR_RETRY_INTERVAL = "linearRetryInterval"; + public static final String PROPERTY_MAX_RETRY_DURATION = "maxRetryDuration"; + public static final String CONNECTION_TIMEOUT = "connectTimeout"; + public static final String READ_TIMEOUT = "readTimeout"; + private static final String KV_DELIMITER = ":"; + private static final String DELIMITER = "\n"; + private static final String REGEX_HASHED_VAR = "#(\\w+)"; + private static final String PLACEHOLDER = "#"; + private static final Set METHODS = ImmutableSet.of(HttpMethod.GET, HttpMethod.POST, + HttpMethod.PUT, HttpMethod.DELETE, "PATCH"); + + @Name(URL) + @Description("The URL to post data to. Additionally, a placeholder like #columnName can be added to the URL that " + + "can be substituted with column value at the runtime. E.g. https://customer-url/user/#user_id. Here user_id " + + "column should exist in input schema. (Macro Enabled)") + @Macro + private final String url; + + @Name(METHOD) + @Description("The http request method. Defaults to POST. (Macro Enabled)") + @Macro + private final String method; + + @Name(BATCH_SIZE) + @Description("Batch size. Defaults to 1. (Macro Enabled)") + @Macro + private final Integer batchSize; + + @Name(WRITE_JSON_AS_ARRAY) + @Nullable + @Description("Whether to write json as array. Defaults to false. (Macro Enabled)") + @Macro + private final Boolean writeJsonAsArray; + + @Name(JSON_BATCH_KEY) + @Nullable + @Description("Optional key to be used for wrapping json array as object. " + + "Leave empty for no wrapping of the array (Macro Enabled)") + @Macro + private final String jsonBatchKey; + + @Name(DELIMETER_FOR_MESSAGE) + @Nullable + @Description("Delimiter for messages to be used while batching. Defaults to \"\\n\". (Macro Enabled)") + @Macro + private final String delimiterForMessages; + + @Name(MESSAGE_FORMAT) + @Description("Format to send messsage in. (Macro Enabled)") + @Macro + private final String messageFormat; + + @Name(BODY) + @Nullable + @Description("Optional custom message. This is required if the message format is set to 'Custom'." + + "User can leverage incoming message fields in the post payload. For example-" + + "User has defined payload as \\{ \"messageType\" : \"update\", \"name\" : \"#firstName\" \\}" + + "where #firstName will be substituted for the value that is in firstName in the incoming message. " + + "(Macro enabled)") + @Macro + private final String body; + + @Name(REQUEST_HEADERS) + @Nullable + @Description("Request headers to set when performing the http request. (Macro enabled)") + @Macro + private final String requestHeaders; + + @Name(CHARSET) + @Description("Charset. Defaults to UTF-8. (Macro enabled)") + @Macro + private final String charset; + + @Name(FOLLOW_REDIRECTS) + @Description("Whether to automatically follow redirects. Defaults to true. (Macro enabled)") + @Macro + private final Boolean followRedirects; + + @Name(DISABLE_SSL_VALIDATION) + @Description("If user enables SSL validation, they will be expected to add the certificate to the trustStore" + + " on each machine. Defaults to true. (Macro enabled)") + @Macro + private final Boolean disableSSLValidation; + + @Nullable + @Name(PROPERTY_HTTP_ERROR_HANDLING) + @Description("Defines the error handling strategy to use for certain HTTP response codes." + + "The left column contains a regular expression for HTTP status code. The right column contains an action which" + + "is done in case of match. If HTTP status code matches multiple regular expressions, " + + "the first specified in mapping is matched.") + protected String httpErrorsHandling; + + @Nullable + @Name(PROPERTY_ERROR_HANDLING) + @Description("Error handling strategy to use when the HTTP response cannot be transformed to an output record.") + protected String errorHandling; + + @Nullable + @Name(PROPERTY_RETRY_POLICY) + @Description("Policy used to calculate delay between retries. Default Retry Policy is Exponential.") + protected String retryPolicy; + + @Nullable + @Name(PROPERTY_LINEAR_RETRY_INTERVAL) + @Description("Interval in seconds between retries. Is only used if retry policy is \"linear\".") + @Macro + protected Long linearRetryInterval; + + @Nullable + @Name(PROPERTY_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default value is 600 seconds (10 minute).") + @Macro + protected Long maxRetryDuration; + + @Name(CONNECTION_TIMEOUT) + @Description("Sets the connection timeout in milliseconds. Set to 0 for infinite. Default is 60000 (1 minute). " + + "(Macro enabled)") + @Nullable + @Macro + private final Integer connectTimeout; + + @Name(READ_TIMEOUT) + @Description("The time in milliseconds to wait for a read. Set to 0 for infinite. Defaults to 60000 (1 minute). " + + "(Macro enabled)") + @Nullable + @Macro + private final Integer readTimeout; + + public HTTPSinkConfig(String referenceName, String url, String method, Integer batchSize, + @Nullable String delimiterForMessages, String messageFormat, @Nullable String body, + @Nullable String requestHeaders, String charset, + boolean followRedirects, boolean disableSSLValidation, @Nullable String httpErrorsHandling, + String errorHandling, String retryPolicy, @Nullable Long linearRetryInterval, + Long maxRetryDuration, int readTimeout, int connectTimeout, + String oauth2Enabled, String authType, @Nullable String jsonBatchKey, + Boolean writeJsonAsArray) { + super(referenceName); + this.url = url; + this.method = method; + this.batchSize = batchSize; + this.delimiterForMessages = delimiterForMessages; + this.messageFormat = messageFormat; + this.body = body; + this.requestHeaders = requestHeaders; + this.charset = charset; + this.followRedirects = followRedirects; + this.disableSSLValidation = disableSSLValidation; + this.httpErrorsHandling = httpErrorsHandling; + this.errorHandling = errorHandling; + this.retryPolicy = retryPolicy; + this.linearRetryInterval = linearRetryInterval; + this.maxRetryDuration = maxRetryDuration; + this.readTimeout = readTimeout; + this.connectTimeout = connectTimeout; + this.jsonBatchKey = jsonBatchKey; + this.writeJsonAsArray = writeJsonAsArray; + this.oauth2Enabled = oauth2Enabled; + this.authType = authType; + } + + private HTTPSinkConfig(Builder builder) { + super(builder.referenceName); + url = builder.url; + method = builder.method; + batchSize = builder.batchSize; + delimiterForMessages = builder.delimiterForMessages; + messageFormat = builder.messageFormat; + body = builder.body; + requestHeaders = builder.requestHeaders; + charset = builder.charset; + followRedirects = builder.followRedirects; + disableSSLValidation = builder.disableSSLValidation; + connectTimeout = builder.connectTimeout; + readTimeout = builder.readTimeout; + jsonBatchKey = builder.jsonBatchKey; + writeJsonAsArray = builder.writeJsonAsArray; + oauth2Enabled = builder.oauth2Enabled; + authType = builder.authType; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static Builder newBuilder(HTTPSinkConfig copy) { + Builder builder = new Builder(); + builder.referenceName = copy.referenceName; + builder.url = copy.getUrl(); + builder.method = copy.getMethod(); + builder.batchSize = copy.getBatchSize(); + builder.delimiterForMessages = copy.getDelimiterForMessages(); + builder.messageFormat = copy.getMessageFormat().getValue(); + builder.body = copy.getBody(); + builder.requestHeaders = copy.getRequestHeaders(); + builder.charset = copy.getCharset(); + builder.followRedirects = copy.getFollowRedirects(); + builder.disableSSLValidation = copy.getDisableSSLValidation(); + builder.connectTimeout = copy.getConnectTimeout(); + builder.readTimeout = copy.getReadTimeout(); + builder.oauth2Enabled = copy.getOAuth2Enabled(); + builder.authType = copy.getAuthTypeString(); + return builder; + } + + public String getUrl() { + return url; } - return results; - } - public static Map getMapFromKeyValueString(String keyValueString) { - Map result = new LinkedHashMap<>(); + public String getMethod() { + return method; + } - if (Strings.isNullOrEmpty(keyValueString)) { - return result; + public Integer getBatchSize() { + return batchSize; } - String[] mappings = keyValueString.split(","); - for (String map : mappings) { - String[] columns = map.split(":"); - if (columns.length < 2) { //For scenario where either of key or value not provided - throw new IllegalArgumentException(String.format("Missing value for key %s", columns[0])); - } - result.put(columns[0], columns[1]); + public boolean shouldWriteJsonAsArray() { + return writeJsonAsArray != null && writeJsonAsArray; } - return result; - } - public void validate(FailureCollector collector) { - super.validate(collector); + public String getJsonBatchKey() { + return jsonBatchKey; + } - if (!containsMacro(URL)) { - try { - new URL(url); - } catch (MalformedURLException e) { - collector.addFailure(String.format("URL '%s' is malformed: %s", url, e.getMessage()), null) - .withConfigProperty(URL); - } - } - - if (!containsMacro(CONNECTION_TIMEOUT) && Objects.nonNull(connectTimeout) && connectTimeout < 0) { - collector.addFailure("Connection Timeout cannot be a negative number.", null) - .withConfigProperty(CONNECTION_TIMEOUT); - } - - try { - convertHeadersToMap(requestHeaders); - } catch (IllegalArgumentException e) { - collector.addFailure(e.getMessage(), null) - .withConfigProperty(REQUEST_HEADERS); - } - - if (!containsMacro(METHOD) && !METHODS.contains(method.toUpperCase())) { - collector.addFailure( - String.format("Invalid request method %s, must be one of %s.", method, Joiner.on(',').join(METHODS)), null) - .withConfigProperty(METHOD); - } - - if (!containsMacro(BATCH_SIZE) && batchSize != null && batchSize < 1) { - collector.addFailure("Batch size must be greater than 0.", null) - .withConfigProperty(BATCH_SIZE); + @Nullable + public String getDelimiterForMessages() { + return Strings.isNullOrEmpty(delimiterForMessages) ? "\n" : delimiterForMessages; } - // Validate Linear Retry Interval - if (!containsMacro(PROPERTY_RETRY_POLICY) && getRetryPolicy() == RetryPolicy.LINEAR) { - assertIsSet(getLinearRetryInterval(), PROPERTY_LINEAR_RETRY_INTERVAL, "retry policy is linear"); + public MessageFormatType getMessageFormat() { + return MessageFormatType.valueOf(messageFormat.toUpperCase()); } - if (!containsMacro(READ_TIMEOUT) && Objects.nonNull(readTimeout) && readTimeout < 0) { - collector.addFailure("Read Timeout cannot be a negative number.", null) - .withConfigProperty(READ_TIMEOUT); + + @Nullable + public String getBody() { + return body; } - if (!containsMacro(MESSAGE_FORMAT) && !containsMacro("body") && messageFormat.equalsIgnoreCase("Custom") - && body == null) { - collector.addFailure("For Custom message format, message cannot be null.", null) - .withConfigProperty(MESSAGE_FORMAT); + @Nullable + public String getRequestHeaders() { + return requestHeaders; } - if (!containsMacro(PROPERTY_MAX_RETRY_DURATION) && Objects.nonNull(maxRetryDuration) && maxRetryDuration < 0) { - collector.addFailure("Max Retry Duration cannot be a negative number.", null) - .withConfigProperty(PROPERTY_MAX_RETRY_DURATION); + public String getCharset() { + return charset; } - } - public void validateSchema(@Nullable Schema schema, FailureCollector collector) { - if (schema == null) { - return; + public Boolean getFollowRedirects() { + return followRedirects; } - List fields = schema.getFields(); - if (fields == null || fields.isEmpty()) { - collector.addFailure("Schema must contain at least one field", null); - throw collector.getOrThrowException(); + + public Boolean getDisableSSLValidation() { + return disableSSLValidation; } - if (containsMacro(URL) || containsMacro(METHOD)) { - return; + @Nullable + public String getHttpErrorsHandling() { + return httpErrorsHandling; } - - if ((method.equals("PUT") || method.equals("PATCH") || method.equals("DELETE")) && url.contains(PLACEHOLDER)) { - Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); - Matcher matcher = pattern.matcher(url); - List fieldNames = fields.stream().map(field -> field.getName()).collect(Collectors.toList()); - while (matcher.find()) { - if (!fieldNames.contains(matcher.group(1))) { - collector.addFailure(String.format("Schema must contain '%s' field mentioned in the url", matcher.group(1)), - null).withConfigProperty(URL); - } - } + + public ErrorHandling getErrorHandling() { + return getEnumValueByString(ErrorHandling.class, errorHandling, PROPERTY_ERROR_HANDLING); } - } - private Map convertHeadersToMap(String headersString) { - Map headersMap = new HashMap<>(); - if (!Strings.isNullOrEmpty(headersString)) { - for (String chunk : headersString.split(DELIMITER)) { - String[] keyValue = chunk.split(KV_DELIMITER, 2); - if (keyValue.length == 2) { - headersMap.put(keyValue[0], keyValue[1]); - } else { - throw new IllegalArgumentException(String.format("Unable to parse key-value pair '%s'.", chunk)); + public RetryPolicy getRetryPolicy() { + if (retryPolicy == null) { + return RetryPolicy.EXPONENTIAL; } - } + return getEnumValueByString(RetryPolicy.class, retryPolicy, PROPERTY_RETRY_POLICY); } - return headersMap; - } - - /** - * Builder for creating a {@link HTTPSinkConfig}. - */ - public static final class Builder { - private String referenceName; - private String url; - private String method; - private Integer batchSize; - private Boolean writeJsonAsArray; - private String jsonBatchKey; - private String delimiterForMessages; - private String messageFormat; - private String body; - private String requestHeaders; - private String charset; - private Boolean followRedirects; - private Boolean disableSSLValidation; - private Integer connectTimeout; - private Integer readTimeout; - private String oauth2Enabled; - private String authType; - private Builder() { + private static T + getEnumValueByString(Class enumClass, String stringValue, String propertyName) { + return Stream.of(enumClass.getEnumConstants()) + .filter(keyType -> keyType.getValue().equalsIgnoreCase(stringValue)) + .findAny() + .orElseThrow(() -> new InvalidConfigPropertyException( + String.format("Unsupported value for '%s': '%s'", propertyName, stringValue), propertyName)); } - public Builder setReferenceName(String referenceName) { - this.referenceName = referenceName; - return this; + @Nullable + public Long getLinearRetryInterval() { + return linearRetryInterval; } - public Builder setUrl(String url) { - this.url = url; - return this; + public Long getMaxRetryDuration() { + if (maxRetryDuration == null) { + return 600L; + } + return maxRetryDuration; } - public Builder setMethod(String method) { - this.method = method; - return this; + @Nullable + public Integer getConnectTimeout() { + return connectTimeout; } - public Builder setBatchSize(Integer batchSize) { - this.batchSize = batchSize; - return this; + @Nullable + public Integer getReadTimeout() { + return readTimeout; } - public Builder setWriteJsonAsArray(Boolean writeJsonAsArray) { - this.writeJsonAsArray = writeJsonAsArray; - return this; + public Map getRequestHeadersMap() { + return convertHeadersToMap(requestHeaders); } - public Builder setJsonBatchKey(String jsonBatchKey) { - this.jsonBatchKey = jsonBatchKey; - return this; + public Map getHeadersMap(String header) { + return convertHeadersToMap(header); } - public Builder setDelimiterForMessages(String delimiterForMessages) { - this.delimiterForMessages = delimiterForMessages; - return this; + public String getReferenceNameOrNormalizedFQN() { + return Strings.isNullOrEmpty(referenceName) ? ReferenceNames.normalizeFqn(url) : referenceName; } - public Builder setMessageFormat(String messageFormat) { - this.messageFormat = messageFormat; - return this; - } + public List getHttpErrorHandlingEntries() { + Map httpErrorsHandlingMap = getMapFromKeyValueString(httpErrorsHandling); + List results = new ArrayList<>(httpErrorsHandlingMap.size()); - public Builder setBody(String body) { - this.body = body; - return this; + for (Map.Entry entry : httpErrorsHandlingMap.entrySet()) { + String regex = entry.getKey(); + try { + results.add(new HttpErrorHandlerEntity(Pattern.compile(regex), + getEnumValueByString(RetryableErrorHandling.class, + entry.getValue(), PROPERTY_HTTP_ERROR_HANDLING))); + } catch (PatternSyntaxException e) { + // We embed causing exception message into this one. Since this message is shown on UI when validation fails. + throw new InvalidConfigPropertyException( + String.format( + "Error handling regex '%s' is not valid. %s", regex, e.getMessage()), PROPERTY_HTTP_ERROR_HANDLING); + } + } + return results; } - public Builder setRequestHeaders(String requestHeaders) { - this.requestHeaders = requestHeaders; - return this; - } + public static Map getMapFromKeyValueString(String keyValueString) { + Map result = new LinkedHashMap<>(); - public Builder setCharset(String charset) { - this.charset = charset; - return this; - } + if (Strings.isNullOrEmpty(keyValueString)) { + return result; + } - public Builder setFollowRedirects(Boolean followRedirects) { - this.followRedirects = followRedirects; - return this; + String[] mappings = keyValueString.split(","); + for (String map : mappings) { + String[] columns = map.split(":"); + if (columns.length < 2) { //For scenario where either of key or value not provided + throw new IllegalArgumentException(String.format("Missing value for key %s", columns[0])); + } + result.put(columns[0], columns[1]); + } + return result; } - public Builder setDisableSSLValidation(Boolean disableSSLValidation) { - this.disableSSLValidation = disableSSLValidation; - return this; - } + public void validate(FailureCollector collector) { + super.validate(collector); + + if (!containsMacro(URL)) { + try { + new URL(url); + } catch (MalformedURLException e) { + collector.addFailure(String.format("URL '%s' is malformed: %s", url, e.getMessage()), null) + .withConfigProperty(URL); + } + } + + if (!containsMacro(CONNECTION_TIMEOUT) && Objects.nonNull(connectTimeout) && connectTimeout < 0) { + collector.addFailure("Connection Timeout cannot be a negative number.", null) + .withConfigProperty(CONNECTION_TIMEOUT); + } - public Builder setConnectTimeout(Integer connectTimeout) { - this.connectTimeout = connectTimeout; - return this; + try { + convertHeadersToMap(requestHeaders); + } catch (IllegalArgumentException e) { + collector.addFailure(e.getMessage(), null) + .withConfigProperty(REQUEST_HEADERS); + } + + if (!containsMacro(METHOD) && !METHODS.contains(method.toUpperCase())) { + collector.addFailure( + String.format("Invalid request method %s, must be one of %s.", method, Joiner.on(',').join(METHODS)), null) + .withConfigProperty(METHOD); + } + + if (!containsMacro(BATCH_SIZE) && batchSize != null && batchSize < 1) { + collector.addFailure("Batch size must be greater than 0.", null) + .withConfigProperty(BATCH_SIZE); + } + + // Validate Linear Retry Interval + if (!containsMacro(PROPERTY_RETRY_POLICY) && getRetryPolicy() == RetryPolicy.LINEAR) { + assertIsSet(getLinearRetryInterval(), PROPERTY_LINEAR_RETRY_INTERVAL, "retry policy is linear"); + } + if (!containsMacro(READ_TIMEOUT) && Objects.nonNull(readTimeout) && readTimeout < 0) { + collector.addFailure("Read Timeout cannot be a negative number.", null) + .withConfigProperty(READ_TIMEOUT); + } + + if (!containsMacro(MESSAGE_FORMAT) && !containsMacro("body") && messageFormat.equalsIgnoreCase("Custom") + && body == null) { + collector.addFailure("For Custom message format, message cannot be null.", null) + .withConfigProperty(MESSAGE_FORMAT); + } + + if (!containsMacro(PROPERTY_MAX_RETRY_DURATION) && Objects.nonNull(maxRetryDuration) && maxRetryDuration < 0) { + collector.addFailure("Max Retry Duration cannot be a negative number.", null) + .withConfigProperty(PROPERTY_MAX_RETRY_DURATION); + } } - public Builder setReadTimeout(Integer readTimeout) { - this.readTimeout = readTimeout; - return this; + public void validateSchema(@Nullable Schema schema, FailureCollector collector) { + if (schema == null) { + return; + } + List fields = schema.getFields(); + if (fields == null || fields.isEmpty()) { + collector.addFailure("Schema must contain at least one field", null); + throw collector.getOrThrowException(); + } + + if (containsMacro(URL) || containsMacro(METHOD)) { + return; + } + + if ((method.equals("PUT") || method.equals("PATCH") || method.equals("DELETE")) && url.contains(PLACEHOLDER)) { + Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); + Matcher matcher = pattern.matcher(url); + List fieldNames = fields.stream().map(Schema.Field::getName).collect(Collectors.toList()); + while (matcher.find()) { + if (!fieldNames.contains(matcher.group(1))) { + collector.addFailure(String.format("Schema must contain '%s' field mentioned in the url", matcher.group(1)), + null).withConfigProperty(URL); + } + } + } } - public HTTPSinkConfig build() { - return new HTTPSinkConfig(this); + private Map convertHeadersToMap(String headersString) { + Map headersMap = new HashMap<>(); + if (!Strings.isNullOrEmpty(headersString)) { + for (String chunk : headersString.split(DELIMITER)) { + String[] keyValue = chunk.split(KV_DELIMITER, 2); + if (keyValue.length == 2) { + headersMap.put(keyValue[0], keyValue[1]); + } else { + throw new IllegalArgumentException(String.format("Unable to parse key-value pair '%s'.", chunk)); + } + } + } + return headersMap; + } + + /** + * Builder for creating a {@link HTTPSinkConfig}. + */ + public static final class Builder { + private String referenceName; + private String url; + private String method; + private Integer batchSize; + private Boolean writeJsonAsArray; + private String jsonBatchKey; + private String delimiterForMessages; + private String messageFormat; + private String body; + private String requestHeaders; + private String charset; + private Boolean followRedirects; + private Boolean disableSSLValidation; + private Integer connectTimeout; + private Integer readTimeout; + private String oauth2Enabled; + private String authType; + + private Builder() { + } + + public Builder setReferenceName(String referenceName) { + this.referenceName = referenceName; + return this; + } + + public Builder setUrl(String url) { + this.url = url; + return this; + } + + public Builder setMethod(String method) { + this.method = method; + return this; + } + + public Builder setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + return this; + } + + public Builder setWriteJsonAsArray(Boolean writeJsonAsArray) { + this.writeJsonAsArray = writeJsonAsArray; + return this; + } + + public Builder setJsonBatchKey(String jsonBatchKey) { + this.jsonBatchKey = jsonBatchKey; + return this; + } + + public Builder setDelimiterForMessages(String delimiterForMessages) { + this.delimiterForMessages = delimiterForMessages; + return this; + } + + public Builder setMessageFormat(String messageFormat) { + this.messageFormat = messageFormat; + return this; + } + + public Builder setBody(String body) { + this.body = body; + return this; + } + + public Builder setRequestHeaders(String requestHeaders) { + this.requestHeaders = requestHeaders; + return this; + } + + public Builder setCharset(String charset) { + this.charset = charset; + return this; + } + + public Builder setFollowRedirects(Boolean followRedirects) { + this.followRedirects = followRedirects; + return this; + } + + public Builder setDisableSSLValidation(Boolean disableSSLValidation) { + this.disableSSLValidation = disableSSLValidation; + return this; + } + + public Builder setConnectTimeout(Integer connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } + + public Builder setReadTimeout(Integer readTimeout) { + this.readTimeout = readTimeout; + return this; + } + + public HTTPSinkConfig build() { + return new HTTPSinkConfig(this); + } } - } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java b/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java index 63725d3f..9401843b 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java @@ -25,9 +25,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import java.util.*; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,194 +37,196 @@ * The message is then returned to the HTTPRecordWriter. */ public class MessageBuffer { - private static final String REGEX_HASHED_VAR = "#(\\w+)"; - private final List buffer; - private final String jsonBatchKey; - private final Boolean shouldWriteJsonAsArray; - private final String delimiterForMessages; - private final String charset; - private final String customMessageBody; - private final Function, String> messageFormatter; - private final String contentType; - private final Schema wrappedMessageSchema; - - - /** - * Constructor for MessageBuffer. - * - * @param messageFormat The format of the message. Can be JSON, FORM or CUSTOM. - * @param jsonBatchKey The key to be used for the JSON batch message. - * @param shouldWriteJsonAsArray Whether the JSON message should be written as an array. - * @param delimiterForMessages The delimiter to be used for messages. - * @param charset The charset to be used for the message. - * @param customMessageBody The custom message body to be used. - */ - public MessageBuffer( - MessageFormatType messageFormat, String jsonBatchKey, boolean shouldWriteJsonAsArray, - String delimiterForMessages, String charset, String customMessageBody, Schema inputSchema - ) { - this.jsonBatchKey = jsonBatchKey; - this.delimiterForMessages = delimiterForMessages; - this.charset = charset; - this.shouldWriteJsonAsArray = shouldWriteJsonAsArray; - this.customMessageBody = customMessageBody; - this.buffer = new ArrayList<>(); - switch (messageFormat) { - case JSON: - messageFormatter = this::formatAsJson; - contentType = "application/json"; - break; - case FORM: - messageFormatter = this::formatAsForm; - contentType = "application/x-www-form-urlencoded"; - break; - case CUSTOM: - messageFormatter = this::formatAsCustom; - contentType = "text/plain"; - break; - default: - throw new IllegalArgumentException("Invalid message format: " + messageFormat); + private static final String REGEX_HASHED_VAR = "#(\\w+)"; + private final List buffer; + private final String jsonBatchKey; + private final Boolean shouldWriteJsonAsArray; + private final String delimiterForMessages; + private final String charset; + private final String customMessageBody; + private final Function, String> messageFormatter; + private final String contentType; + private final Schema wrappedMessageSchema; + + + /** + * Constructor for MessageBuffer. + * + * @param messageFormat The format of the message. Can be JSON, FORM or CUSTOM. + * @param jsonBatchKey The key to be used for the JSON batch message. + * @param shouldWriteJsonAsArray Whether the JSON message should be written as an array. + * @param delimiterForMessages The delimiter to be used for messages. + * @param charset The charset to be used for the message. + * @param customMessageBody The custom message body to be used. + */ + public MessageBuffer( + MessageFormatType messageFormat, String jsonBatchKey, boolean shouldWriteJsonAsArray, + String delimiterForMessages, String charset, String customMessageBody, Schema inputSchema + ) { + this.jsonBatchKey = jsonBatchKey; + this.delimiterForMessages = delimiterForMessages; + this.charset = charset; + this.shouldWriteJsonAsArray = shouldWriteJsonAsArray; + this.customMessageBody = customMessageBody; + this.buffer = new ArrayList<>(); + switch (messageFormat) { + case JSON: + messageFormatter = this::formatAsJson; + contentType = "application/json"; + break; + case FORM: + messageFormatter = this::formatAsForm; + contentType = "application/x-www-form-urlencoded"; + break; + case CUSTOM: + messageFormatter = this::formatAsCustom; + contentType = "text/plain"; + break; + default: + throw new IllegalArgumentException("Invalid message format: " + messageFormat); + } + // A new StructuredRecord is created with the jsonBatchKey as the + // field name and the array of records as the value + Schema bufferRecordArraySchema = Schema.arrayOf(inputSchema); + wrappedMessageSchema = Schema.recordOf("wrapper", + Schema.Field.of(jsonBatchKey, bufferRecordArraySchema)); } - // A new StructuredRecord is created with the jsonBatchKey as the - // field name and the array of records as the value - Schema bufferRecordArraySchema = Schema.arrayOf(inputSchema); - wrappedMessageSchema = Schema.recordOf("wrapper", - Schema.Field.of(jsonBatchKey, bufferRecordArraySchema)); - } - - /** - * Adds a record to the buffer. - * - * @param record The record to be added. - */ - public void add(StructuredRecord record) { - buffer.add(record); - } - - /** - * Clears the buffer. - */ - public void clear() { - buffer.clear(); - } - - /** - * Returns the size of the buffer. - */ - public int size() { - return buffer.size(); - } - - /** - * Returns whether the buffer is empty. - */ - public boolean isEmpty() { - return buffer.isEmpty(); - } - - /** - * Returns the content type of the message. - */ - public String getContentType() { - return contentType; - } - - /** - * Converts the buffer to the appropriate format and returns the message. - */ - public String getMessage() throws IOException { - return messageFormatter.apply(buffer); - } - - private String formatAsJson(List buffer) { - try { - return formatAsJsonInternal(buffer); - } catch (IOException e) { - throw new IllegalStateException("Error formatting JSON message. Reason: " + e.getMessage(), e); + + /** + * Adds a record to the buffer. + * + * @param structuredRecord The record to be added. + */ + public void add(StructuredRecord structuredRecord) { + buffer.add(structuredRecord); } - } - private String formatAsJsonInternal(List buffer) throws IOException { - boolean useJsonBatchKey = !Strings.isNullOrEmpty(jsonBatchKey); - if (!shouldWriteJsonAsArray || !useJsonBatchKey) { - return getBufferAsJsonList(); + /** + * Clears the buffer. + */ + public void clear() { + buffer.clear(); } - StructuredRecord wrappedMessageRecord = StructuredRecord.builder(wrappedMessageSchema) - .set(jsonBatchKey, buffer).build(); - return StructuredRecordStringConverter.toJsonString(wrappedMessageRecord); - } - - private String formatAsForm(List buffer) { - return buffer.stream() - .map(this::createFormMessage) - .collect(Collectors.joining(delimiterForMessages)); - } - - private String formatAsCustom(List buffer) { - return buffer.stream() - .map(this::createCustomMessage) - .collect(Collectors.joining(delimiterForMessages)); - } - - private String getBufferAsJsonList() throws IOException { - StringBuilder sb = new StringBuilder(); - String delimiter = shouldWriteJsonAsArray ? "," : delimiterForMessages; - if (shouldWriteJsonAsArray) { - sb.append("["); + + /** + * Returns the size of the buffer. + */ + public int size() { + return buffer.size(); } - for (StructuredRecord record : buffer) { - sb.append(StructuredRecordStringConverter.toJsonString(record)); - sb.append(delimiter); + + /** + * Returns whether the buffer is empty. + */ + public boolean isEmpty() { + return buffer.isEmpty(); } - if (!buffer.isEmpty()) { - sb.setLength(sb.length() - delimiter.length()); + + /** + * Returns the content type of the message. + */ + public String getContentType() { + return contentType; } - if (shouldWriteJsonAsArray) { - sb.append("]"); + + /** + * Converts the buffer to the appropriate format and returns the message. + */ + public String getMessage() throws IOException { + return messageFormatter.apply(buffer); } - return sb.toString(); - } - - private String createFormMessage(StructuredRecord input) { - boolean first = true; - String formMessage = null; - StringBuilder sb = new StringBuilder(""); - for (Schema.Field field : input.getSchema().getFields()) { - if (first) { - first = false; - } else { - sb.append("&"); - } - sb.append(field.getName()); - sb.append("="); - sb.append((String) input.get(field.getName())); + + private String formatAsJson(List buffer) { + try { + return formatAsJsonInternal(buffer); + } catch (IOException e) { + throw new IllegalStateException("Error formatting JSON message. Reason: " + e.getMessage(), e); + } } - try { - formMessage = URLEncoder.encode(sb.toString(), charset); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Error encoding Form message. Reason: " + e.getMessage(), e); + + private String formatAsJsonInternal(List buffer) throws IOException { + boolean useJsonBatchKey = !Strings.isNullOrEmpty(jsonBatchKey); + if (Boolean.TRUE.equals(!shouldWriteJsonAsArray) || !useJsonBatchKey) { + return getBufferAsJsonList(); + } + StructuredRecord wrappedMessageRecord = StructuredRecord.builder(wrappedMessageSchema) + .set(jsonBatchKey, buffer).build(); + return StructuredRecordStringConverter.toJsonString(wrappedMessageRecord); } - return formMessage; - } - - private String createCustomMessage(StructuredRecord input) { - String customMessage = customMessageBody; - Matcher matcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); - HashMap findReplaceMap = new HashMap(); - while (matcher.find()) { - if (input.get(matcher.group(1)) != null) { - findReplaceMap.put(matcher.group(1), (String) input.get(matcher.group(1))); - } else { - throw new IllegalArgumentException(String.format( - "Field %s doesnt exist in the input schema.", matcher.group(1))); - } + + private String formatAsForm(List buffer) { + return buffer.stream() + .map(this::createFormMessage) + .collect(Collectors.joining(delimiterForMessages)); } - Matcher replaceMatcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); - while (replaceMatcher.find()) { - String val = replaceMatcher.group().replace("#", ""); - customMessage = (customMessage.replace(replaceMatcher.group(), findReplaceMap.get(val))); + + private String formatAsCustom(List buffer) { + return buffer.stream() + .map(this::createCustomMessage) + .collect(Collectors.joining(delimiterForMessages)); + } + + private String getBufferAsJsonList() throws IOException { + StringBuilder sb = new StringBuilder(); + String delimiter = Boolean.TRUE.equals(shouldWriteJsonAsArray) ? "," : delimiterForMessages; + if (Boolean.TRUE.equals(shouldWriteJsonAsArray)) { + sb.append("["); + } + for (StructuredRecord structuredRecord : buffer) { + sb.append(StructuredRecordStringConverter.toJsonString(structuredRecord)); + sb.append(delimiter); + } + if (!buffer.isEmpty()) { + sb.setLength(sb.length() - delimiter.length()); + } + if (Boolean.TRUE.equals(shouldWriteJsonAsArray)) { + sb.append("]"); + } + return sb.toString(); + } + + private String createFormMessage(StructuredRecord input) { + boolean first = true; + String formMessage = null; + StringBuilder sb = new StringBuilder(); + if (input != null && input.getSchema() != null) { + for (Schema.Field field : Objects.requireNonNull(input.getSchema().getFields())) { + if (first) { + first = false; + } else { + sb.append("&"); + } + sb.append(field.getName()); + sb.append("="); + sb.append((String) input.get(field.getName())); + } + } + try { + formMessage = URLEncoder.encode(sb.toString(), charset); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Error encoding Form message. Reason: " + e.getMessage(), e); + } + return formMessage; + } + + private String createCustomMessage(StructuredRecord input) { + String customMessage = customMessageBody; + Matcher matcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); + HashMap findReplaceMap = new HashMap<>(); + while (matcher.find()) { + if (input.get(matcher.group(1)) != null) { + findReplaceMap.put(matcher.group(1), (String) input.get(matcher.group(1))); + } else { + throw new IllegalArgumentException(String.format( + "Field %s doesnt exist in the input schema.", matcher.group(1))); + } + } + Matcher replaceMatcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); + while (replaceMatcher.find()) { + String val = replaceMatcher.group().replace("#", ""); + customMessage = (customMessage.replace(replaceMatcher.group(), findReplaceMap.get(val))); + } + return customMessage; } - return customMessage; - } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java b/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java index 1520f8e0..7db376d4 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java @@ -20,28 +20,27 @@ * This class stores the placeholder information to avoid performing string functions for each record. */ public class PlaceholderBean { - private static final String PLACEHOLDER_FORMAT = "#%s"; - private final String placeHolderKey; - private final String placeHolderKeyWithPrefix; - private final int startIndex; - private final int endIndex; + private static final String PLACEHOLDER_FORMAT = "#%s"; + private final String placeHolderKey; + private final int startIndex; + private final int endIndex; - public PlaceholderBean(String url, String placeHolderKey) { - this.placeHolderKey = placeHolderKey; - this.placeHolderKeyWithPrefix = String.format(PLACEHOLDER_FORMAT, placeHolderKey); - this.startIndex = url.indexOf(placeHolderKeyWithPrefix); - this.endIndex = startIndex + placeHolderKeyWithPrefix.length(); - } + public PlaceholderBean(String url, String placeHolderKey) { + String placeHolderKeyWithPrefix = String.format(PLACEHOLDER_FORMAT, placeHolderKey); + this.placeHolderKey = placeHolderKey; + this.startIndex = url.indexOf(placeHolderKeyWithPrefix); + this.endIndex = startIndex + placeHolderKeyWithPrefix.length(); + } - public String getPlaceHolderKey() { - return placeHolderKey; - } + public String getPlaceHolderKey() { + return placeHolderKey; + } - public int getStartIndex() { - return startIndex; - } + public int getStartIndex() { + return startIndex; + } - public int getEndIndex() { - return endIndex; - } + public int getEndIndex() { + return endIndex; + } } From e7ece0cd3ac049f1a3cbef33c816ae456387f563 Mon Sep 17 00:00:00 2001 From: Amit Kumar Singh Date: Thu, 5 Dec 2024 07:00:24 +0000 Subject: [PATCH 3/7] refactor --- .../http/common/HttpErrorDetailsProvider.java | 176 ++- .../http/sink/batch/HTTPOutputFormat.java | 82 +- .../http/sink/batch/HTTPRecordWriter.java | 570 ++++----- .../cdap/plugin/http/sink/batch/HTTPSink.java | 122 +- .../http/sink/batch/HTTPSinkConfig.java | 1024 ++++++++--------- .../plugin/http/sink/batch/MessageBuffer.java | 361 +++--- .../http/sink/batch/PlaceholderBean.java | 38 +- .../http/source/batch/HttpBatchSource.java | 5 + 8 files changed, 1185 insertions(+), 1193 deletions(-) diff --git a/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java index 293affaa..96efba94 100644 --- a/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java +++ b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java @@ -16,7 +16,9 @@ package io.cdap.plugin.http.common; -import io.cdap.cdap.api.exception.*; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.etl.api.exception.ErrorContext; import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; import com.google.common.base.Throwables; @@ -24,110 +26,92 @@ import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.etl.api.validation.InvalidConfigPropertyException; -import java.io.*; +import java.io.UnsupportedEncodingException; import java.util.List; import java.util.NoSuchElementException; +/** + Error details provided for the HTTP + **/ public class HttpErrorDetailsProvider implements ErrorDetailsProvider { - @Override - public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { - List causalChain = Throwables.getCausalChain(e); - for (Throwable t : causalChain) { - //, UnsupportedOperationException, - if (t instanceof ProgramFailureException) { - // if causal chain already has program failure exception, return null to avoid double wrap. - return null; - } - if (t instanceof IllegalArgumentException) { - return getProgramFailureException((IllegalArgumentException) t, errorContext); - } - if (t instanceof IllegalStateException) { - return getProgramFailureException((IllegalStateException) t, errorContext); - } - if (t instanceof InvalidConfigPropertyException) { - return getProgramFailureException((InvalidConfigPropertyException) t, errorContext); - } - if (t instanceof NoSuchElementException) { - return getProgramFailureException((NoSuchElementException) t, errorContext); - } - if (t instanceof UnsupportedEncodingException) { - return getProgramFailureException((UnsupportedEncodingException) t, errorContext); - } - } + @Override + public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { + List causalChain = Throwables.getCausalChain(e); + for (Throwable t : causalChain) { + if (t instanceof ProgramFailureException) { + // if causal chain already has program failure exception, return null to avoid double wrap. return null; + } + if (t instanceof IllegalArgumentException) { + return getProgramFailureException((IllegalArgumentException) t, errorContext); + } + if (t instanceof IllegalStateException) { + return getProgramFailureException((IllegalStateException) t, errorContext); + } + if (t instanceof InvalidConfigPropertyException) { + return getProgramFailureException((InvalidConfigPropertyException) t, errorContext); + } + if (t instanceof NoSuchElementException) { + return getProgramFailureException((NoSuchElementException) t, errorContext); + } } + return null; + } - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalArgumentException}. - * - * @param e The IllegalArgumentException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalStateException}. - * - * @param e The IllegalStateException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link InvalidConfigPropertyException}. - * - * @param e The InvalidConfigPropertyException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(InvalidConfigPropertyException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalArgumentException}. + * + * @param e The IllegalArgumentException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); + } - /** - * Get a ProgramFailureException with the given error - * information from {@link NoSuchElementException}. - * - * @param e The NoSuchElementException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(NoSuchElementException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } - - - /** - * Get a ProgramFailureException with the given error - * information from {@link UnsupportedEncodingException}. - * - * @param e The UnsupportedEncodingException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(UnsupportedEncodingException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalStateException}. + * + * @param e The IllegalStateException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } + /** + * Get a ProgramFailureException with the given error + * information from {@link InvalidConfigPropertyException}. + * + * @param e The InvalidConfigPropertyException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(InvalidConfigPropertyException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } + /** + * Get a ProgramFailureException with the given error + * information from {@link NoSuchElementException}. + * + * @param e The NoSuchElementException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(NoSuchElementException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java index ab3c2efc..f784a876 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java @@ -33,57 +33,57 @@ * OutputFormat for HTTP writing */ public class HTTPOutputFormat extends OutputFormat { - private static final Gson GSON = new Gson(); - static final String CONFIG_KEY = "http.sink.config"; - static final String INPUT_SCHEMA_KEY = "http.sink.input.schema"; - - @Override - public RecordWriter getRecordWriter(TaskAttemptContext context) { - Configuration hConf = context.getConfiguration(); - HTTPSinkConfig config = GSON.fromJson(hConf.get(CONFIG_KEY), HTTPSinkConfig.class); - Schema inputSchema; - try { - inputSchema = Schema.parseJson(hConf.get(INPUT_SCHEMA_KEY)); - return new HTTPRecordWriter(config, inputSchema); - } catch (IOException e) { - String errorMessage = "Unable to parse and write the record"; - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); - } + private static final Gson GSON = new Gson(); + static final String CONFIG_KEY = "http.sink.config"; + static final String INPUT_SCHEMA_KEY = "http.sink.input.schema"; + + @Override + public RecordWriter getRecordWriter(TaskAttemptContext context) { + Configuration hConf = context.getConfiguration(); + HTTPSinkConfig config = GSON.fromJson(hConf.get(CONFIG_KEY), HTTPSinkConfig.class); + Schema inputSchema; + try { + inputSchema = Schema.parseJson(hConf.get(INPUT_SCHEMA_KEY)); + return new HTTPRecordWriter(config, inputSchema); + } catch (IOException e) { + String errorMessage = "Unable to parse and write the record"; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); } + } - @Override - public void checkOutputSpecs(JobContext jobContext) { + @Override + public void checkOutputSpecs(JobContext jobContext) { - } + } - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext context) { - return new OutputCommitter() { - @Override - public void setupJob(JobContext jobContext) { + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) { + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) { - } + } - @Override - public void setupTask(TaskAttemptContext taskAttemptContext) { + @Override + public void setupTask(TaskAttemptContext taskAttemptContext) { - } + } - @Override - public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) { - return false; - } + @Override + public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) { + return false; + } - @Override - public void commitTask(TaskAttemptContext taskAttemptContext) { + @Override + public void commitTask(TaskAttemptContext taskAttemptContext) { - } + } - @Override - public void abortTask(TaskAttemptContext taskAttemptContext) { + @Override + public void abortTask(TaskAttemptContext taskAttemptContext) { - } - }; - } + } + }; + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java index 7f1c3d6a..9d69af80 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java @@ -17,12 +17,12 @@ package io.cdap.plugin.http.sink.batch; import com.google.auth.oauth2.AccessToken; -import com.google.common.base.Charsets; import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.api.exception.*; -import io.cdap.cdap.etl.api.exception.*; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.plugin.http.common.RetryPolicy; import io.cdap.plugin.http.common.error.ErrorHandling; import io.cdap.plugin.http.common.error.HttpErrorHandler; @@ -83,323 +83,323 @@ * RecordWriter for HTTP. */ public class HTTPRecordWriter extends RecordWriter { - private static final Logger LOG = LoggerFactory.getLogger(HTTPRecordWriter.class); - private static final String REGEX_HASHED_VAR = "#(\\w+)"; - public static final String REQUEST_METHOD_POST = "POST"; - public static final String REQUEST_METHOD_PUT = "PUT"; - public static final String REQUEST_METHOD_DELETE = "DELETE"; - public static final String REQUEST_METHOD_PATCH = "PATCH"; - - private final HTTPSinkConfig config; - private final MessageBuffer messageBuffer; - private String contentType; - private final String url; - private String configURL; - private final List placeHolderList; - private final Map headers; - - private AccessToken accessToken; - private final HttpErrorHandler httpErrorHandler; - private final PollInterval pollInterval; - private int httpStatusCode; - private String httpResponseBody; - private static int retryCount; - - HTTPRecordWriter(HTTPSinkConfig config, Schema inputSchema) { - this.headers = config.getRequestHeadersMap(); - this.config = config; - this.accessToken = null; - this.messageBuffer = new MessageBuffer( - config.getMessageFormat(), config.getJsonBatchKey(), config.shouldWriteJsonAsArray(), - config.getDelimiterForMessages(), config.getCharset(), config.getBody(), inputSchema - ); - this.httpErrorHandler = new HttpErrorHandler(config); - if (config.getRetryPolicy().equals(RetryPolicy.LINEAR)) { - pollInterval = FixedPollInterval.fixed(config.getLinearRetryInterval(), TimeUnit.SECONDS); - } else { - pollInterval = IterativePollInterval.iterative(duration -> duration.multiply(2), - Duration.FIVE_HUNDRED_MILLISECONDS); - } - url = config.getUrl(); - placeHolderList = getPlaceholderListFromURL(); + private static final Logger LOG = LoggerFactory.getLogger(HTTPRecordWriter.class); + private static final String REGEX_HASHED_VAR = "#(\\w+)"; + public static final String REQUEST_METHOD_POST = "POST"; + public static final String REQUEST_METHOD_PUT = "PUT"; + public static final String REQUEST_METHOD_DELETE = "DELETE"; + public static final String REQUEST_METHOD_PATCH = "PATCH"; + + private final HTTPSinkConfig config; + private final MessageBuffer messageBuffer; + private String contentType; + private final String url; + private String configURL; + private final List placeHolderList; + private final Map headers; + + private AccessToken accessToken; + private final HttpErrorHandler httpErrorHandler; + private final PollInterval pollInterval; + private int httpStatusCode; + private String httpResponseBody; + private static int retryCount; + + HTTPRecordWriter(HTTPSinkConfig config, Schema inputSchema) { + this.headers = config.getRequestHeadersMap(); + this.config = config; + this.accessToken = null; + this.messageBuffer = new MessageBuffer( + config.getMessageFormat(), config.getJsonBatchKey(), config.shouldWriteJsonAsArray(), + config.getDelimiterForMessages(), config.getCharset(), config.getBody(), inputSchema + ); + this.httpErrorHandler = new HttpErrorHandler(config); + if (config.getRetryPolicy().equals(RetryPolicy.LINEAR)) { + pollInterval = FixedPollInterval.fixed(config.getLinearRetryInterval(), TimeUnit.SECONDS); + } else { + pollInterval = IterativePollInterval.iterative(duration -> duration.multiply(2), + Duration.FIVE_HUNDRED_MILLISECONDS); + } + url = config.getUrl(); + placeHolderList = getPlaceholderListFromURL(); + } + + @Override + public void write(StructuredRecord input, StructuredRecord unused) { + configURL = url; + if (config.getMethod().equals(REQUEST_METHOD_POST) || config.getMethod().equals(REQUEST_METHOD_PUT) || + config.getMethod().equals(REQUEST_METHOD_PATCH)) { + messageBuffer.add(input); } - @Override - public void write(StructuredRecord input, StructuredRecord unused) { - configURL = url; - if (config.getMethod().equals(REQUEST_METHOD_POST) || config.getMethod().equals(REQUEST_METHOD_PUT) || - config.getMethod().equals(REQUEST_METHOD_PATCH)) { - messageBuffer.add(input); - } - - if (config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || - config.getMethod().equals(REQUEST_METHOD_DELETE) - && !placeHolderList.isEmpty()) { - configURL = updateURLWithPlaceholderValue(input); - } + if (config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || + config.getMethod().equals(REQUEST_METHOD_DELETE) + && !placeHolderList.isEmpty()) { + configURL = updateURLWithPlaceholderValue(input); + } - if (config.getBatchSize() == messageBuffer.size() || config.getMethod().equals(REQUEST_METHOD_DELETE)) { - flushMessageBuffer(); - } + if (config.getBatchSize() == messageBuffer.size() || config.getMethod().equals(REQUEST_METHOD_DELETE)) { + flushMessageBuffer(); } + } - @Override - public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - // Process remaining messages after batch executions. - if (!config.getMethod().equals(REQUEST_METHOD_DELETE)) { - flushMessageBuffer(); - } + @Override + public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { + // Process remaining messages after batch executions. + if (!config.getMethod().equals(REQUEST_METHOD_DELETE)) { + flushMessageBuffer(); } + } - private void disableSSLValidation() { - TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } + private void disableSSLValidation() { + TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } - public void checkClientTrusted(X509Certificate[] certs, String authType) { - } + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } - public void checkServerTrusted(X509Certificate[] certs, String authType) { - } - } - }; - SSLContext sslContext = null; - try { - sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); - } catch (KeyManagementException | NoSuchAlgorithmException e) { - throw new IllegalStateException("Error while installing the trust manager: " + e.getMessage(), e); - } - HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); - HostnameVerifier allHostsValid = (hostname, session) -> true; - HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } } - - private boolean executeHTTPServiceAndCheckStatusCode() { - LOG.debug("HTTP Request Attempt No. : {}", ++retryCount); - - // Try-with-resources ensures proper resource management - try (CloseableHttpClient httpClient = createHttpClient(configURL)) { - URL url = new URL(configURL); - - // Use try-with-resources to ensure response is closed - try (CloseableHttpResponse response = executeHttpRequest(httpClient, url)) { - httpStatusCode = response.getStatusLine().getStatusCode(); - LOG.debug("Response HTTP Status code: {}", httpStatusCode); - httpResponseBody = new HttpResponse(response).getBody(); - } - - RetryableErrorHandling errorHandlingStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode); - boolean shouldRetry = errorHandlingStrategy.shouldRetry(); - - if (!shouldRetry) { - messageBuffer.clear(); - retryCount = 0; - } - return !shouldRetry; - - } catch (MalformedURLException e) { - throw new IllegalArgumentException("Invalid URL: " + configURL, e); - } catch (IOException e) { - LOG.warn("Error making {} request to URL {}.", config.getMethod(), config.getUrl()); - String errorMessage = "Unable to make request. "; - throw ErrorUtils.getProgramFailureException( - new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, e); - } + }; + SSLContext sslContext = null; + try { + sslContext = SSLContext.getInstance("SSL"); + sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); + } catch (KeyManagementException | NoSuchAlgorithmException e) { + throw new IllegalStateException("Error while installing the trust manager: " + e.getMessage(), e); } + HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); + HostnameVerifier allHostsValid = (hostname, session) -> true; + HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); + } - private CloseableHttpResponse executeHttpRequest(CloseableHttpClient httpClient, URL url) { - try { - HttpEntityEnclosingRequestBase request = new HttpRequest(URI.create(url.toString()), config.getMethod()); - - if ("https".equalsIgnoreCase(url.getProtocol())) { - configureHttpsSettings(); - } - - if (!messageBuffer.isEmpty()) { - String requestBodyString = messageBuffer.getMessage(); - if (requestBodyString != null) { - StringEntity requestBody = new StringEntity(requestBodyString, StandardCharsets.UTF_8.name()); - request.setEntity(requestBody); - } - } - - request.setHeaders(getRequestHeaders()); - - // Execute the request and return the response - return httpClient.execute(request); - - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Error encoding the request Reason: " + e.getMessage(), e); - } catch (IOException e) { - String errorMessage = String.format("Unable to execute HTTP request to %s.", url); - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); - } catch (Exception e) { - String errorMessage = String.format("Unexpected error occurred while executing HTTP request to URL: %s", url); - throw ErrorUtils.getProgramFailureException( - new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), errorMessage, - errorMessage, ErrorType.UNKNOWN, true, e); - } - } + private boolean executeHTTPServiceAndCheckStatusCode() { + LOG.debug("HTTP Request Attempt No. : {}", ++retryCount); - private void configureHttpsSettings() { - System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2"); - if (Boolean.TRUE.equals(config.getDisableSSLValidation())) { - disableSSLValidation(); - } - } + // Try-with-resources ensures proper resource management + try (CloseableHttpClient httpClient = createHttpClient(configURL)) { + URL url = new URL(configURL); - public CloseableHttpClient createHttpClient(String pageUriStr) { - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); - - // set timeouts - long connectTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getConnectTimeout()); - long readTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getReadTimeout()); - RequestConfig.Builder requestBuilder = RequestConfig.custom(); - requestBuilder.setSocketTimeout((int) readTimeoutMillis); - requestBuilder.setConnectTimeout((int) connectTimeoutMillis); - requestBuilder.setConnectionRequestTimeout((int) connectTimeoutMillis); - httpClientBuilder.setDefaultRequestConfig(requestBuilder.build()); - - // basic auth - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - if (!Strings.isNullOrEmpty(config.getUsername()) && !Strings.isNullOrEmpty(config.getPassword())) { - URI uri = URI.create(pageUriStr); - AuthScope authScope = new AuthScope(new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme())); - credentialsProvider.setCredentials(authScope, - new UsernamePasswordCredentials(config.getUsername(), config.getPassword())); - } + // Use try-with-resources to ensure response is closed + try (CloseableHttpResponse response = executeHttpRequest(httpClient, url)) { + httpStatusCode = response.getStatusLine().getStatusCode(); + LOG.debug("Response HTTP Status code: {}", httpStatusCode); + httpResponseBody = new HttpResponse(response).getBody(); + } - // proxy and proxy auth - if (!Strings.isNullOrEmpty(config.getProxyUrl())) { - HttpHost proxyHost = HttpHost.create(config.getProxyUrl()); - if (!Strings.isNullOrEmpty(config.getProxyUsername()) && !Strings.isNullOrEmpty(config.getProxyPassword())) { - credentialsProvider.setCredentials(new AuthScope(proxyHost), - new UsernamePasswordCredentials( - config.getProxyUsername(), config.getProxyPassword())); - } - httpClientBuilder.setProxy(proxyHost); - } - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + RetryableErrorHandling errorHandlingStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode); + boolean shouldRetry = errorHandlingStrategy.shouldRetry(); - return httpClientBuilder.build(); + if (!shouldRetry) { + messageBuffer.clear(); + retryCount = 0; + } + return !shouldRetry; + + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Invalid URL: " + configURL, e); + } catch (IOException e) { + LOG.warn("Error making {} request to URL {}.", config.getMethod(), config.getUrl()); + String errorMessage = "Unable to make request. "; + throw ErrorUtils.getProgramFailureException( + new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, e); } + } - private Header[] getRequestHeaders() throws IOException { - ArrayList
clientHeaders = new ArrayList<>(); + private CloseableHttpResponse executeHttpRequest(CloseableHttpClient httpClient, URL url) { + try { + HttpEntityEnclosingRequestBase request = new HttpRequest(URI.create(url.toString()), config.getMethod()); - if (accessToken == null || OAuthUtil.tokenExpired(accessToken)) { - accessToken = OAuthUtil.getAccessToken(config); - } + if ("https".equalsIgnoreCase(url.getProtocol())) { + configureHttpsSettings(); + } - if (accessToken != null) { - Header authorizationHeader = getAuthorizationHeader(accessToken); - clientHeaders.add(authorizationHeader); + if (!messageBuffer.isEmpty()) { + String requestBodyString = messageBuffer.getMessage(); + if (requestBodyString != null) { + StringEntity requestBody = new StringEntity(requestBodyString, StandardCharsets.UTF_8.name()); + request.setEntity(requestBody); } + } + + request.setHeaders(getRequestHeaders()); + + // Execute the request and return the response + return httpClient.execute(request); + + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Error encoding the request Reason: " + e.getMessage(), e); + } catch (IOException e) { + String errorMessage = String.format("Unable to execute HTTP request to %s.", url); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); + } catch (Exception e) { + String errorMessage = String.format("Unexpected error occurred while executing HTTP request to URL: %s", url); + throw ErrorUtils.getProgramFailureException( + new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), errorMessage, + errorMessage, ErrorType.UNKNOWN, true, e); + } + } - headers.put("Request-Method", config.getMethod().toUpperCase()); - headers.put("Instance-Follow-Redirects", String.valueOf(config.getFollowRedirects())); - headers.put("charset", config.getCharset()); + private void configureHttpsSettings() { + System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2"); + if (Boolean.TRUE.equals(config.getDisableSSLValidation())) { + disableSSLValidation(); + } + } + + public CloseableHttpClient createHttpClient(String pageUriStr) { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + + // set timeouts + long connectTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getConnectTimeout()); + long readTimeoutMillis = TimeUnit.SECONDS.toMillis(config.getReadTimeout()); + RequestConfig.Builder requestBuilder = RequestConfig.custom(); + requestBuilder.setSocketTimeout((int) readTimeoutMillis); + requestBuilder.setConnectTimeout((int) connectTimeoutMillis); + requestBuilder.setConnectionRequestTimeout((int) connectTimeoutMillis); + httpClientBuilder.setDefaultRequestConfig(requestBuilder.build()); + + // basic auth + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + if (!Strings.isNullOrEmpty(config.getUsername()) && !Strings.isNullOrEmpty(config.getPassword())) { + URI uri = URI.create(pageUriStr); + AuthScope authScope = new AuthScope(new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme())); + credentialsProvider.setCredentials(authScope, + new UsernamePasswordCredentials(config.getUsername(), config.getPassword())); + } - if ((config.getMethod().equals(REQUEST_METHOD_POST) - || config.getMethod().equals(REQUEST_METHOD_PATCH) - || config.getMethod().equals(REQUEST_METHOD_PUT)) && !headers.containsKey("Content-Type")) { - headers.put("Content-Type", contentType); - } + // proxy and proxy auth + if (!Strings.isNullOrEmpty(config.getProxyUrl())) { + HttpHost proxyHost = HttpHost.create(config.getProxyUrl()); + if (!Strings.isNullOrEmpty(config.getProxyUsername()) && !Strings.isNullOrEmpty(config.getProxyPassword())) { + credentialsProvider.setCredentials(new AuthScope(proxyHost), + new UsernamePasswordCredentials( + config.getProxyUsername(), config.getProxyPassword())); + } + httpClientBuilder.setProxy(proxyHost); + } + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpClientBuilder.build(); + } - // set default headers - if (headers != null) { - for (Map.Entry headerEntry : this.headers.entrySet()) { - clientHeaders.add(new BasicHeader(headerEntry.getKey(), headerEntry.getValue())); - } - } + private Header[] getRequestHeaders() throws IOException { + ArrayList
clientHeaders = new ArrayList<>(); - return clientHeaders.toArray(new Header[clientHeaders.size()]); + if (accessToken == null || OAuthUtil.tokenExpired(accessToken)) { + accessToken = OAuthUtil.getAccessToken(config); } - private Header getAuthorizationHeader(AccessToken accessToken) { - return new BasicHeader("Authorization", String.format("Bearer %s", accessToken.getTokenValue())); + if (accessToken != null) { + Header authorizationHeader = getAuthorizationHeader(accessToken); + clientHeaders.add(authorizationHeader); } - /** - * @return List of placeholders which should be replaced by actual value in the URL. - */ - private List getPlaceholderListFromURL() { - List placeholderList = new ArrayList<>(); - if (!(config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || - config.getMethod().equals(REQUEST_METHOD_DELETE))) { - return placeholderList; - } - Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); - Matcher matcher = pattern.matcher(url); - while (matcher.find()) { - placeholderList.add(new PlaceholderBean(url, matcher.group(1))); - } - return placeholderList; // Return blank list if no match found - } + headers.put("Request-Method", config.getMethod().toUpperCase()); + headers.put("Instance-Follow-Redirects", String.valueOf(config.getFollowRedirects())); + headers.put("charset", config.getCharset()); - private String updateURLWithPlaceholderValue(StructuredRecord inputRecord) { - try { - StringBuilder finalURLBuilder = new StringBuilder(url); - //Running a loop backwards so that it does not impact the start and end index for next record. - for (int i = placeHolderList.size() - 1; i >= 0; i--) { - PlaceholderBean key = placeHolderList.get(i); - String replacement = inputRecord.get(key.getPlaceHolderKey()); - if (replacement != null) { - String encodedReplacement = URLEncoder.encode(replacement, config.getCharset()); - finalURLBuilder.replace(key.getStartIndex(), key.getEndIndex(), encodedReplacement); - } - } - return finalURLBuilder.toString(); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Error encoding URL with placeholder value. Reason: " + e.getMessage(), e); - } + if ((config.getMethod().equals(REQUEST_METHOD_POST) + || config.getMethod().equals(REQUEST_METHOD_PATCH) + || config.getMethod().equals(REQUEST_METHOD_PUT)) && !headers.containsKey("Content-Type")) { + headers.put("Content-Type", contentType); } - /** - * Clears the message buffer if it is empty and the HTTP method is not 'DELETE'. - */ - private void flushMessageBuffer() { - if (messageBuffer.isEmpty() && !config.getMethod().equals(REQUEST_METHOD_DELETE)) { - return; - } - contentType = messageBuffer.getContentType(); - try { - Awaitility - .await().with() - .pollInterval(pollInterval) - .pollDelay(config.getWaitTimeBetweenPages(), TimeUnit.MILLISECONDS) - .timeout(config.getMaxRetryDuration(), TimeUnit.SECONDS) - .until(this::executeHTTPServiceAndCheckStatusCode); - } catch (Exception e) { - String errorMessage = "Error while executing http request for remaining input messages after the batch execution."; - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new RuntimeException(errorMessage)); - } - messageBuffer.clear(); - ErrorHandling postRetryStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode) - .getAfterRetryStrategy(); - - switch (postRetryStrategy) { - case SUCCESS: - break; - case STOP: - throw new IllegalStateException(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", - config.getUrl(), httpStatusCode, httpResponseBody)); - case SKIP: - case SEND: - LOG.warn(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", - config.getUrl(), httpStatusCode, httpResponseBody)); - break; - default: - throw new IllegalArgumentException(String.format("Unexpected http error handling: '%s'", postRetryStrategy)); - } + // set default headers + if (headers != null) { + for (Map.Entry headerEntry : this.headers.entrySet()) { + clientHeaders.add(new BasicHeader(headerEntry.getKey(), headerEntry.getValue())); + } + } + return clientHeaders.toArray(new Header[clientHeaders.size()]); + } + + private Header getAuthorizationHeader(AccessToken accessToken) { + return new BasicHeader("Authorization", String.format("Bearer %s", accessToken.getTokenValue())); + } + + /** + * @return List of placeholders which should be replaced by actual value in the URL. + */ + private List getPlaceholderListFromURL() { + List placeholderList = new ArrayList<>(); + if (!(config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || + config.getMethod().equals(REQUEST_METHOD_DELETE))) { + return placeholderList; + } + Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); + Matcher matcher = pattern.matcher(url); + while (matcher.find()) { + placeholderList.add(new PlaceholderBean(url, matcher.group(1))); + } + return placeholderList; // Return blank list if no match found + } + + private String updateURLWithPlaceholderValue(StructuredRecord inputRecord) { + try { + StringBuilder finalURLBuilder = new StringBuilder(url); + //Running a loop backwards so that it does not impact the start and end index for next record. + for (int i = placeHolderList.size() - 1; i >= 0; i--) { + PlaceholderBean key = placeHolderList.get(i); + String replacement = inputRecord.get(key.getPlaceHolderKey()); + if (replacement != null) { + String encodedReplacement = URLEncoder.encode(replacement, config.getCharset()); + finalURLBuilder.replace(key.getStartIndex(), key.getEndIndex(), encodedReplacement); + } + } + return finalURLBuilder.toString(); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Error encoding URL with placeholder value. Reason: " + e.getMessage(), e); + } + } + + /** + * Clears the message buffer if it is empty and the HTTP method is not 'DELETE'. + */ + private void flushMessageBuffer() { + if (messageBuffer.isEmpty() && !config.getMethod().equals(REQUEST_METHOD_DELETE)) { + return; + } + contentType = messageBuffer.getContentType(); + try { + Awaitility + .await().with() + .pollInterval(pollInterval) + .pollDelay(config.getWaitTimeBetweenPages(), TimeUnit.MILLISECONDS) + .timeout(config.getMaxRetryDuration(), TimeUnit.SECONDS) + .until(this::executeHTTPServiceAndCheckStatusCode); + } catch (Exception e) { + String errorMessage = "Error while executing http request for remaining input messages after the batch execution."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new RuntimeException(errorMessage)); } + messageBuffer.clear(); + + ErrorHandling postRetryStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode) + .getAfterRetryStrategy(); + + switch (postRetryStrategy) { + case SUCCESS: + break; + case STOP: + throw new IllegalStateException(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", + config.getUrl(), httpStatusCode, httpResponseBody)); + case SKIP: + case SEND: + LOG.warn(String.format("Fetching from url '%s' returned status code '%d' and body '%s'", + config.getUrl(), httpStatusCode, httpResponseBody)); + break; + default: + throw new IllegalArgumentException(String.format("Unexpected http error handling: '%s'", postRetryStrategy)); + } + + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java index 8d2aab31..f4190b2f 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java @@ -47,75 +47,75 @@ @Name("HTTP") @Description("Sink plugin to send the messages from the pipeline to an external http endpoint.") public class HTTPSink extends BatchSink { + private final HTTPSinkConfig config; + + public HTTPSink(HTTPSinkConfig config) { + this.config = config; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); + FailureCollector collector = stageConfigurer.getFailureCollector(); + config.validate(collector); + config.validateSchema(stageConfigurer.getInputSchema(), collector); + collector.getOrThrowException(); + } + + @Override + public void prepareRun(BatchSinkContext context) { + FailureCollector collector = context.getFailureCollector(); + config.validate(collector); + config.validateSchema(context.getInputSchema(), collector); + collector.getOrThrowException(); + + Schema inputSchema = context.getInputSchema(); + Asset asset = Asset.builder(config.getReferenceNameOrNormalizedFQN()) + .setFqn(config.getUrl()).build(); + LineageRecorder lineageRecorder = new LineageRecorder(context, asset); + lineageRecorder.createExternalDataset(context.getInputSchema()); + List fields; + if (inputSchema == null) { + fields = Collections.emptyList(); + } else { + assert inputSchema.getFields() != null; + fields = inputSchema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList()); + } + lineageRecorder.recordWrite("Write", String.format("Wrote to HTTP '%s'", config.getUrl()), fields); + + context.addOutput(Output.of(config.getReferenceNameOrNormalizedFQN(), + new HTTPSink.HTTPOutputFormatProvider(config, inputSchema))); + + // set error details provider + context.setErrorDetailsProvider( + new ErrorDetailsProviderSpec(HttpErrorDetailsProvider.class.getName())); + + } + + /** + * Output format provider for HTTP Sink. + */ + private static class HTTPOutputFormatProvider implements OutputFormatProvider { + private static final Gson GSON = new Gson(); private final HTTPSinkConfig config; + private final Schema inputSchema; - public HTTPSink(HTTPSinkConfig config) { - this.config = config; + HTTPOutputFormatProvider(HTTPSinkConfig config, Schema inputSchema) { + this.config = config; + this.inputSchema = inputSchema; } @Override - public void configurePipeline(PipelineConfigurer pipelineConfigurer) { - super.configurePipeline(pipelineConfigurer); - StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); - FailureCollector collector = stageConfigurer.getFailureCollector(); - config.validate(collector); - config.validateSchema(stageConfigurer.getInputSchema(), collector); - collector.getOrThrowException(); + public String getOutputFormatClassName() { + return HTTPOutputFormat.class.getName(); } @Override - public void prepareRun(BatchSinkContext context) { - FailureCollector collector = context.getFailureCollector(); - config.validate(collector); - config.validateSchema(context.getInputSchema(), collector); - collector.getOrThrowException(); - - Schema inputSchema = context.getInputSchema(); - Asset asset = Asset.builder(config.getReferenceNameOrNormalizedFQN()) - .setFqn(config.getUrl()).build(); - LineageRecorder lineageRecorder = new LineageRecorder(context, asset); - lineageRecorder.createExternalDataset(context.getInputSchema()); - List fields; - if (inputSchema == null) { - fields = Collections.emptyList(); - } else { - assert inputSchema.getFields() != null; - fields = inputSchema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList()); - } - lineageRecorder.recordWrite("Write", String.format("Wrote to HTTP '%s'", config.getUrl()), fields); - - context.addOutput(Output.of(config.getReferenceNameOrNormalizedFQN(), - new HTTPSink.HTTPOutputFormatProvider(config, inputSchema))); - - // set error details provider - context.setErrorDetailsProvider( - new ErrorDetailsProviderSpec(HttpErrorDetailsProvider.class.getName())); - - } - - /** - * Output format provider for HTTP Sink. - */ - private static class HTTPOutputFormatProvider implements OutputFormatProvider { - private static final Gson GSON = new Gson(); - private final HTTPSinkConfig config; - private final Schema inputSchema; - - HTTPOutputFormatProvider(HTTPSinkConfig config, Schema inputSchema) { - this.config = config; - this.inputSchema = inputSchema; - } - - @Override - public String getOutputFormatClassName() { - return HTTPOutputFormat.class.getName(); - } - - @Override - public Map getOutputFormatConfiguration() { - return ImmutableMap.of("http.sink.config", GSON.toJson(config), - "http.sink.input.schema", inputSchema == null ? "" : inputSchema.toString()); - } + public Map getOutputFormatConfiguration() { + return ImmutableMap.of("http.sink.config", GSON.toJson(config), + "http.sink.input.schema", inputSchema == null ? "" : inputSchema.toString()); } + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java index c746a2b9..47575aca 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSinkConfig.java @@ -56,573 +56,573 @@ * Config class for {@link HTTPSink}. */ public class HTTPSinkConfig extends BaseHttpConfig { - public static final String URL = "url"; - public static final String METHOD = "method"; - public static final String BATCH_SIZE = "batchSize"; - public static final String WRITE_JSON_AS_ARRAY = "writeJsonAsArray"; - public static final String JSON_BATCH_KEY = "jsonBatchKey"; - public static final String DELIMETER_FOR_MESSAGE = "delimiterForMessages"; - public static final String MESSAGE_FORMAT = "messageFormat"; - public static final String BODY = "body"; - public static final String REQUEST_HEADERS = "requestHeaders"; - public static final String CHARSET = "charset"; - public static final String FOLLOW_REDIRECTS = "followRedirects"; - public static final String DISABLE_SSL_VALIDATION = "disableSSLValidation"; - public static final String PROPERTY_HTTP_ERROR_HANDLING = "httpErrorsHandling"; - public static final String PROPERTY_ERROR_HANDLING = "errorHandling"; - public static final String PROPERTY_RETRY_POLICY = "retryPolicy"; - public static final String PROPERTY_LINEAR_RETRY_INTERVAL = "linearRetryInterval"; - public static final String PROPERTY_MAX_RETRY_DURATION = "maxRetryDuration"; - public static final String CONNECTION_TIMEOUT = "connectTimeout"; - public static final String READ_TIMEOUT = "readTimeout"; - private static final String KV_DELIMITER = ":"; - private static final String DELIMITER = "\n"; - private static final String REGEX_HASHED_VAR = "#(\\w+)"; - private static final String PLACEHOLDER = "#"; - private static final Set METHODS = ImmutableSet.of(HttpMethod.GET, HttpMethod.POST, - HttpMethod.PUT, HttpMethod.DELETE, "PATCH"); - - @Name(URL) - @Description("The URL to post data to. Additionally, a placeholder like #columnName can be added to the URL that " + - "can be substituted with column value at the runtime. E.g. https://customer-url/user/#user_id. Here user_id " + - "column should exist in input schema. (Macro Enabled)") - @Macro - private final String url; - - @Name(METHOD) - @Description("The http request method. Defaults to POST. (Macro Enabled)") - @Macro - private final String method; - - @Name(BATCH_SIZE) - @Description("Batch size. Defaults to 1. (Macro Enabled)") - @Macro - private final Integer batchSize; - - @Name(WRITE_JSON_AS_ARRAY) - @Nullable - @Description("Whether to write json as array. Defaults to false. (Macro Enabled)") - @Macro - private final Boolean writeJsonAsArray; - - @Name(JSON_BATCH_KEY) - @Nullable - @Description("Optional key to be used for wrapping json array as object. " + - "Leave empty for no wrapping of the array (Macro Enabled)") - @Macro - private final String jsonBatchKey; - - @Name(DELIMETER_FOR_MESSAGE) - @Nullable - @Description("Delimiter for messages to be used while batching. Defaults to \"\\n\". (Macro Enabled)") - @Macro - private final String delimiterForMessages; - - @Name(MESSAGE_FORMAT) - @Description("Format to send messsage in. (Macro Enabled)") - @Macro - private final String messageFormat; - - @Name(BODY) - @Nullable - @Description("Optional custom message. This is required if the message format is set to 'Custom'." + - "User can leverage incoming message fields in the post payload. For example-" + - "User has defined payload as \\{ \"messageType\" : \"update\", \"name\" : \"#firstName\" \\}" + - "where #firstName will be substituted for the value that is in firstName in the incoming message. " + - "(Macro enabled)") - @Macro - private final String body; - - @Name(REQUEST_HEADERS) - @Nullable - @Description("Request headers to set when performing the http request. (Macro enabled)") - @Macro - private final String requestHeaders; - - @Name(CHARSET) - @Description("Charset. Defaults to UTF-8. (Macro enabled)") - @Macro - private final String charset; - - @Name(FOLLOW_REDIRECTS) - @Description("Whether to automatically follow redirects. Defaults to true. (Macro enabled)") - @Macro - private final Boolean followRedirects; - - @Name(DISABLE_SSL_VALIDATION) - @Description("If user enables SSL validation, they will be expected to add the certificate to the trustStore" + - " on each machine. Defaults to true. (Macro enabled)") - @Macro - private final Boolean disableSSLValidation; - - @Nullable - @Name(PROPERTY_HTTP_ERROR_HANDLING) - @Description("Defines the error handling strategy to use for certain HTTP response codes." + - "The left column contains a regular expression for HTTP status code. The right column contains an action which" + - "is done in case of match. If HTTP status code matches multiple regular expressions, " + - "the first specified in mapping is matched.") - protected String httpErrorsHandling; - - @Nullable - @Name(PROPERTY_ERROR_HANDLING) - @Description("Error handling strategy to use when the HTTP response cannot be transformed to an output record.") - protected String errorHandling; - - @Nullable - @Name(PROPERTY_RETRY_POLICY) - @Description("Policy used to calculate delay between retries. Default Retry Policy is Exponential.") - protected String retryPolicy; - - @Nullable - @Name(PROPERTY_LINEAR_RETRY_INTERVAL) - @Description("Interval in seconds between retries. Is only used if retry policy is \"linear\".") - @Macro - protected Long linearRetryInterval; - - @Nullable - @Name(PROPERTY_MAX_RETRY_DURATION) - @Description("Maximum time in seconds retries can take. Default value is 600 seconds (10 minute).") - @Macro - protected Long maxRetryDuration; - - @Name(CONNECTION_TIMEOUT) - @Description("Sets the connection timeout in milliseconds. Set to 0 for infinite. Default is 60000 (1 minute). " + - "(Macro enabled)") - @Nullable - @Macro - private final Integer connectTimeout; - - @Name(READ_TIMEOUT) - @Description("The time in milliseconds to wait for a read. Set to 0 for infinite. Defaults to 60000 (1 minute). " + - "(Macro enabled)") - @Nullable - @Macro - private final Integer readTimeout; - - public HTTPSinkConfig(String referenceName, String url, String method, Integer batchSize, - @Nullable String delimiterForMessages, String messageFormat, @Nullable String body, - @Nullable String requestHeaders, String charset, - boolean followRedirects, boolean disableSSLValidation, @Nullable String httpErrorsHandling, - String errorHandling, String retryPolicy, @Nullable Long linearRetryInterval, - Long maxRetryDuration, int readTimeout, int connectTimeout, - String oauth2Enabled, String authType, @Nullable String jsonBatchKey, - Boolean writeJsonAsArray) { - super(referenceName); - this.url = url; - this.method = method; - this.batchSize = batchSize; - this.delimiterForMessages = delimiterForMessages; - this.messageFormat = messageFormat; - this.body = body; - this.requestHeaders = requestHeaders; - this.charset = charset; - this.followRedirects = followRedirects; - this.disableSSLValidation = disableSSLValidation; - this.httpErrorsHandling = httpErrorsHandling; - this.errorHandling = errorHandling; - this.retryPolicy = retryPolicy; - this.linearRetryInterval = linearRetryInterval; - this.maxRetryDuration = maxRetryDuration; - this.readTimeout = readTimeout; - this.connectTimeout = connectTimeout; - this.jsonBatchKey = jsonBatchKey; - this.writeJsonAsArray = writeJsonAsArray; - this.oauth2Enabled = oauth2Enabled; - this.authType = authType; - } - - private HTTPSinkConfig(Builder builder) { - super(builder.referenceName); - url = builder.url; - method = builder.method; - batchSize = builder.batchSize; - delimiterForMessages = builder.delimiterForMessages; - messageFormat = builder.messageFormat; - body = builder.body; - requestHeaders = builder.requestHeaders; - charset = builder.charset; - followRedirects = builder.followRedirects; - disableSSLValidation = builder.disableSSLValidation; - connectTimeout = builder.connectTimeout; - readTimeout = builder.readTimeout; - jsonBatchKey = builder.jsonBatchKey; - writeJsonAsArray = builder.writeJsonAsArray; - oauth2Enabled = builder.oauth2Enabled; - authType = builder.authType; - } - - public static Builder newBuilder() { - return new Builder(); - } - - public static Builder newBuilder(HTTPSinkConfig copy) { - Builder builder = new Builder(); - builder.referenceName = copy.referenceName; - builder.url = copy.getUrl(); - builder.method = copy.getMethod(); - builder.batchSize = copy.getBatchSize(); - builder.delimiterForMessages = copy.getDelimiterForMessages(); - builder.messageFormat = copy.getMessageFormat().getValue(); - builder.body = copy.getBody(); - builder.requestHeaders = copy.getRequestHeaders(); - builder.charset = copy.getCharset(); - builder.followRedirects = copy.getFollowRedirects(); - builder.disableSSLValidation = copy.getDisableSSLValidation(); - builder.connectTimeout = copy.getConnectTimeout(); - builder.readTimeout = copy.getReadTimeout(); - builder.oauth2Enabled = copy.getOAuth2Enabled(); - builder.authType = copy.getAuthTypeString(); - return builder; - } - - public String getUrl() { - return url; - } + public static final String URL = "url"; + public static final String METHOD = "method"; + public static final String BATCH_SIZE = "batchSize"; + public static final String WRITE_JSON_AS_ARRAY = "writeJsonAsArray"; + public static final String JSON_BATCH_KEY = "jsonBatchKey"; + public static final String DELIMETER_FOR_MESSAGE = "delimiterForMessages"; + public static final String MESSAGE_FORMAT = "messageFormat"; + public static final String BODY = "body"; + public static final String REQUEST_HEADERS = "requestHeaders"; + public static final String CHARSET = "charset"; + public static final String FOLLOW_REDIRECTS = "followRedirects"; + public static final String DISABLE_SSL_VALIDATION = "disableSSLValidation"; + public static final String PROPERTY_HTTP_ERROR_HANDLING = "httpErrorsHandling"; + public static final String PROPERTY_ERROR_HANDLING = "errorHandling"; + public static final String PROPERTY_RETRY_POLICY = "retryPolicy"; + public static final String PROPERTY_LINEAR_RETRY_INTERVAL = "linearRetryInterval"; + public static final String PROPERTY_MAX_RETRY_DURATION = "maxRetryDuration"; + public static final String CONNECTION_TIMEOUT = "connectTimeout"; + public static final String READ_TIMEOUT = "readTimeout"; + private static final String KV_DELIMITER = ":"; + private static final String DELIMITER = "\n"; + private static final String REGEX_HASHED_VAR = "#(\\w+)"; + private static final String PLACEHOLDER = "#"; + private static final Set METHODS = ImmutableSet.of(HttpMethod.GET, HttpMethod.POST, + HttpMethod.PUT, HttpMethod.DELETE, "PATCH"); + + @Name(URL) + @Description("The URL to post data to. Additionally, a placeholder like #columnName can be added to the URL that " + + "can be substituted with column value at the runtime. E.g. https://customer-url/user/#user_id. Here user_id " + + "column should exist in input schema. (Macro Enabled)") + @Macro + private final String url; + + @Name(METHOD) + @Description("The http request method. Defaults to POST. (Macro Enabled)") + @Macro + private final String method; + + @Name(BATCH_SIZE) + @Description("Batch size. Defaults to 1. (Macro Enabled)") + @Macro + private final Integer batchSize; + + @Name(WRITE_JSON_AS_ARRAY) + @Nullable + @Description("Whether to write json as array. Defaults to false. (Macro Enabled)") + @Macro + private final Boolean writeJsonAsArray; + + @Name(JSON_BATCH_KEY) + @Nullable + @Description("Optional key to be used for wrapping json array as object. " + + "Leave empty for no wrapping of the array (Macro Enabled)") + @Macro + private final String jsonBatchKey; + + @Name(DELIMETER_FOR_MESSAGE) + @Nullable + @Description("Delimiter for messages to be used while batching. Defaults to \"\\n\". (Macro Enabled)") + @Macro + private final String delimiterForMessages; + + @Name(MESSAGE_FORMAT) + @Description("Format to send messsage in. (Macro Enabled)") + @Macro + private final String messageFormat; + + @Name(BODY) + @Nullable + @Description("Optional custom message. This is required if the message format is set to 'Custom'." + + "User can leverage incoming message fields in the post payload. For example-" + + "User has defined payload as \\{ \"messageType\" : \"update\", \"name\" : \"#firstName\" \\}" + + "where #firstName will be substituted for the value that is in firstName in the incoming message. " + + "(Macro enabled)") + @Macro + private final String body; + + @Name(REQUEST_HEADERS) + @Nullable + @Description("Request headers to set when performing the http request. (Macro enabled)") + @Macro + private final String requestHeaders; + + @Name(CHARSET) + @Description("Charset. Defaults to UTF-8. (Macro enabled)") + @Macro + private final String charset; + + @Name(FOLLOW_REDIRECTS) + @Description("Whether to automatically follow redirects. Defaults to true. (Macro enabled)") + @Macro + private final Boolean followRedirects; + + @Name(DISABLE_SSL_VALIDATION) + @Description("If user enables SSL validation, they will be expected to add the certificate to the trustStore" + + " on each machine. Defaults to true. (Macro enabled)") + @Macro + private final Boolean disableSSLValidation; + + @Nullable + @Name(PROPERTY_HTTP_ERROR_HANDLING) + @Description("Defines the error handling strategy to use for certain HTTP response codes." + + "The left column contains a regular expression for HTTP status code. The right column contains an action which" + + "is done in case of match. If HTTP status code matches multiple regular expressions, " + + "the first specified in mapping is matched.") + protected String httpErrorsHandling; + + @Nullable + @Name(PROPERTY_ERROR_HANDLING) + @Description("Error handling strategy to use when the HTTP response cannot be transformed to an output record.") + protected String errorHandling; + + @Nullable + @Name(PROPERTY_RETRY_POLICY) + @Description("Policy used to calculate delay between retries. Default Retry Policy is Exponential.") + protected String retryPolicy; + + @Nullable + @Name(PROPERTY_LINEAR_RETRY_INTERVAL) + @Description("Interval in seconds between retries. Is only used if retry policy is \"linear\".") + @Macro + protected Long linearRetryInterval; + + @Nullable + @Name(PROPERTY_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default value is 600 seconds (10 minute).") + @Macro + protected Long maxRetryDuration; + + @Name(CONNECTION_TIMEOUT) + @Description("Sets the connection timeout in milliseconds. Set to 0 for infinite. Default is 60000 (1 minute). " + + "(Macro enabled)") + @Nullable + @Macro + private final Integer connectTimeout; + + @Name(READ_TIMEOUT) + @Description("The time in milliseconds to wait for a read. Set to 0 for infinite. Defaults to 60000 (1 minute). " + + "(Macro enabled)") + @Nullable + @Macro + private final Integer readTimeout; + + public HTTPSinkConfig(String referenceName, String url, String method, Integer batchSize, + @Nullable String delimiterForMessages, String messageFormat, @Nullable String body, + @Nullable String requestHeaders, String charset, + boolean followRedirects, boolean disableSSLValidation, @Nullable String httpErrorsHandling, + String errorHandling, String retryPolicy, @Nullable Long linearRetryInterval, + Long maxRetryDuration, int readTimeout, int connectTimeout, + String oauth2Enabled, String authType, @Nullable String jsonBatchKey, + Boolean writeJsonAsArray) { + super(referenceName); + this.url = url; + this.method = method; + this.batchSize = batchSize; + this.delimiterForMessages = delimiterForMessages; + this.messageFormat = messageFormat; + this.body = body; + this.requestHeaders = requestHeaders; + this.charset = charset; + this.followRedirects = followRedirects; + this.disableSSLValidation = disableSSLValidation; + this.httpErrorsHandling = httpErrorsHandling; + this.errorHandling = errorHandling; + this.retryPolicy = retryPolicy; + this.linearRetryInterval = linearRetryInterval; + this.maxRetryDuration = maxRetryDuration; + this.readTimeout = readTimeout; + this.connectTimeout = connectTimeout; + this.jsonBatchKey = jsonBatchKey; + this.writeJsonAsArray = writeJsonAsArray; + this.oauth2Enabled = oauth2Enabled; + this.authType = authType; + } + + private HTTPSinkConfig(Builder builder) { + super(builder.referenceName); + url = builder.url; + method = builder.method; + batchSize = builder.batchSize; + delimiterForMessages = builder.delimiterForMessages; + messageFormat = builder.messageFormat; + body = builder.body; + requestHeaders = builder.requestHeaders; + charset = builder.charset; + followRedirects = builder.followRedirects; + disableSSLValidation = builder.disableSSLValidation; + connectTimeout = builder.connectTimeout; + readTimeout = builder.readTimeout; + jsonBatchKey = builder.jsonBatchKey; + writeJsonAsArray = builder.writeJsonAsArray; + oauth2Enabled = builder.oauth2Enabled; + authType = builder.authType; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static Builder newBuilder(HTTPSinkConfig copy) { + Builder builder = new Builder(); + builder.referenceName = copy.referenceName; + builder.url = copy.getUrl(); + builder.method = copy.getMethod(); + builder.batchSize = copy.getBatchSize(); + builder.delimiterForMessages = copy.getDelimiterForMessages(); + builder.messageFormat = copy.getMessageFormat().getValue(); + builder.body = copy.getBody(); + builder.requestHeaders = copy.getRequestHeaders(); + builder.charset = copy.getCharset(); + builder.followRedirects = copy.getFollowRedirects(); + builder.disableSSLValidation = copy.getDisableSSLValidation(); + builder.connectTimeout = copy.getConnectTimeout(); + builder.readTimeout = copy.getReadTimeout(); + builder.oauth2Enabled = copy.getOAuth2Enabled(); + builder.authType = copy.getAuthTypeString(); + return builder; + } + + public String getUrl() { + return url; + } + + public String getMethod() { + return method; + } + + public Integer getBatchSize() { + return batchSize; + } + + public boolean shouldWriteJsonAsArray() { + return writeJsonAsArray != null && writeJsonAsArray; + } + + public String getJsonBatchKey() { + return jsonBatchKey; + } + + @Nullable + public String getDelimiterForMessages() { + return Strings.isNullOrEmpty(delimiterForMessages) ? "\n" : delimiterForMessages; + } + + public MessageFormatType getMessageFormat() { + return MessageFormatType.valueOf(messageFormat.toUpperCase()); + } + + @Nullable + public String getBody() { + return body; + } + + @Nullable + public String getRequestHeaders() { + return requestHeaders; + } + + public String getCharset() { + return charset; + } + + public Boolean getFollowRedirects() { + return followRedirects; + } + + public Boolean getDisableSSLValidation() { + return disableSSLValidation; + } + + @Nullable + public String getHttpErrorsHandling() { + return httpErrorsHandling; + } + + public ErrorHandling getErrorHandling() { + return getEnumValueByString(ErrorHandling.class, errorHandling, PROPERTY_ERROR_HANDLING); + } + + public RetryPolicy getRetryPolicy() { + if (retryPolicy == null) { + return RetryPolicy.EXPONENTIAL; + } + return getEnumValueByString(RetryPolicy.class, retryPolicy, PROPERTY_RETRY_POLICY); + } + + private static T + getEnumValueByString(Class enumClass, String stringValue, String propertyName) { + return Stream.of(enumClass.getEnumConstants()) + .filter(keyType -> keyType.getValue().equalsIgnoreCase(stringValue)) + .findAny() + .orElseThrow(() -> new InvalidConfigPropertyException( + String.format("Unsupported value for '%s': '%s'", propertyName, stringValue), propertyName)); + } + + @Nullable + public Long getLinearRetryInterval() { + return linearRetryInterval; + } + + public Long getMaxRetryDuration() { + if (maxRetryDuration == null) { + return 600L; + } + return maxRetryDuration; + } + + @Nullable + public Integer getConnectTimeout() { + return connectTimeout; + } + + @Nullable + public Integer getReadTimeout() { + return readTimeout; + } - public String getMethod() { - return method; - } + public Map getRequestHeadersMap() { + return convertHeadersToMap(requestHeaders); + } - public Integer getBatchSize() { - return batchSize; - } + public Map getHeadersMap(String header) { + return convertHeadersToMap(header); + } - public boolean shouldWriteJsonAsArray() { - return writeJsonAsArray != null && writeJsonAsArray; - } + public String getReferenceNameOrNormalizedFQN() { + return Strings.isNullOrEmpty(referenceName) ? ReferenceNames.normalizeFqn(url) : referenceName; + } - public String getJsonBatchKey() { - return jsonBatchKey; - } + public List getHttpErrorHandlingEntries() { + Map httpErrorsHandlingMap = getMapFromKeyValueString(httpErrorsHandling); + List results = new ArrayList<>(httpErrorsHandlingMap.size()); - @Nullable - public String getDelimiterForMessages() { - return Strings.isNullOrEmpty(delimiterForMessages) ? "\n" : delimiterForMessages; + for (Map.Entry entry : httpErrorsHandlingMap.entrySet()) { + String regex = entry.getKey(); + try { + results.add(new HttpErrorHandlerEntity(Pattern.compile(regex), + getEnumValueByString(RetryableErrorHandling.class, + entry.getValue(), PROPERTY_HTTP_ERROR_HANDLING))); + } catch (PatternSyntaxException e) { + // We embed causing exception message into this one. Since this message is shown on UI when validation fails. + throw new InvalidConfigPropertyException( + String.format( + "Error handling regex '%s' is not valid. %s", regex, e.getMessage()), PROPERTY_HTTP_ERROR_HANDLING); + } } + return results; + } - public MessageFormatType getMessageFormat() { - return MessageFormatType.valueOf(messageFormat.toUpperCase()); - } + public static Map getMapFromKeyValueString(String keyValueString) { + Map result = new LinkedHashMap<>(); - @Nullable - public String getBody() { - return body; + if (Strings.isNullOrEmpty(keyValueString)) { + return result; } - @Nullable - public String getRequestHeaders() { - return requestHeaders; + String[] mappings = keyValueString.split(","); + for (String map : mappings) { + String[] columns = map.split(":"); + if (columns.length < 2) { //For scenario where either of key or value not provided + throw new IllegalArgumentException(String.format("Missing value for key %s", columns[0])); + } + result.put(columns[0], columns[1]); } + return result; + } - public String getCharset() { - return charset; - } + public void validate(FailureCollector collector) { + super.validate(collector); - public Boolean getFollowRedirects() { - return followRedirects; + if (!containsMacro(URL)) { + try { + new URL(url); + } catch (MalformedURLException e) { + collector.addFailure(String.format("URL '%s' is malformed: %s", url, e.getMessage()), null) + .withConfigProperty(URL); + } + } + + if (!containsMacro(CONNECTION_TIMEOUT) && Objects.nonNull(connectTimeout) && connectTimeout < 0) { + collector.addFailure("Connection Timeout cannot be a negative number.", null) + .withConfigProperty(CONNECTION_TIMEOUT); + } + + try { + convertHeadersToMap(requestHeaders); + } catch (IllegalArgumentException e) { + collector.addFailure(e.getMessage(), null) + .withConfigProperty(REQUEST_HEADERS); + } + + if (!containsMacro(METHOD) && !METHODS.contains(method.toUpperCase())) { + collector.addFailure( + String.format("Invalid request method %s, must be one of %s.", method, Joiner.on(',').join(METHODS)), null) + .withConfigProperty(METHOD); } - public Boolean getDisableSSLValidation() { - return disableSSLValidation; + if (!containsMacro(BATCH_SIZE) && batchSize != null && batchSize < 1) { + collector.addFailure("Batch size must be greater than 0.", null) + .withConfigProperty(BATCH_SIZE); } - @Nullable - public String getHttpErrorsHandling() { - return httpErrorsHandling; + // Validate Linear Retry Interval + if (!containsMacro(PROPERTY_RETRY_POLICY) && getRetryPolicy() == RetryPolicy.LINEAR) { + assertIsSet(getLinearRetryInterval(), PROPERTY_LINEAR_RETRY_INTERVAL, "retry policy is linear"); + } + if (!containsMacro(READ_TIMEOUT) && Objects.nonNull(readTimeout) && readTimeout < 0) { + collector.addFailure("Read Timeout cannot be a negative number.", null) + .withConfigProperty(READ_TIMEOUT); } - public ErrorHandling getErrorHandling() { - return getEnumValueByString(ErrorHandling.class, errorHandling, PROPERTY_ERROR_HANDLING); + if (!containsMacro(MESSAGE_FORMAT) && !containsMacro("body") && messageFormat.equalsIgnoreCase("Custom") + && body == null) { + collector.addFailure("For Custom message format, message cannot be null.", null) + .withConfigProperty(MESSAGE_FORMAT); } - public RetryPolicy getRetryPolicy() { - if (retryPolicy == null) { - return RetryPolicy.EXPONENTIAL; - } - return getEnumValueByString(RetryPolicy.class, retryPolicy, PROPERTY_RETRY_POLICY); + if (!containsMacro(PROPERTY_MAX_RETRY_DURATION) && Objects.nonNull(maxRetryDuration) && maxRetryDuration < 0) { + collector.addFailure("Max Retry Duration cannot be a negative number.", null) + .withConfigProperty(PROPERTY_MAX_RETRY_DURATION); } + } - private static T - getEnumValueByString(Class enumClass, String stringValue, String propertyName) { - return Stream.of(enumClass.getEnumConstants()) - .filter(keyType -> keyType.getValue().equalsIgnoreCase(stringValue)) - .findAny() - .orElseThrow(() -> new InvalidConfigPropertyException( - String.format("Unsupported value for '%s': '%s'", propertyName, stringValue), propertyName)); + public void validateSchema(@Nullable Schema schema, FailureCollector collector) { + if (schema == null) { + return; + } + List fields = schema.getFields(); + if (fields == null || fields.isEmpty()) { + collector.addFailure("Schema must contain at least one field", null); + throw collector.getOrThrowException(); } - @Nullable - public Long getLinearRetryInterval() { - return linearRetryInterval; + if (containsMacro(URL) || containsMacro(METHOD)) { + return; } - public Long getMaxRetryDuration() { - if (maxRetryDuration == null) { - return 600L; + if ((method.equals("PUT") || method.equals("PATCH") || method.equals("DELETE")) && url.contains(PLACEHOLDER)) { + Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); + Matcher matcher = pattern.matcher(url); + List fieldNames = fields.stream().map(Schema.Field::getName).collect(Collectors.toList()); + while (matcher.find()) { + if (!fieldNames.contains(matcher.group(1))) { + collector.addFailure(String.format("Schema must contain '%s' field mentioned in the url", matcher.group(1)), + null).withConfigProperty(URL); } - return maxRetryDuration; + } } + } - @Nullable - public Integer getConnectTimeout() { - return connectTimeout; + private Map convertHeadersToMap(String headersString) { + Map headersMap = new HashMap<>(); + if (!Strings.isNullOrEmpty(headersString)) { + for (String chunk : headersString.split(DELIMITER)) { + String[] keyValue = chunk.split(KV_DELIMITER, 2); + if (keyValue.length == 2) { + headersMap.put(keyValue[0], keyValue[1]); + } else { + throw new IllegalArgumentException(String.format("Unable to parse key-value pair '%s'.", chunk)); + } + } } + return headersMap; + } - @Nullable - public Integer getReadTimeout() { - return readTimeout; - } + /** + * Builder for creating a {@link HTTPSinkConfig}. + */ + public static final class Builder { + private String referenceName; + private String url; + private String method; + private Integer batchSize; + private Boolean writeJsonAsArray; + private String jsonBatchKey; + private String delimiterForMessages; + private String messageFormat; + private String body; + private String requestHeaders; + private String charset; + private Boolean followRedirects; + private Boolean disableSSLValidation; + private Integer connectTimeout; + private Integer readTimeout; + private String oauth2Enabled; + private String authType; - public Map getRequestHeadersMap() { - return convertHeadersToMap(requestHeaders); + private Builder() { } - public Map getHeadersMap(String header) { - return convertHeadersToMap(header); + public Builder setReferenceName(String referenceName) { + this.referenceName = referenceName; + return this; } - public String getReferenceNameOrNormalizedFQN() { - return Strings.isNullOrEmpty(referenceName) ? ReferenceNames.normalizeFqn(url) : referenceName; + public Builder setUrl(String url) { + this.url = url; + return this; } - public List getHttpErrorHandlingEntries() { - Map httpErrorsHandlingMap = getMapFromKeyValueString(httpErrorsHandling); - List results = new ArrayList<>(httpErrorsHandlingMap.size()); - - for (Map.Entry entry : httpErrorsHandlingMap.entrySet()) { - String regex = entry.getKey(); - try { - results.add(new HttpErrorHandlerEntity(Pattern.compile(regex), - getEnumValueByString(RetryableErrorHandling.class, - entry.getValue(), PROPERTY_HTTP_ERROR_HANDLING))); - } catch (PatternSyntaxException e) { - // We embed causing exception message into this one. Since this message is shown on UI when validation fails. - throw new InvalidConfigPropertyException( - String.format( - "Error handling regex '%s' is not valid. %s", regex, e.getMessage()), PROPERTY_HTTP_ERROR_HANDLING); - } - } - return results; + public Builder setMethod(String method) { + this.method = method; + return this; } - public static Map getMapFromKeyValueString(String keyValueString) { - Map result = new LinkedHashMap<>(); - - if (Strings.isNullOrEmpty(keyValueString)) { - return result; - } - - String[] mappings = keyValueString.split(","); - for (String map : mappings) { - String[] columns = map.split(":"); - if (columns.length < 2) { //For scenario where either of key or value not provided - throw new IllegalArgumentException(String.format("Missing value for key %s", columns[0])); - } - result.put(columns[0], columns[1]); - } - return result; + public Builder setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + return this; } - public void validate(FailureCollector collector) { - super.validate(collector); - - if (!containsMacro(URL)) { - try { - new URL(url); - } catch (MalformedURLException e) { - collector.addFailure(String.format("URL '%s' is malformed: %s", url, e.getMessage()), null) - .withConfigProperty(URL); - } - } - - if (!containsMacro(CONNECTION_TIMEOUT) && Objects.nonNull(connectTimeout) && connectTimeout < 0) { - collector.addFailure("Connection Timeout cannot be a negative number.", null) - .withConfigProperty(CONNECTION_TIMEOUT); - } - - try { - convertHeadersToMap(requestHeaders); - } catch (IllegalArgumentException e) { - collector.addFailure(e.getMessage(), null) - .withConfigProperty(REQUEST_HEADERS); - } - - if (!containsMacro(METHOD) && !METHODS.contains(method.toUpperCase())) { - collector.addFailure( - String.format("Invalid request method %s, must be one of %s.", method, Joiner.on(',').join(METHODS)), null) - .withConfigProperty(METHOD); - } - - if (!containsMacro(BATCH_SIZE) && batchSize != null && batchSize < 1) { - collector.addFailure("Batch size must be greater than 0.", null) - .withConfigProperty(BATCH_SIZE); - } - - // Validate Linear Retry Interval - if (!containsMacro(PROPERTY_RETRY_POLICY) && getRetryPolicy() == RetryPolicy.LINEAR) { - assertIsSet(getLinearRetryInterval(), PROPERTY_LINEAR_RETRY_INTERVAL, "retry policy is linear"); - } - if (!containsMacro(READ_TIMEOUT) && Objects.nonNull(readTimeout) && readTimeout < 0) { - collector.addFailure("Read Timeout cannot be a negative number.", null) - .withConfigProperty(READ_TIMEOUT); - } - - if (!containsMacro(MESSAGE_FORMAT) && !containsMacro("body") && messageFormat.equalsIgnoreCase("Custom") - && body == null) { - collector.addFailure("For Custom message format, message cannot be null.", null) - .withConfigProperty(MESSAGE_FORMAT); - } - - if (!containsMacro(PROPERTY_MAX_RETRY_DURATION) && Objects.nonNull(maxRetryDuration) && maxRetryDuration < 0) { - collector.addFailure("Max Retry Duration cannot be a negative number.", null) - .withConfigProperty(PROPERTY_MAX_RETRY_DURATION); - } + public Builder setWriteJsonAsArray(Boolean writeJsonAsArray) { + this.writeJsonAsArray = writeJsonAsArray; + return this; } - public void validateSchema(@Nullable Schema schema, FailureCollector collector) { - if (schema == null) { - return; - } - List fields = schema.getFields(); - if (fields == null || fields.isEmpty()) { - collector.addFailure("Schema must contain at least one field", null); - throw collector.getOrThrowException(); - } - - if (containsMacro(URL) || containsMacro(METHOD)) { - return; - } - - if ((method.equals("PUT") || method.equals("PATCH") || method.equals("DELETE")) && url.contains(PLACEHOLDER)) { - Pattern pattern = Pattern.compile(REGEX_HASHED_VAR); - Matcher matcher = pattern.matcher(url); - List fieldNames = fields.stream().map(Schema.Field::getName).collect(Collectors.toList()); - while (matcher.find()) { - if (!fieldNames.contains(matcher.group(1))) { - collector.addFailure(String.format("Schema must contain '%s' field mentioned in the url", matcher.group(1)), - null).withConfigProperty(URL); - } - } - } + public Builder setJsonBatchKey(String jsonBatchKey) { + this.jsonBatchKey = jsonBatchKey; + return this; } - private Map convertHeadersToMap(String headersString) { - Map headersMap = new HashMap<>(); - if (!Strings.isNullOrEmpty(headersString)) { - for (String chunk : headersString.split(DELIMITER)) { - String[] keyValue = chunk.split(KV_DELIMITER, 2); - if (keyValue.length == 2) { - headersMap.put(keyValue[0], keyValue[1]); - } else { - throw new IllegalArgumentException(String.format("Unable to parse key-value pair '%s'.", chunk)); - } - } - } - return headersMap; - } - - /** - * Builder for creating a {@link HTTPSinkConfig}. - */ - public static final class Builder { - private String referenceName; - private String url; - private String method; - private Integer batchSize; - private Boolean writeJsonAsArray; - private String jsonBatchKey; - private String delimiterForMessages; - private String messageFormat; - private String body; - private String requestHeaders; - private String charset; - private Boolean followRedirects; - private Boolean disableSSLValidation; - private Integer connectTimeout; - private Integer readTimeout; - private String oauth2Enabled; - private String authType; - - private Builder() { - } - - public Builder setReferenceName(String referenceName) { - this.referenceName = referenceName; - return this; - } - - public Builder setUrl(String url) { - this.url = url; - return this; - } - - public Builder setMethod(String method) { - this.method = method; - return this; - } - - public Builder setBatchSize(Integer batchSize) { - this.batchSize = batchSize; - return this; - } - - public Builder setWriteJsonAsArray(Boolean writeJsonAsArray) { - this.writeJsonAsArray = writeJsonAsArray; - return this; - } - - public Builder setJsonBatchKey(String jsonBatchKey) { - this.jsonBatchKey = jsonBatchKey; - return this; - } - - public Builder setDelimiterForMessages(String delimiterForMessages) { - this.delimiterForMessages = delimiterForMessages; - return this; - } + public Builder setDelimiterForMessages(String delimiterForMessages) { + this.delimiterForMessages = delimiterForMessages; + return this; + } - public Builder setMessageFormat(String messageFormat) { - this.messageFormat = messageFormat; - return this; - } + public Builder setMessageFormat(String messageFormat) { + this.messageFormat = messageFormat; + return this; + } - public Builder setBody(String body) { - this.body = body; - return this; - } + public Builder setBody(String body) { + this.body = body; + return this; + } - public Builder setRequestHeaders(String requestHeaders) { - this.requestHeaders = requestHeaders; - return this; - } + public Builder setRequestHeaders(String requestHeaders) { + this.requestHeaders = requestHeaders; + return this; + } - public Builder setCharset(String charset) { - this.charset = charset; - return this; - } + public Builder setCharset(String charset) { + this.charset = charset; + return this; + } - public Builder setFollowRedirects(Boolean followRedirects) { - this.followRedirects = followRedirects; - return this; - } + public Builder setFollowRedirects(Boolean followRedirects) { + this.followRedirects = followRedirects; + return this; + } - public Builder setDisableSSLValidation(Boolean disableSSLValidation) { - this.disableSSLValidation = disableSSLValidation; - return this; - } + public Builder setDisableSSLValidation(Boolean disableSSLValidation) { + this.disableSSLValidation = disableSSLValidation; + return this; + } - public Builder setConnectTimeout(Integer connectTimeout) { - this.connectTimeout = connectTimeout; - return this; - } + public Builder setConnectTimeout(Integer connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } - public Builder setReadTimeout(Integer readTimeout) { - this.readTimeout = readTimeout; - return this; - } + public Builder setReadTimeout(Integer readTimeout) { + this.readTimeout = readTimeout; + return this; + } - public HTTPSinkConfig build() { - return new HTTPSinkConfig(this); - } + public HTTPSinkConfig build() { + return new HTTPSinkConfig(this); } + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java b/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java index 9401843b..4c4e5ac6 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/MessageBuffer.java @@ -25,7 +25,10 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Objects; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -37,196 +40,196 @@ * The message is then returned to the HTTPRecordWriter. */ public class MessageBuffer { - private static final String REGEX_HASHED_VAR = "#(\\w+)"; - private final List buffer; - private final String jsonBatchKey; - private final Boolean shouldWriteJsonAsArray; - private final String delimiterForMessages; - private final String charset; - private final String customMessageBody; - private final Function, String> messageFormatter; - private final String contentType; - private final Schema wrappedMessageSchema; - - - /** - * Constructor for MessageBuffer. - * - * @param messageFormat The format of the message. Can be JSON, FORM or CUSTOM. - * @param jsonBatchKey The key to be used for the JSON batch message. - * @param shouldWriteJsonAsArray Whether the JSON message should be written as an array. - * @param delimiterForMessages The delimiter to be used for messages. - * @param charset The charset to be used for the message. - * @param customMessageBody The custom message body to be used. - */ - public MessageBuffer( - MessageFormatType messageFormat, String jsonBatchKey, boolean shouldWriteJsonAsArray, - String delimiterForMessages, String charset, String customMessageBody, Schema inputSchema - ) { - this.jsonBatchKey = jsonBatchKey; - this.delimiterForMessages = delimiterForMessages; - this.charset = charset; - this.shouldWriteJsonAsArray = shouldWriteJsonAsArray; - this.customMessageBody = customMessageBody; - this.buffer = new ArrayList<>(); - switch (messageFormat) { - case JSON: - messageFormatter = this::formatAsJson; - contentType = "application/json"; - break; - case FORM: - messageFormatter = this::formatAsForm; - contentType = "application/x-www-form-urlencoded"; - break; - case CUSTOM: - messageFormatter = this::formatAsCustom; - contentType = "text/plain"; - break; - default: - throw new IllegalArgumentException("Invalid message format: " + messageFormat); - } - // A new StructuredRecord is created with the jsonBatchKey as the - // field name and the array of records as the value - Schema bufferRecordArraySchema = Schema.arrayOf(inputSchema); - wrappedMessageSchema = Schema.recordOf("wrapper", - Schema.Field.of(jsonBatchKey, bufferRecordArraySchema)); + private static final String REGEX_HASHED_VAR = "#(\\w+)"; + private final List buffer; + private final String jsonBatchKey; + private final Boolean shouldWriteJsonAsArray; + private final String delimiterForMessages; + private final String charset; + private final String customMessageBody; + private final Function, String> messageFormatter; + private final String contentType; + private final Schema wrappedMessageSchema; + + + /** + * Constructor for MessageBuffer. + * + * @param messageFormat The format of the message. Can be JSON, FORM or CUSTOM. + * @param jsonBatchKey The key to be used for the JSON batch message. + * @param shouldWriteJsonAsArray Whether the JSON message should be written as an array. + * @param delimiterForMessages The delimiter to be used for messages. + * @param charset The charset to be used for the message. + * @param customMessageBody The custom message body to be used. + */ + public MessageBuffer( + MessageFormatType messageFormat, String jsonBatchKey, boolean shouldWriteJsonAsArray, + String delimiterForMessages, String charset, String customMessageBody, Schema inputSchema + ) { + this.jsonBatchKey = jsonBatchKey; + this.delimiterForMessages = delimiterForMessages; + this.charset = charset; + this.shouldWriteJsonAsArray = shouldWriteJsonAsArray; + this.customMessageBody = customMessageBody; + this.buffer = new ArrayList<>(); + switch (messageFormat) { + case JSON: + messageFormatter = this::formatAsJson; + contentType = "application/json"; + break; + case FORM: + messageFormatter = this::formatAsForm; + contentType = "application/x-www-form-urlencoded"; + break; + case CUSTOM: + messageFormatter = this::formatAsCustom; + contentType = "text/plain"; + break; + default: + throw new IllegalArgumentException("Invalid message format: " + messageFormat); } - - /** - * Adds a record to the buffer. - * - * @param structuredRecord The record to be added. - */ - public void add(StructuredRecord structuredRecord) { - buffer.add(structuredRecord); + // A new StructuredRecord is created with the jsonBatchKey as the + // field name and the array of records as the value + Schema bufferRecordArraySchema = Schema.arrayOf(inputSchema); + wrappedMessageSchema = Schema.recordOf("wrapper", + Schema.Field.of(jsonBatchKey, bufferRecordArraySchema)); + } + + /** + * Adds a record to the buffer. + * + * @param structuredRecord The record to be added. + */ + public void add(StructuredRecord structuredRecord) { + buffer.add(structuredRecord); + } + + /** + * Clears the buffer. + */ + public void clear() { + buffer.clear(); + } + + /** + * Returns the size of the buffer. + */ + public int size() { + return buffer.size(); + } + + /** + * Returns whether the buffer is empty. + */ + public boolean isEmpty() { + return buffer.isEmpty(); + } + + /** + * Returns the content type of the message. + */ + public String getContentType() { + return contentType; + } + + /** + * Converts the buffer to the appropriate format and returns the message. + */ + public String getMessage() throws IOException { + return messageFormatter.apply(buffer); + } + + private String formatAsJson(List buffer) { + try { + return formatAsJsonInternal(buffer); + } catch (IOException e) { + throw new IllegalStateException("Error formatting JSON message. Reason: " + e.getMessage(), e); } + } - /** - * Clears the buffer. - */ - public void clear() { - buffer.clear(); + private String formatAsJsonInternal(List buffer) throws IOException { + boolean useJsonBatchKey = !Strings.isNullOrEmpty(jsonBatchKey); + if (Boolean.TRUE.equals(!shouldWriteJsonAsArray) || !useJsonBatchKey) { + return getBufferAsJsonList(); } - - /** - * Returns the size of the buffer. - */ - public int size() { - return buffer.size(); + StructuredRecord wrappedMessageRecord = StructuredRecord.builder(wrappedMessageSchema) + .set(jsonBatchKey, buffer).build(); + return StructuredRecordStringConverter.toJsonString(wrappedMessageRecord); + } + + private String formatAsForm(List buffer) { + return buffer.stream() + .map(this::createFormMessage) + .collect(Collectors.joining(delimiterForMessages)); + } + + private String formatAsCustom(List buffer) { + return buffer.stream() + .map(this::createCustomMessage) + .collect(Collectors.joining(delimiterForMessages)); + } + + private String getBufferAsJsonList() throws IOException { + StringBuilder sb = new StringBuilder(); + String delimiter = Boolean.TRUE.equals(shouldWriteJsonAsArray) ? "," : delimiterForMessages; + if (Boolean.TRUE.equals(shouldWriteJsonAsArray)) { + sb.append("["); } - - /** - * Returns whether the buffer is empty. - */ - public boolean isEmpty() { - return buffer.isEmpty(); + for (StructuredRecord structuredRecord : buffer) { + sb.append(StructuredRecordStringConverter.toJsonString(structuredRecord)); + sb.append(delimiter); } - - /** - * Returns the content type of the message. - */ - public String getContentType() { - return contentType; + if (!buffer.isEmpty()) { + sb.setLength(sb.length() - delimiter.length()); } - - /** - * Converts the buffer to the appropriate format and returns the message. - */ - public String getMessage() throws IOException { - return messageFormatter.apply(buffer); + if (Boolean.TRUE.equals(shouldWriteJsonAsArray)) { + sb.append("]"); } - - private String formatAsJson(List buffer) { - try { - return formatAsJsonInternal(buffer); - } catch (IOException e) { - throw new IllegalStateException("Error formatting JSON message. Reason: " + e.getMessage(), e); + return sb.toString(); + } + + private String createFormMessage(StructuredRecord input) { + boolean first = true; + String formMessage = null; + StringBuilder sb = new StringBuilder(); + if (input != null && input.getSchema() != null) { + for (Schema.Field field : Objects.requireNonNull(input.getSchema().getFields())) { + if (first) { + first = false; + } else { + sb.append("&"); } + sb.append(field.getName()); + sb.append("="); + sb.append((String) input.get(field.getName())); + } } - - private String formatAsJsonInternal(List buffer) throws IOException { - boolean useJsonBatchKey = !Strings.isNullOrEmpty(jsonBatchKey); - if (Boolean.TRUE.equals(!shouldWriteJsonAsArray) || !useJsonBatchKey) { - return getBufferAsJsonList(); - } - StructuredRecord wrappedMessageRecord = StructuredRecord.builder(wrappedMessageSchema) - .set(jsonBatchKey, buffer).build(); - return StructuredRecordStringConverter.toJsonString(wrappedMessageRecord); - } - - private String formatAsForm(List buffer) { - return buffer.stream() - .map(this::createFormMessage) - .collect(Collectors.joining(delimiterForMessages)); + try { + formMessage = URLEncoder.encode(sb.toString(), charset); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Error encoding Form message. Reason: " + e.getMessage(), e); } - - private String formatAsCustom(List buffer) { - return buffer.stream() - .map(this::createCustomMessage) - .collect(Collectors.joining(delimiterForMessages)); + return formMessage; + } + + private String createCustomMessage(StructuredRecord input) { + String customMessage = customMessageBody; + Matcher matcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); + HashMap findReplaceMap = new HashMap<>(); + while (matcher.find()) { + if (input.get(matcher.group(1)) != null) { + findReplaceMap.put(matcher.group(1), (String) input.get(matcher.group(1))); + } else { + throw new IllegalArgumentException(String.format( + "Field %s doesnt exist in the input schema.", matcher.group(1))); + } } - - private String getBufferAsJsonList() throws IOException { - StringBuilder sb = new StringBuilder(); - String delimiter = Boolean.TRUE.equals(shouldWriteJsonAsArray) ? "," : delimiterForMessages; - if (Boolean.TRUE.equals(shouldWriteJsonAsArray)) { - sb.append("["); - } - for (StructuredRecord structuredRecord : buffer) { - sb.append(StructuredRecordStringConverter.toJsonString(structuredRecord)); - sb.append(delimiter); - } - if (!buffer.isEmpty()) { - sb.setLength(sb.length() - delimiter.length()); - } - if (Boolean.TRUE.equals(shouldWriteJsonAsArray)) { - sb.append("]"); - } - return sb.toString(); - } - - private String createFormMessage(StructuredRecord input) { - boolean first = true; - String formMessage = null; - StringBuilder sb = new StringBuilder(); - if (input != null && input.getSchema() != null) { - for (Schema.Field field : Objects.requireNonNull(input.getSchema().getFields())) { - if (first) { - first = false; - } else { - sb.append("&"); - } - sb.append(field.getName()); - sb.append("="); - sb.append((String) input.get(field.getName())); - } - } - try { - formMessage = URLEncoder.encode(sb.toString(), charset); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Error encoding Form message. Reason: " + e.getMessage(), e); - } - return formMessage; - } - - private String createCustomMessage(StructuredRecord input) { - String customMessage = customMessageBody; - Matcher matcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); - HashMap findReplaceMap = new HashMap<>(); - while (matcher.find()) { - if (input.get(matcher.group(1)) != null) { - findReplaceMap.put(matcher.group(1), (String) input.get(matcher.group(1))); - } else { - throw new IllegalArgumentException(String.format( - "Field %s doesnt exist in the input schema.", matcher.group(1))); - } - } - Matcher replaceMatcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); - while (replaceMatcher.find()) { - String val = replaceMatcher.group().replace("#", ""); - customMessage = (customMessage.replace(replaceMatcher.group(), findReplaceMap.get(val))); - } - return customMessage; + Matcher replaceMatcher = Pattern.compile(REGEX_HASHED_VAR).matcher(customMessage); + while (replaceMatcher.find()) { + String val = replaceMatcher.group().replace("#", ""); + customMessage = (customMessage.replace(replaceMatcher.group(), findReplaceMap.get(val))); } + return customMessage; + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java b/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java index 7db376d4..d219df1c 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/PlaceholderBean.java @@ -20,27 +20,27 @@ * This class stores the placeholder information to avoid performing string functions for each record. */ public class PlaceholderBean { - private static final String PLACEHOLDER_FORMAT = "#%s"; - private final String placeHolderKey; - private final int startIndex; - private final int endIndex; + private static final String PLACEHOLDER_FORMAT = "#%s"; + private final String placeHolderKey; + private final int startIndex; + private final int endIndex; - public PlaceholderBean(String url, String placeHolderKey) { - String placeHolderKeyWithPrefix = String.format(PLACEHOLDER_FORMAT, placeHolderKey); - this.placeHolderKey = placeHolderKey; - this.startIndex = url.indexOf(placeHolderKeyWithPrefix); - this.endIndex = startIndex + placeHolderKeyWithPrefix.length(); - } + public PlaceholderBean(String url, String placeHolderKey) { + String placeHolderKeyWithPrefix = String.format(PLACEHOLDER_FORMAT, placeHolderKey); + this.placeHolderKey = placeHolderKey; + this.startIndex = url.indexOf(placeHolderKeyWithPrefix); + this.endIndex = startIndex + placeHolderKeyWithPrefix.length(); + } - public String getPlaceHolderKey() { - return placeHolderKey; - } + public String getPlaceHolderKey() { + return placeHolderKey; + } - public int getStartIndex() { - return startIndex; - } + public int getStartIndex() { + return startIndex; + } - public int getEndIndex() { - return endIndex; - } + public int getEndIndex() { + return endIndex; + } } diff --git a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java index 9fff34d5..9447971c 100644 --- a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java +++ b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java @@ -31,8 +31,10 @@ import io.cdap.cdap.etl.api.batch.BatchRuntimeContext; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.exception.*; import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.http.common.*; import io.cdap.plugin.http.common.pagination.page.BasePage; import io.cdap.plugin.http.common.pagination.page.PageEntry; import org.apache.hadoop.io.NullWritable; @@ -94,6 +96,9 @@ public void prepareRun(BatchSourceContext context) { .collect(Collectors.toList())); context.setInput(Input.of(config.getReferenceNameOrNormalizedFQN(), new HttpInputFormatProvider(config))); + // set error details provider + context.setErrorDetailsProvider( + new ErrorDetailsProviderSpec(HttpErrorDetailsProvider.class.getName())); } @Override From 58ef802df51384ef963bb5382e52bcdeb27f110f Mon Sep 17 00:00:00 2001 From: Amit Kumar Singh Date: Thu, 5 Dec 2024 08:40:06 +0000 Subject: [PATCH 4/7] Updated error details for http source and common --- .../http/common/HttpErrorDetailsProvider.java | 150 +++++++++--------- .../http/sink/batch/HTTPOutputFormat.java | 4 +- .../http/sink/batch/HTTPRecordWriter.java | 3 +- .../cdap/plugin/http/sink/batch/HTTPSink.java | 6 +- .../http/source/batch/HttpBatchSource.java | 22 +-- .../source/batch/HttpBatchSourceConfig.java | 49 +++--- .../http/source/batch/HttpInputFormat.java | 3 +- .../source/common/BaseHttpSourceConfig.java | 4 +- .../common/DelimitedSchemaDetector.java | 13 +- .../http/source/common/RawStringPerLine.java | 9 +- 10 files changed, 139 insertions(+), 124 deletions(-) diff --git a/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java index 96efba94..882e8985 100644 --- a/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java +++ b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java @@ -26,92 +26,92 @@ import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.etl.api.validation.InvalidConfigPropertyException; -import java.io.UnsupportedEncodingException; import java.util.List; import java.util.NoSuchElementException; /** - Error details provided for the HTTP + * Error details provided for the HTTP **/ public class HttpErrorDetailsProvider implements ErrorDetailsProvider { - @Override - public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { - List causalChain = Throwables.getCausalChain(e); - for (Throwable t : causalChain) { - if (t instanceof ProgramFailureException) { - // if causal chain already has program failure exception, return null to avoid double wrap. + @Override + public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { + List causalChain = Throwables.getCausalChain(e); + for (Throwable t : causalChain) { + if (t instanceof ProgramFailureException) { + // if causal chain already has program failure exception, return null to avoid double wrap. + return null; + } + if (t instanceof IllegalArgumentException) { + return getProgramFailureException((IllegalArgumentException) t, errorContext); + } + if (t instanceof IllegalStateException) { + return getProgramFailureException((IllegalStateException) t, errorContext); + } + if (t instanceof InvalidConfigPropertyException) { + return getProgramFailureException((InvalidConfigPropertyException) t, errorContext); + } + if (t instanceof NoSuchElementException) { + return getProgramFailureException((NoSuchElementException) t, errorContext); + } + } return null; - } - if (t instanceof IllegalArgumentException) { - return getProgramFailureException((IllegalArgumentException) t, errorContext); - } - if (t instanceof IllegalStateException) { - return getProgramFailureException((IllegalStateException) t, errorContext); - } - if (t instanceof InvalidConfigPropertyException) { - return getProgramFailureException((InvalidConfigPropertyException) t, errorContext); - } - if (t instanceof NoSuchElementException) { - return getProgramFailureException((NoSuchElementException) t, errorContext); - } } - return null; - } - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalArgumentException}. - * - * @param e The IllegalArgumentException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); - } + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalArgumentException}. + * + * @param e The IllegalArgumentException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); + } - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalStateException}. - * - * @param e The IllegalStateException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } + /** + * Get a ProgramFailureException with the given error + * information from {@link IllegalStateException}. + * + * @param e The IllegalStateException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } - /** - * Get a ProgramFailureException with the given error - * information from {@link InvalidConfigPropertyException}. - * - * @param e The InvalidConfigPropertyException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(InvalidConfigPropertyException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } + /** + * Get a ProgramFailureException with the given error + * information from {@link InvalidConfigPropertyException}. + * + * @param e The InvalidConfigPropertyException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(InvalidConfigPropertyException e, + ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } - /** - * Get a ProgramFailureException with the given error - * information from {@link NoSuchElementException}. - * - * @param e The NoSuchElementException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(NoSuchElementException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } + /** + * Get a ProgramFailureException with the given error + * information from {@link NoSuchElementException}. + * + * @param e The NoSuchElementException to get the error information from. + * @return A ProgramFailureException with the given error information. + */ + private ProgramFailureException getProgramFailureException(NoSuchElementException e, ErrorContext errorContext) { + String errorMessage = e.getMessage(); + String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); + } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java index f784a876..902ec782 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java @@ -19,7 +19,9 @@ import com.google.gson.Gson; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.api.exception.*; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.OutputCommitter; diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java index 9d69af80..df49467f 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java @@ -376,7 +376,8 @@ private void flushMessageBuffer() { .timeout(config.getMaxRetryDuration(), TimeUnit.SECONDS) .until(this::executeHTTPServiceAndCheckStatusCode); } catch (Exception e) { - String errorMessage = "Error while executing http request for remaining input messages after the batch execution."; + String errorMessage = "Error while executing http request for remaining input messages" + + " after the batch execution."; throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new RuntimeException(errorMessage)); } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java index f4190b2f..7c0f311b 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java @@ -84,13 +84,13 @@ public void prepareRun(BatchSinkContext context) { } lineageRecorder.recordWrite("Write", String.format("Wrote to HTTP '%s'", config.getUrl()), fields); - context.addOutput(Output.of(config.getReferenceNameOrNormalizedFQN(), - new HTTPSink.HTTPOutputFormatProvider(config, inputSchema))); - // set error details provider context.setErrorDetailsProvider( new ErrorDetailsProviderSpec(HttpErrorDetailsProvider.class.getName())); + context.addOutput(Output.of(config.getReferenceNameOrNormalizedFQN(), + new HTTPSink.HTTPOutputFormatProvider(config, inputSchema))); + } /** diff --git a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java index 9447971c..108027e3 100644 --- a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java +++ b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSource.java @@ -31,16 +31,19 @@ import io.cdap.cdap.etl.api.batch.BatchRuntimeContext; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; -import io.cdap.cdap.etl.api.exception.*; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.LineageRecorder; -import io.cdap.plugin.http.common.*; -import io.cdap.plugin.http.common.pagination.page.BasePage; + +import io.cdap.plugin.http.common.HttpErrorDetailsProvider; import io.cdap.plugin.http.common.pagination.page.PageEntry; import org.apache.hadoop.io.NullWritable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -90,15 +93,16 @@ public void prepareRun(BatchSourceContext context) { .setFqn(config.getUrl()).build(); LineageRecorder lineageRecorder = new LineageRecorder(context, asset); lineageRecorder.createExternalDataset(schema); - lineageRecorder.recordRead("Read", String.format("Read from HTTP '%s'", config.getUrl()), - Preconditions.checkNotNull(schema.getFields()).stream() - .map(Schema.Field::getName) - .collect(Collectors.toList())); + List getNameList = Objects.nonNull(schema) ? Preconditions.checkNotNull(schema.getFields()).stream() + .map(Schema.Field::getName) + .collect(Collectors.toList()) : new ArrayList<>(); + lineageRecorder.recordRead("Read", String.format("Read from HTTP '%s'", config.getUrl()), getNameList); - context.setInput(Input.of(config.getReferenceNameOrNormalizedFQN(), new HttpInputFormatProvider(config))); // set error details provider context.setErrorDetailsProvider( - new ErrorDetailsProviderSpec(HttpErrorDetailsProvider.class.getName())); + new ErrorDetailsProviderSpec(HttpErrorDetailsProvider.class.getName())); + + context.setInput(Input.of(config.getReferenceNameOrNormalizedFQN(), new HttpInputFormatProvider(config))); } @Override diff --git a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java index 0f96f996..5f735e7c 100644 --- a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java +++ b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java @@ -17,7 +17,9 @@ import com.google.common.base.Strings; import com.google.gson.JsonSyntaxException; -import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.http.common.http.AuthType; import io.cdap.plugin.http.common.http.HttpClient; @@ -59,19 +61,14 @@ public void validate(FailureCollector failureCollector) { } public void validateCredentials(FailureCollector collector) { - try { - if (getAuthType() == AuthType.OAUTH2) { - validateOAuth2Credentials(collector); - } else if (getAuthType() == AuthType.BASIC_AUTH) { - validateBasicAuthCredentials(collector); - } - } catch (IOException e) { - String errorMessage = "Unable to authenticate the given info : " + e.getMessage(); - collector.addFailure(errorMessage, null); + if (getAuthType() == AuthType.OAUTH2) { + validateOAuth2Credentials(collector); + } else if (getAuthType() == AuthType.BASIC_AUTH) { + validateBasicAuthCredentials(collector); } } - private void validateOAuth2Credentials(FailureCollector collector) throws IOException { + private void validateOAuth2Credentials(FailureCollector collector) { if (!containsMacro(PROPERTY_CLIENT_ID) && !containsMacro(PROPERTY_CLIENT_SECRET) && !containsMacro(PROPERTY_TOKEN_URL) && !containsMacro(PROPERTY_REFRESH_TOKEN) && !containsMacro(PROPERTY_PROXY_PASSWORD) && !containsMacro(PROPERTY_PROXY_USERNAME) && @@ -93,25 +90,24 @@ private void validateOAuth2Credentials(FailureCollector collector) throws IOExce } catch (JsonSyntaxException | HttpHostConnectException e) { String errorMessage = "Error occurred during credential validation : " + e.getMessage(); collector.addFailure(errorMessage, null); + } catch (IOException e) { + String errorMessage = "Unable to validate OAuth and process the request."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); } } } - public void validateBasicAuthCredentials(FailureCollector collector) throws IOException { - try { - if (!containsMacro(PROPERTY_URL) && !containsMacro(PROPERTY_USERNAME) && !containsMacro(PROPERTY_PASSWORD) && - !containsMacro(PROPERTY_PROXY_USERNAME) && !containsMacro(PROPERTY_PROXY_PASSWORD) - && !containsMacro(PROPERTY_PROXY_URL)) { - HttpClient httpClient = new HttpClient(this); - validateBasicAuthResponse(collector, httpClient); - } - } catch (HttpHostConnectException e) { - String errorMessage = "Error occurred during credential validation : " + e.getMessage(); - collector.addFailure(errorMessage, "Please ensure that correct credentials are provided."); + public void validateBasicAuthCredentials(FailureCollector collector) { + if (!containsMacro(PROPERTY_URL) && !containsMacro(PROPERTY_USERNAME) && !containsMacro(PROPERTY_PASSWORD) && + !containsMacro(PROPERTY_PROXY_USERNAME) && !containsMacro(PROPERTY_PROXY_PASSWORD) + && !containsMacro(PROPERTY_PROXY_URL)) { + HttpClient httpClient = new HttpClient(this); + validateBasicAuthResponse(collector, httpClient); } } - public void validateBasicAuthResponse(FailureCollector collector, HttpClient httpClient) throws IOException { + public void validateBasicAuthResponse(FailureCollector collector, HttpClient httpClient) { try (CloseableHttpResponse response = httpClient.executeHTTP(getUrl())) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { @@ -123,6 +119,10 @@ public void validateBasicAuthResponse(FailureCollector collector, HttpClient htt collector.addFailure(errorMessage, "Please ensure that correct credentials are provided."); } } + } catch (IOException e) { + String errorMessage = "Unable to process the response and validate credentials"; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); } } @@ -195,10 +195,11 @@ public static class HttpBatchSourceConfigBuilder { private String password; - public HttpBatchSourceConfigBuilder setReferenceName (String referenceName) { + public HttpBatchSourceConfigBuilder setReferenceName(String referenceName) { this.referenceName = referenceName; return this; } + public HttpBatchSourceConfigBuilder setAuthUrl(String authUrl) { this.authUrl = authUrl; return this; diff --git a/src/main/java/io/cdap/plugin/http/source/batch/HttpInputFormat.java b/src/main/java/io/cdap/plugin/http/source/batch/HttpInputFormat.java index 69cee410..857fe72b 100644 --- a/src/main/java/io/cdap/plugin/http/source/batch/HttpInputFormat.java +++ b/src/main/java/io/cdap/plugin/http/source/batch/HttpInputFormat.java @@ -15,7 +15,6 @@ */ package io.cdap.plugin.http.source.batch; -import io.cdap.plugin.http.common.pagination.page.BasePage; import io.cdap.plugin.http.common.pagination.page.PageEntry; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -29,7 +28,7 @@ /** * InputFormat for mapreduce job, which provides a single split of data. Since in general pagination cannot - * parallelized. + * parallelize. */ public class HttpInputFormat extends InputFormat { @Override diff --git a/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java b/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java index acfa09f7..c17f0536 100644 --- a/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java +++ b/src/main/java/io/cdap/plugin/http/source/common/BaseHttpSourceConfig.java @@ -45,6 +45,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Stream; @@ -545,7 +546,7 @@ public Map getFullFieldsMapping() { Map result = new HashMap<>(); if (!Strings.isNullOrEmpty(schema)) { - for (Schema.Field field : getSchema().getFields()) { + for (Schema.Field field : Objects.requireNonNull(Objects.requireNonNull(getSchema()).getFields())) { result.put(field.getName(), "/" + field.getName()); } } @@ -563,6 +564,7 @@ public String getReferenceNameOrNormalizedFQN() { return Strings.isNullOrEmpty(referenceName) ? ReferenceNames.normalizeFqn(url) : referenceName; } + public void validate(FailureCollector failureCollector) { super.validate(failureCollector); diff --git a/src/main/java/io/cdap/plugin/http/source/common/DelimitedSchemaDetector.java b/src/main/java/io/cdap/plugin/http/source/common/DelimitedSchemaDetector.java index d7bc3665..9172981c 100644 --- a/src/main/java/io/cdap/plugin/http/source/common/DelimitedSchemaDetector.java +++ b/src/main/java/io/cdap/plugin/http/source/common/DelimitedSchemaDetector.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Objects; /** * Class that detects the schema of the delimited file. @@ -45,23 +46,23 @@ public static Schema detectSchema(HttpBatchSourceConfig config, String delimiter rowValue = getRowValues(line, config.getEnableQuotesValues(), delimiter); if (rowIndex == 0) { columnNames = DataTypeDetectorUtils.setColumnNames(line, config.getCsvSkipFirstRow(), - config.getEnableQuotesValues(), delimiter); - if (config.getCsvSkipFirstRow()) { + config.getEnableQuotesValues(), delimiter); + if (Boolean.TRUE.equals(config.getCsvSkipFirstRow())) { continue; } } DataTypeDetectorUtils.detectDataTypeOfRowValues(new HashMap<>(), dataTypeDetectorStatusKeeper, columnNames, - rowValue); + rowValue); } dataTypeDetectorStatusKeeper.validateDataTypeDetector(); } catch (Exception e) { failureCollector.addFailure(String.format("Error while reading the file to infer the schema. Error: %s", - e.getMessage()), null) - .withStacktrace(e.getStackTrace()); + e.getMessage()), null) + .withStacktrace(e.getStackTrace()); return null; } List fields = DataTypeDetectorUtils.detectDataTypeOfEachDatasetColumn( - new HashMap<>(), columnNames, dataTypeDetectorStatusKeeper); + new HashMap<>(), (Objects.nonNull(columnNames) ? columnNames : new String[0]), dataTypeDetectorStatusKeeper); return Schema.recordOf("text", fields); } diff --git a/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java b/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java index 2943d0d1..2afe5bfd 100644 --- a/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java +++ b/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java @@ -16,6 +16,9 @@ package io.cdap.plugin.http.source.common; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.plugin.http.common.http.HttpResponse; import java.io.BufferedReader; @@ -61,14 +64,16 @@ public boolean hasNext() { isLineRead = true; return lastLine != null; } catch (IOException e) { // we need to catch this, since hasNext() does not have "throws" in parent - throw new RuntimeException("Failed to read line from http page buffer", e); + String errorMessage = "Unable to read line from http page buffer"; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); } } @Override public String next() { if (!hasNext()) { // calling hasNext will also read the line; - throw new NoSuchElementException(); + throw new NoSuchElementException("Unable to read the next line."); } isLineRead = false; return lastLine; From 3021427c990dd29e464ec1400a9f6bea206e063d Mon Sep 17 00:00:00 2001 From: Amit Kumar Singh Date: Thu, 5 Dec 2024 09:44:38 +0000 Subject: [PATCH 5/7] check style fixes --- .../http/common/HttpErrorDetailsProvider.java | 17 ++++++++++------- .../cdap/plugin/http/sink/batch/HTTPSink.java | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java index 882e8985..f16dccf0 100644 --- a/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java +++ b/src/main/java/io/cdap/plugin/http/common/HttpErrorDetailsProvider.java @@ -16,14 +16,13 @@ package io.cdap.plugin.http.common; +import com.google.common.base.Throwables; import io.cdap.cdap.api.exception.ErrorCategory; import io.cdap.cdap.api.exception.ErrorType; import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.etl.api.exception.ErrorContext; import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; -import com.google.common.base.Throwables; -import io.cdap.cdap.api.exception.ErrorCategory.ErrorCategoryEnum; -import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.etl.api.validation.InvalidConfigPropertyException; import java.util.List; @@ -68,7 +67,8 @@ public ProgramFailureException getExceptionDetails(Exception e, ErrorContext err private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { String errorMessage = e.getMessage(); String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); } @@ -82,7 +82,8 @@ private ProgramFailureException getProgramFailureException(IllegalArgumentExcept private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { String errorMessage = e.getMessage(); String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); } @@ -97,7 +98,8 @@ private ProgramFailureException getProgramFailureException(InvalidConfigProperty ErrorContext errorContext) { String errorMessage = e.getMessage(); String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); } @@ -111,7 +113,8 @@ private ProgramFailureException getProgramFailureException(InvalidConfigProperty private ProgramFailureException getProgramFailureException(NoSuchElementException e, ErrorContext errorContext) { String errorMessage = e.getMessage(); String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategoryEnum.PLUGIN), errorMessage, + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java index 7c0f311b..dd2f2d35 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java @@ -30,9 +30,9 @@ import io.cdap.cdap.etl.api.StageConfigurer; import io.cdap.cdap.etl.api.batch.BatchSink; import io.cdap.cdap.etl.api.batch.BatchSinkContext; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.LineageRecorder; -import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.plugin.http.common.HttpErrorDetailsProvider; import java.util.Collections; From 0256935126ec2bc234d2ea66e678079f28b19045 Mon Sep 17 00:00:00 2001 From: Amit Kumar Singh Date: Thu, 5 Dec 2024 11:22:02 +0000 Subject: [PATCH 6/7] pr comments fix --- .../http/sink/batch/HTTPOutputFormat.java | 4 +--- .../http/sink/batch/HTTPRecordWriter.java | 19 ++++++------------- .../cdap/plugin/http/sink/batch/HTTPSink.java | 5 +++-- .../source/batch/HttpBatchSourceConfig.java | 5 ++++- .../http/source/common/RawStringPerLine.java | 2 +- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java index 902ec782..03aedd78 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java @@ -48,9 +48,7 @@ public RecordWriter getRecordWriter(TaskAtte inputSchema = Schema.parseJson(hConf.get(INPUT_SCHEMA_KEY)); return new HTTPRecordWriter(config, inputSchema); } catch (IOException e) { - String errorMessage = "Unable to parse and write the record"; - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); + throw new IllegalStateException("Unable to parse the input schema. Reason: " + e.getMessage(), e); } } diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java index df49467f..848637d1 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPRecordWriter.java @@ -57,7 +57,6 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; -import java.net.ProtocolException; import java.net.URI; import java.net.URL; import java.net.URLEncoder; @@ -75,7 +74,6 @@ import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; @@ -133,8 +131,7 @@ public void write(StructuredRecord input, StructuredRecord unused) { } if (config.getMethod().equals(REQUEST_METHOD_PUT) || config.getMethod().equals(REQUEST_METHOD_PATCH) || - config.getMethod().equals(REQUEST_METHOD_DELETE) - && !placeHolderList.isEmpty()) { + config.getMethod().equals(REQUEST_METHOD_DELETE) && !placeHolderList.isEmpty()) { configURL = updateURLWithPlaceholderValue(input); } @@ -180,15 +177,11 @@ private boolean executeHTTPServiceAndCheckStatusCode() { LOG.debug("HTTP Request Attempt No. : {}", ++retryCount); // Try-with-resources ensures proper resource management - try (CloseableHttpClient httpClient = createHttpClient(configURL)) { - URL url = new URL(configURL); - - // Use try-with-resources to ensure response is closed - try (CloseableHttpResponse response = executeHttpRequest(httpClient, url)) { - httpStatusCode = response.getStatusLine().getStatusCode(); - LOG.debug("Response HTTP Status code: {}", httpStatusCode); - httpResponseBody = new HttpResponse(response).getBody(); - } + try (CloseableHttpClient httpClient = createHttpClient(configURL); + CloseableHttpResponse response = executeHttpRequest(httpClient, new URL(configURL))) { + httpStatusCode = response.getStatusLine().getStatusCode(); + LOG.debug("Response HTTP Status code: {}", httpStatusCode); + httpResponseBody = new HttpResponse(response).getBody(); RetryableErrorHandling errorHandlingStrategy = httpErrorHandler.getErrorHandlingStrategy(httpStatusCode); boolean shouldRetry = errorHandlingStrategy.shouldRetry(); diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java index dd2f2d35..4a04dcca 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPSink.java @@ -38,6 +38,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -79,8 +80,8 @@ public void prepareRun(BatchSinkContext context) { if (inputSchema == null) { fields = Collections.emptyList(); } else { - assert inputSchema.getFields() != null; - fields = inputSchema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList()); + fields = Objects.requireNonNull(Objects.requireNonNull(inputSchema).getFields()).stream() + .map(Schema.Field::getName).collect(Collectors.toList()); } lineageRecorder.recordWrite("Write", String.format("Wrote to HTTP '%s'", config.getUrl()), fields); diff --git a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java index 5f735e7c..044f4026 100644 --- a/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java +++ b/src/main/java/io/cdap/plugin/http/source/batch/HttpBatchSourceConfig.java @@ -119,10 +119,13 @@ public void validateBasicAuthResponse(FailureCollector collector, HttpClient htt collector.addFailure(errorMessage, "Please ensure that correct credentials are provided."); } } + } catch (HttpHostConnectException e) { + String errorMessage = "Error occurred during credential validation : " + e.getMessage(); + collector.addFailure(errorMessage, "Please ensure that correct credentials are provided."); } catch (IOException e) { String errorMessage = "Unable to process the response and validate credentials"; throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, e); } } diff --git a/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java b/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java index 2afe5bfd..9514dba3 100644 --- a/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java +++ b/src/main/java/io/cdap/plugin/http/source/common/RawStringPerLine.java @@ -66,7 +66,7 @@ public boolean hasNext() { } catch (IOException e) { // we need to catch this, since hasNext() does not have "throws" in parent String errorMessage = "Unable to read line from http page buffer"; throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, new IOException(errorMessage)); + errorMessage, e.getMessage(), ErrorType.UNKNOWN, true, e); } } From 785a9cda1869f03255ed0975a7c54b947c4c125a Mon Sep 17 00:00:00 2001 From: Amit Kumar Singh Date: Fri, 6 Dec 2024 07:12:50 +0000 Subject: [PATCH 7/7] remove unused imprts --- .../java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java index 03aedd78..45cd88d4 100644 --- a/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java +++ b/src/main/java/io/cdap/plugin/http/sink/batch/HTTPOutputFormat.java @@ -19,9 +19,6 @@ import com.google.gson.Gson; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.api.exception.ErrorCategory; -import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.cdap.api.exception.ErrorUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.OutputCommitter;