diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json index 943e2d24e..586a8993b 100644 --- a/amazon-redshift-plugin/widgets/Redshift-batchsource.json +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ @@ -228,7 +259,7 @@ "name": "connection" } ] - }, + } ], "jump-config": { "datasets": [ diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json index 3a2af8e01..f392e3a78 100644 --- a/amazon-redshift-plugin/widgets/Redshift-connector.json +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -69,6 +69,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-action.json b/aurora-mysql-plugin/widgets/AuroraMysql-action.json index efc5f98ff..bd2bac558 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-action.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-action.json @@ -90,6 +90,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json b/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json index a435e4e4f..6663be7ce 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json @@ -116,6 +116,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json index 50b435645..bd2bb88a9 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json @@ -135,6 +135,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json b/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json index cc33cf0a1..64da4f1bc 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json @@ -105,6 +105,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json index 1f3bca862..e012f65eb 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json @@ -79,6 +79,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json index 53979d6d4..bfc83bd4e 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json @@ -121,6 +121,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json index 14b00b974..fc2503c67 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json @@ -124,6 +124,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json index 3fdb1a14b..8b328160d 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json index 66d6ebb85..0dd6f8f41 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json @@ -112,6 +112,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json index 89a7d7736..3a3277ed8 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json @@ -176,6 +176,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json index 4ac7747f4..a90154670 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json @@ -175,6 +175,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json index b5c2c9993..1cebc7850 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json index eab240679..e14646154 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json @@ -112,6 +112,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json index 2fda594dd..8d6578413 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json @@ -192,6 +192,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json index 96ea97ac2..ea449120d 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json @@ -175,6 +175,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json index 9824f91bd..36013ac40 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 67dc8e82e..1d49676be 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -41,6 +41,12 @@ guava + + + dev.failsafe + failsafe + + io.cdap.cdap diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index c5320e25e..f28d2c450 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,12 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; + public static final int DEFAULT_MAX_RETRY_COUNT = 5; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; @Name(JDBC_PLUGIN_NAME) @@ -72,6 +78,38 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC @Macro public String connectionArguments; + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 5 seconds.") + @Nullable + @Macro + private Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 80 seconds.") + @Nullable + @Macro + private Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 5.") + @Nullable + @Macro + private Integer maxRetryCount; + + + public Integer getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + public Integer getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + public Integer getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } + + public ConnectionConfig() { } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java new file mode 100644 index 000000000..e16531e33 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package io.cdap.plugin.db; + +import java.sql.SQLTransientException; +import java.util.HashSet; +import java.util.Set; + + +/** + * Checks whether the given exception or one of its causes is a known retryable SQLException. + */ +public class RetryExceptions { + public static boolean isRetryable(Throwable t) { + Set seen = new HashSet<>(); + while (t != null && seen.add(t)) { + if (t instanceof SQLTransientException) { + return true; + } + t = t.getCause(); + } + return false; + } +} + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java index 0eaac3148..bef93a0de 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java @@ -16,12 +16,15 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; import java.sql.SQLException; @@ -33,7 +36,6 @@ public abstract class AbstractDBAction extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final QueryConfig config; private final Boolean enableAutoCommit; - public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { this.config = config; this.enableAutoCommit = enableAutoCommit; @@ -44,7 +46,8 @@ public void run(ActionContext context) throws Exception { Class driverClass = context.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); try { - executeQuery.run(); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(()-> executeQuery.run()); } catch (Exception e) { if (e instanceof SQLException) { DBErrorDetailsProvider dbe = new DBErrorDetailsProvider(); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index 5e22abf85..540d2e7f0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -16,6 +16,7 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.StageConfigurer; @@ -25,6 +26,7 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Connection; import java.sql.Driver; @@ -92,50 +94,60 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) */ private void processArguments(Class driverClass, FailureCollector failureCollector, SettableArguments settableArguments) - throws SQLException, IllegalAccessException, InstantiationException { + throws SQLException, IllegalAccessException, InstantiationException { DriverCleanup driverCleanup; - driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), - config.getJdbcPluginName()); + config.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); try { - Connection connection = DriverManager - .getConnection(config.getConnectionString(), connectionProperties); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(config.getQuery()); - boolean hasRecord = resultSet.next(); - if (!hasRecord) { - failureCollector.addFailure("No record found.", - "The argument selection conditions must match only one record."); - return; - } - if (settableArguments != null) { - setArguments(resultSet, failureCollector, settableArguments); - } - if (resultSet.next()) { - failureCollector - .addFailure("More than one records found.", - "The argument selection conditions must match only one record."); - } + executeWithRetry(failureCollector, settableArguments, connectionProperties); } finally { driverCleanup.destroy(); } } + private void executeWithRetry(FailureCollector failureCollector, SettableArguments settableArguments, + Properties connectionProperties) { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager + .getConnection(config.getConnectionString(), connectionProperties)) { + ResultSet resultSet; + try (Statement statement = connection.createStatement()) { + resultSet = statement.executeQuery(config.getQuery()); + } + boolean hasRecord = resultSet.next(); + if (!hasRecord) { + failureCollector.addFailure("No record found.", + "The argument selection conditions must match only one record."); + return; + } + if (settableArguments != null) { + setArguments(resultSet, settableArguments); + } + if (resultSet.next()) { + failureCollector + .addFailure("More than one records found.", + "The argument selection conditions must match only one record."); + } + } + }); + } + /** * Converts column from jdbc results set into pipeline arguments * * @param resultSet - result set from db {@link ResultSet} - * @param failureCollector - context failure collector @{link FailureCollector} * @param arguments - context argument setter {@link SettableArguments} - * @throws SQLException - raises {@link SQLException} when configuration is not valid */ - private void setArguments(ResultSet resultSet, FailureCollector failureCollector, - SettableArguments arguments) throws SQLException { - String[] columns = config.getArgumentsColumns().split(","); - for (String column : columns) { - arguments.set(column, resultSet.getString(column)); - } + private void setArguments(ResultSet resultSet, SettableArguments arguments) { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + String[] columns = config.getArgumentsColumns().split(","); + for (String column : columns) { + arguments.set(column, resultSet.getString(column)); + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java index e4b91adbd..a9ce100cf 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java @@ -16,11 +16,13 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchActionContext; import io.cdap.cdap.etl.api.batch.PostAction; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; @@ -51,7 +53,8 @@ public void run(BatchActionContext batchContext) throws Exception { Class driverClass = batchContext.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - executeQuery.run(); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> executeQuery.run()); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index e2ccfc57e..1d99974e0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -16,8 +16,10 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Connection; import java.sql.Driver; @@ -48,6 +50,7 @@ public DBRun(QueryConfig config, Class driverClass, Boolean en * to use and which connection string to use come from the plugin configuration. */ public void run() throws SQLException, InstantiationException, IllegalAccessException { + DriverCleanup driverCleanup = null; try { driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), @@ -55,18 +58,21 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(config.getConnectionString(), connectionProperties)) { - executeInitQueries(connection, config.getInitQueries()); - if (!enableAutoCommit) { - connection.setAutoCommit(false); - } - try (Statement statement = connection.createStatement()) { - statement.execute(config.query); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(config.getConnectionString(), connectionProperties)) { + executeInitQueries(connection, config.getInitQueries()); if (!enableAutoCommit) { - connection.commit(); + connection.setAutoCommit(false); + } + try (Statement statement = connection.createStatement()) { + statement.execute(config.query); + if (!enableAutoCommit) { + connection.commit(); + } } } - } + }); } finally { if (driverCleanup != null) { driverCleanup.destroy(); @@ -74,11 +80,16 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce } } - private void executeInitQueries(Connection connection, List initQueries) throws SQLException { - for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } - } + private void executeInitQueries(Connection connection, List initQueries) { + + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())) + .run(() -> { + for (String query : initQueries) { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java index 5b92a85f7..3d9ed16ff 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java @@ -155,4 +155,16 @@ public Operation getOperationName() { public String getRelationTableKey() { return relationTableKey; } + + public Integer getInitialRetryDuration() { + return getConnection().getInitialRetryDuration(); + } + + public Integer getMaxRetryDuration() { + return getConnection().getMaxRetryDuration(); + } + + public Integer getMaxRetryCount() { + return getConnection().getMaxRetryCount(); + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java index 41c577397..f15939ab7 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java @@ -268,4 +268,16 @@ public Integer getFetchSize() { return fetchSize; } + public Integer getInitialRetryDuration() { + return getConnection().getInitialRetryDuration(); + } + + public Integer getMaxRetryDuration() { + return getConnection().getMaxRetryDuration(); + } + + public Integer getMaxRetryCount() { + return getConnection().getMaxRetryCount(); + } + } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java index 55cfe363f..e1fde69a1 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java @@ -50,4 +50,10 @@ public interface DatabaseConnectionConfig { */ String getPassword(); + Integer getInitialRetryDuration(); + + Integer getMaxRetryDuration(); + + Integer getMaxRetryCount(); + } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java index 4bee056f8..3423440d0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java @@ -36,6 +36,12 @@ * */ public abstract class AbstractDBConnectorConfig extends PluginConfig implements DBConnectorProperties { + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; + public static final int DEFAULT_MAX_RETRY_COUNT = 5; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; @Name(ConnectionConfig.JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + @@ -63,6 +69,26 @@ public abstract class AbstractDBConnectorConfig extends PluginConfig implements @Macro protected String connectionArguments; + + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 5 seconds.") + @Nullable + @Macro + private Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 80 seconds.") + @Nullable + @Macro + private Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 5.") + @Nullable + @Macro + private Integer maxRetryCount; + + @Nullable @Override public String getUser() { @@ -74,6 +100,18 @@ public String getUser() { public String getPassword() { return password; } + + public Integer getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + public Integer getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + public Integer getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } @Override public Properties getConnectionArgumentsProperties() { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index 8a9b7b6e4..f49cbae19 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db.connector; import com.google.common.collect.Maps; +import dev.failsafe.Failsafe; import io.cdap.cdap.api.data.batch.InputFormatProvider; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchConnector; @@ -33,6 +34,7 @@ import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.source.DataDrivenETLDBInputFormat; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -172,13 +174,16 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, protected Schema loadTableSchema(Connection connection, String query, @Nullable Integer timeoutSec, String sessionID) throws SQLException { - Statement statement = connection.createStatement(); - statement.setMaxRows(1); - if (timeoutSec != null) { - statement.setQueryTimeout(timeoutSec); - } - ResultSet resultSet = statement.executeQuery(query); - return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).get(() -> { + Statement statement = connection.createStatement(); + statement.setMaxRows(1); + if (timeoutSec != null) { + statement.setQueryTimeout(timeoutSec); + } + ResultSet resultSet = statement.executeQuery(query); + return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + }); } protected void setConnectionProperties(Map properties, ConnectorSpecRequest request) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 0bb4bf123..f1d5e6d4c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -18,6 +18,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -53,6 +55,7 @@ import io.cdap.plugin.db.config.DatabaseSinkConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.lib.db.DBConfiguration; @@ -302,6 +305,8 @@ private Schema inferSchema(Class driverClass) { dbSinkConfig.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { try (Connection connection = DriverManager.getConnection(dbSinkConfig.getConnectionString(), connectionProperties)) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); @@ -330,6 +335,7 @@ private Schema inferSchema(Class driverClass) { e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } + }); } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); } @@ -369,18 +375,20 @@ private void setResultSetMetadata() throws Exception { Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (Statement statement = connection.createStatement(); - // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata - // that can be used to construct DBRecord objects to sink to the database table. - ResultSet rs = statement.executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, fullyQualifiedTableName)) - ) { - columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (Statement statement = connection.createStatement(); + // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata + // that can be used to construct DBRecord objects to sink to the database table. + ResultSet rs = statement.executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", + dbColumns, fullyQualifiedTableName)) + ) { + columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); + } } - } - + }); this.columnTypes = Collections.unmodifiableList(columnTypes); } @@ -438,26 +446,31 @@ private void validateSchema(FailureCollector collector, Class Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { - if (!tables.next()) { - collector.addFailure( - String.format("Table '%s' does not exist.", tableName), - String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + - "points to a valid database.", fullyQualifiedTableName, connectionString)) - .withConfigProperty(DBSinkConfig.TABLE_NAME); - return; + try { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { + if (!tables.next()) { + collector.addFailure( + String.format("Table '%s' does not exist.", tableName), + String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + + "points to a valid database.", fullyQualifiedTableName, connectionString)) + .withConfigProperty(DBSinkConfig.TABLE_NAME); + return; + } + } + setColumnsInfo(inputSchema.getFields()); + try (PreparedStatement pStmt = connection.prepareStatement(String.format("SELECT %s FROM %s WHERE 1 = 0", + dbColumns, + fullyQualifiedTableName)); + ResultSet rs = pStmt.executeQuery()) { + getFieldsValidator().validateFields(inputSchema, rs, collector); + } } - } - setColumnsInfo(inputSchema.getFields()); - try (PreparedStatement pStmt = connection.prepareStatement(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, - fullyQualifiedTableName)); - ResultSet rs = pStmt.executeQuery()) { - getFieldsValidator().validateFields(inputSchema, rs, collector); - } - } catch (SQLException e) { + }); + } catch (FailsafeException e) { LOG.error("Exception while trying to validate schema of database table {} for connection {}.", fullyQualifiedTableName, connectionString, e); collector.addFailure( @@ -486,9 +499,12 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 54d1e2ab6..470e3c0c3 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -18,6 +18,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import dev.failsafe.Failsafe; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -52,13 +53,13 @@ import io.cdap.plugin.db.config.DatabaseSourceConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.io.IOException; import java.sql.Connection; import java.sql.Driver; @@ -137,7 +138,6 @@ public Schema getSchema(Class driverClass) throws IllegalAcces SQLException, InstantiationException { DriverCleanup driverCleanup; try { - driverCleanup = loadPluginClassAndGetDriver(driverClass); try { return getSchema(); @@ -168,13 +168,15 @@ public Schema getSchema() throws SQLException { } private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { - Statement statement = connection.createStatement(); - statement.setMaxRows(1); - if (query.contains("$CONDITIONS")) { - query = removeConditionsClause(query); - } - ResultSet resultSet = statement.executeQuery(query); - return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> { + Statement statement = connection.createStatement(); + statement.setMaxRows(1); + String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; + ResultSet resultSet = statement.executeQuery(finalQuery); + return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + }); } @VisibleForTesting @@ -191,41 +193,52 @@ private Schema loadSchemaFromDB(Class driverClass) String connectionString = sourceConfig.getConnectionString(); DriverCleanup driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, connectionString, sourceConfig.getJdbcPluginName()); - Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, sourceConfig.getInitQueries()); - return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); - - } catch (SQLException e) { - // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), - e.getSQLState(), e.getErrorCode()); - String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + - "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); - } - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, - e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), - e.getSQLState(), e.getErrorCode())); - } finally { - driverCleanup.destroy(); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, sourceConfig.getInitQueries()); + return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); + } + catch (SQLException e) { + // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc + // driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", + e.getMessage(), + e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error occurred while trying to" + + " get schema from database." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), + e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessage, + externalDocumentationLink); + } + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); + } finally { + driverCleanup.destroy(); + } + }); } - } private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())).run(() -> { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + }); } } @@ -266,7 +279,9 @@ private Connection getConnection() throws SQLException { String connectionString = createConnectionString(); Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - return DriverManager.getConnection(connectionString, connectionProperties); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> DriverManager.getConnection(connectionString, connectionProperties)); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java new file mode 100644 index 000000000..1aca23ec7 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package io.cdap.plugin.util; + +import dev.failsafe.RetryPolicy; +import io.cdap.cdap.api.Config; +import io.cdap.plugin.db.RetryExceptions; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLTransientConnectionException; +import java.time.Duration; + +/** + * Utility class for creating standardized {@link dev.failsafe.RetryPolicy} configurations + * to handle transient SQL exceptions using the Failsafe library. + */ +public class RetryPolicyUtil extends Config { + public static final Logger LOG = LoggerFactory.getLogger(RetryPolicyUtil.class); + + /** + * Create a RetryPolicy using custom config values. + */ + public static RetryPolicy createConnectionRetryPolicy(Integer initialRetryDuration, + Integer maxRetryDuration, Integer maxRetryCount) { + return RetryPolicy.builder() + .handleIf((failure) -> RetryExceptions.isRetryable(failure)) + .withBackoff(Duration.ofSeconds(initialRetryDuration), Duration.ofSeconds(maxRetryDuration)) + .withMaxRetries(maxRetryCount) + .onRetry(e -> LOG.debug("Retrying... Attempt {}", + e.getAttemptCount())) + .onFailedAttempt(e -> LOG.debug("Failed Attempt : {}", e.getLastException())) + .onFailure(e -> LOG.debug("Failed after retries." + + " Reason: {}", + e.getException() != null ? e.getException().getMessage() : "Unknown error")) + .build(); + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java new file mode 100644 index 000000000..e85e50e8b --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java @@ -0,0 +1,81 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db; + +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import io.cdap.plugin.util.RetryPolicyUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.SQLSyntaxErrorException; +import java.sql.SQLTransientConnectionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RetryPolicyUtilTest { + + private AbstractDBConnectorConfig mockConfig; + + @Before + public void setup() { + mockConfig = mock(AbstractDBConnectorConfig.class); + when(mockConfig.getInitialRetryDuration()).thenReturn(5); + when(mockConfig.getMaxRetryDuration()).thenReturn(10); + when(mockConfig.getMaxRetryCount()).thenReturn(2); + } + + @Test + public void testCreateConnectionRetryPolicy_Retryable() { + RetryPolicy retryPolicy = RetryPolicyUtil + .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); + + AtomicInteger attemptCounter = new AtomicInteger(); + + FailsafeException ex = Assert.assertThrows(FailsafeException.class, () -> Failsafe.with(retryPolicy).run(() -> { + attemptCounter.incrementAndGet(); + throw new SQLTransientConnectionException("Temporary issue"); + })); + + Assert.assertTrue(ex.getCause() instanceof SQLTransientConnectionException); + Assert.assertEquals(3, attemptCounter.get()); + } + + @Test + public void testCreateConnectionRetryPolicy_NonRetryable() { + RetryPolicy retryPolicy = RetryPolicyUtil + .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); + + AtomicInteger attemptCounter = new AtomicInteger(); + + FailsafeException ex = Assert.assertThrows(FailsafeException.class, () -> + Failsafe.with(retryPolicy).run(() -> { + attemptCounter.incrementAndGet(); + throw new SQLSyntaxErrorException("Bad SQL syntax"); + })); + Assert.assertTrue(ex.getCause() instanceof SQLSyntaxErrorException); + Assert.assertEquals(1, attemptCounter.get()); + } +} + diff --git a/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java b/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java index fa9e371da..23abe08a9 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java @@ -216,10 +216,14 @@ public void testValidateFieldsWithNullable() throws Exception { public void validateFieldCompatible(Schema.Type fieldType, Schema.LogicalType fieldLogicalType, int sqlType, boolean isCompatible, int precision, boolean isSigned) { String errorMessage = String.format("Expected type '%s' is %s with sql type '%d'", - fieldType, - isCompatible ? "compatible" : "not compatible", - sqlType); - Assert.assertEquals(errorMessage, isCompatible, VALIDATOR.isFieldCompatible(fieldType, fieldLogicalType, sqlType, - precision, isSigned)); + fieldType, + isCompatible ? "compatible" : "not compatible", + sqlType); + try { + boolean actualCompatible = VALIDATOR.isFieldCompatible(fieldType, fieldLogicalType, sqlType, precision, isSigned); + Assert.assertEquals(errorMessage, isCompatible, actualCompatible); + } catch (Exception e) { + throw new AssertionError("Unexpected exception during compatibility check: " + e.getMessage(), e); + } } } diff --git a/db2-plugin/widgets/Db2-action.json b/db2-plugin/widgets/Db2-action.json index 3e9159c0d..c144f8605 100644 --- a/db2-plugin/widgets/Db2-action.json +++ b/db2-plugin/widgets/Db2-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/db2-plugin/widgets/Db2-batchsink.json b/db2-plugin/widgets/Db2-batchsink.json index 5345f03d5..d7c97199b 100644 --- a/db2-plugin/widgets/Db2-batchsink.json +++ b/db2-plugin/widgets/Db2-batchsink.json @@ -100,6 +100,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/db2-plugin/widgets/Db2-batchsource.json b/db2-plugin/widgets/Db2-batchsource.json index 1c221606d..df8e93bdf 100644 --- a/db2-plugin/widgets/Db2-batchsource.json +++ b/db2-plugin/widgets/Db2-batchsource.json @@ -119,6 +119,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/db2-plugin/widgets/Db2-postaction.json b/db2-plugin/widgets/Db2-postaction.json index cd75dec04..1a13f10a9 100644 --- a/db2-plugin/widgets/Db2-postaction.json +++ b/db2-plugin/widgets/Db2-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-database-plugin/widgets/Database-action.json b/generic-database-plugin/widgets/Database-action.json index c849c4cba..3fde94085 100644 --- a/generic-database-plugin/widgets/Database-action.json +++ b/generic-database-plugin/widgets/Database-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-database-plugin/widgets/Database-batchsink.json b/generic-database-plugin/widgets/Database-batchsink.json index 90332c5c5..9b95ef80a 100644 --- a/generic-database-plugin/widgets/Database-batchsink.json +++ b/generic-database-plugin/widgets/Database-batchsink.json @@ -115,6 +115,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/generic-database-plugin/widgets/Database-batchsource.json b/generic-database-plugin/widgets/Database-batchsource.json index 579b87bd9..bf363027d 100644 --- a/generic-database-plugin/widgets/Database-batchsource.json +++ b/generic-database-plugin/widgets/Database-batchsource.json @@ -134,6 +134,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/generic-database-plugin/widgets/Database-postaction.json b/generic-database-plugin/widgets/Database-postaction.json index abf02ef22..2d07b6055 100644 --- a/generic-database-plugin/widgets/Database-postaction.json +++ b/generic-database-plugin/widgets/Database-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json b/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json index 423792391..49d3ddb91 100644 --- a/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json +++ b/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json @@ -93,6 +93,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mariadb-plugin/widgets/Mariadb-action.json b/mariadb-plugin/widgets/Mariadb-action.json index bb78abb27..7588b6016 100644 --- a/mariadb-plugin/widgets/Mariadb-action.json +++ b/mariadb-plugin/widgets/Mariadb-action.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/memsql-plugin/widgets/Memsql-action.json b/memsql-plugin/widgets/Memsql-action.json index 61cbb1e47..67888e80f 100644 --- a/memsql-plugin/widgets/Memsql-action.json +++ b/memsql-plugin/widgets/Memsql-action.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/memsql-plugin/widgets/Memsql-batchsink.json b/memsql-plugin/widgets/Memsql-batchsink.json index 98c2c1f8e..7f060289c 100644 --- a/memsql-plugin/widgets/Memsql-batchsink.json +++ b/memsql-plugin/widgets/Memsql-batchsink.json @@ -170,6 +170,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/memsql-plugin/widgets/Memsql-batchsource.json b/memsql-plugin/widgets/Memsql-batchsource.json index ef5d17b39..81644861b 100644 --- a/memsql-plugin/widgets/Memsql-batchsource.json +++ b/memsql-plugin/widgets/Memsql-batchsource.json @@ -205,6 +205,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/memsql-plugin/widgets/Memsql-postaction.json b/memsql-plugin/widgets/Memsql-postaction.json index 72e67abd9..661b946c8 100644 --- a/memsql-plugin/widgets/Memsql-postaction.json +++ b/memsql-plugin/widgets/Memsql-postaction.json @@ -172,6 +172,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index c326cd81d..198ac1fd4 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/mssql-plugin/widgets/SqlServer-action.json b/mssql-plugin/widgets/SqlServer-action.json index 303944d71..511793545 100644 --- a/mssql-plugin/widgets/SqlServer-action.json +++ b/mssql-plugin/widgets/SqlServer-action.json @@ -204,6 +204,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index fb20cad9d..3f3ad5887 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -257,6 +257,37 @@ "name": "currentLanguage" } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index b3494e485..d35db27ae 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -276,6 +276,37 @@ "name": "currentLanguage" } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/mssql-plugin/widgets/SqlServer-postaction.json b/mssql-plugin/widgets/SqlServer-postaction.json index 5cec14b89..5f191000e 100644 --- a/mssql-plugin/widgets/SqlServer-postaction.json +++ b/mssql-plugin/widgets/SqlServer-postaction.json @@ -219,6 +219,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index f60f5526f..52982aa32 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -78,6 +78,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/mysql-plugin/widgets/Mysql-action.json b/mysql-plugin/widgets/Mysql-action.json index ae5d0b555..45cbed4ba 100644 --- a/mysql-plugin/widgets/Mysql-action.json +++ b/mysql-plugin/widgets/Mysql-action.json @@ -161,6 +161,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index 58596aae2..ca23f71c8 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -217,6 +217,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 506e837f7..ce6cd047e 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -252,6 +252,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/mysql-plugin/widgets/Mysql-postaction.json b/mysql-plugin/widgets/Mysql-postaction.json index e34a40928..498f6f064 100644 --- a/mysql-plugin/widgets/Mysql-postaction.json +++ b/mysql-plugin/widgets/Mysql-postaction.json @@ -176,6 +176,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/netezza-plugin/widgets/Netezza-action.json b/netezza-plugin/widgets/Netezza-action.json index de523e860..e84555c95 100644 --- a/netezza-plugin/widgets/Netezza-action.json +++ b/netezza-plugin/widgets/Netezza-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/netezza-plugin/widgets/Netezza-batchsink.json b/netezza-plugin/widgets/Netezza-batchsink.json index c8634a58b..73884e782 100644 --- a/netezza-plugin/widgets/Netezza-batchsink.json +++ b/netezza-plugin/widgets/Netezza-batchsink.json @@ -95,6 +95,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/netezza-plugin/widgets/Netezza-batchsource.json b/netezza-plugin/widgets/Netezza-batchsource.json index c1e0f26c3..196546791 100644 --- a/netezza-plugin/widgets/Netezza-batchsource.json +++ b/netezza-plugin/widgets/Netezza-batchsource.json @@ -110,6 +110,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/netezza-plugin/widgets/Netezza-postaction.json b/netezza-plugin/widgets/Netezza-postaction.json index 85ea43ad9..5de3780d3 100644 --- a/netezza-plugin/widgets/Netezza-postaction.json +++ b/netezza-plugin/widgets/Netezza-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/oracle-plugin/widgets/Oracle-action.json b/oracle-plugin/widgets/Oracle-action.json index 815aa4b0f..15d4f81c3 100644 --- a/oracle-plugin/widgets/Oracle-action.json +++ b/oracle-plugin/widgets/Oracle-action.json @@ -107,6 +107,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 8d6168780..8d8fc79a2 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -220,6 +220,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 5eca20cc4..5adf07834 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -248,6 +248,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 628027caf..a72ecd203 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -148,6 +148,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters" : [ diff --git a/oracle-plugin/widgets/Oracle-postaction.json b/oracle-plugin/widgets/Oracle-postaction.json index 9a18077c4..3e913ae9b 100644 --- a/oracle-plugin/widgets/Oracle-postaction.json +++ b/oracle-plugin/widgets/Oracle-postaction.json @@ -122,6 +122,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/pom.xml b/pom.xml index 384a50374..c496330fd 100644 --- a/pom.xml +++ b/pom.xml @@ -298,6 +298,12 @@ ${junit.version} test + + + dev.failsafe + failsafe + 3.3.2 + org.hsqldb hsqldb diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 9a7a02e14..88a1714fa 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/postgresql-plugin/widgets/Postgres-action.json b/postgresql-plugin/widgets/Postgres-action.json index 351c023f1..afed87295 100644 --- a/postgresql-plugin/widgets/Postgres-action.json +++ b/postgresql-plugin/widgets/Postgres-action.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 14e6f8154..f58cb1995 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -169,6 +169,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 60de4725f..6d0656e39 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -172,6 +172,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/postgresql-plugin/widgets/Postgres-postaction.json b/postgresql-plugin/widgets/Postgres-postaction.json index 5a0daf595..6b3ebe1f3 100644 --- a/postgresql-plugin/widgets/Postgres-postaction.json +++ b/postgresql-plugin/widgets/Postgres-postaction.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/saphana-plugin/widgets/SapHana-action.json b/saphana-plugin/widgets/SapHana-action.json index 7e60ac35d..5fb59faab 100644 --- a/saphana-plugin/widgets/SapHana-action.json +++ b/saphana-plugin/widgets/SapHana-action.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/saphana-plugin/widgets/SapHana-batchsink.json b/saphana-plugin/widgets/SapHana-batchsink.json index a9d8c6343..56958358f 100644 --- a/saphana-plugin/widgets/SapHana-batchsink.json +++ b/saphana-plugin/widgets/SapHana-batchsink.json @@ -103,6 +103,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/saphana-plugin/widgets/SapHana-batchsource.json b/saphana-plugin/widgets/SapHana-batchsource.json index 9352b02f7..7df341a7e 100644 --- a/saphana-plugin/widgets/SapHana-batchsource.json +++ b/saphana-plugin/widgets/SapHana-batchsource.json @@ -127,6 +127,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/saphana-plugin/widgets/SapHana-postaction.json b/saphana-plugin/widgets/SapHana-postaction.json index ad2c8b938..e260ebf0e 100644 --- a/saphana-plugin/widgets/SapHana-postaction.json +++ b/saphana-plugin/widgets/SapHana-postaction.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/teradata-plugin/widgets/Teradata-action.json b/teradata-plugin/widgets/Teradata-action.json index 2ffba361c..0662ed778 100644 --- a/teradata-plugin/widgets/Teradata-action.json +++ b/teradata-plugin/widgets/Teradata-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/teradata-plugin/widgets/Teradata-batchsink.json b/teradata-plugin/widgets/Teradata-batchsink.json index f455991d4..861bbbaa2 100644 --- a/teradata-plugin/widgets/Teradata-batchsink.json +++ b/teradata-plugin/widgets/Teradata-batchsink.json @@ -95,6 +95,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/teradata-plugin/widgets/Teradata-batchsource.json b/teradata-plugin/widgets/Teradata-batchsource.json index 94f5314e5..2d10020a5 100644 --- a/teradata-plugin/widgets/Teradata-batchsource.json +++ b/teradata-plugin/widgets/Teradata-batchsource.json @@ -115,6 +115,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/teradata-plugin/widgets/Teradata-postaction.json b/teradata-plugin/widgets/Teradata-postaction.json index 35ead0013..deeccbb69 100644 --- a/teradata-plugin/widgets/Teradata-postaction.json +++ b/teradata-plugin/widgets/Teradata-postaction.json @@ -90,6 +90,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] }