From d7b8b464b071cda6f003cd760ba600cab7e0355f Mon Sep 17 00:00:00 2001 From: Saurabh Rusia Date: Wed, 7 Oct 2026 13:39:43 +0530 Subject: [PATCH] DLPX-99548 Hop 2.x dropped HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE, breaking Teradata batch inserts on nullable TIMESTAMP columns PR URL: https://www.github.com/delphix/hop/pull/18 --- .../main/java/org/apache/hop/core/Const.java | 12 +++++++ .../hop/core/row/value/ValueMetaBase.java | 14 ++++++--- ...alueMetaBaseSetPreparedStmntValueTest.java | 31 +++++++++++++++++++ .../junit/rules/RestoreHopEnvironment.java | 1 + .../rules/RestoreHopEnvironmentExtension.java | 1 + 5 files changed, 55 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/hop/core/Const.java b/core/src/main/java/org/apache/hop/core/Const.java index 143f3124298..a8d3dd54fb8 100644 --- a/core/src/main/java/org/apache/hop/core/Const.java +++ b/core/src/main/java/org/apache/hop/core/Const.java @@ -466,6 +466,18 @@ public String getMessage() { public static final String HOP_LENIENT_STRING_TO_NUMBER_CONVERSION = "HOP_LENIENT_STRING_TO_NUMBER_CONVERSION"; + /* + * Restore the flag HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE dropped from upstream ValueMetaBase during + * the 1.2.0 -> 2.x migration. See DLPX-87054. + */ + /** Systemwide flag to ignore timezone while writing date/timestamp value to the database. */ + @Variable( + value = "N", + description = + "System wide flag to ignore timezone while writing date/timestamp value to the database. Needed for Teradata, whose JDBC driver assigns a different SQL type to timestamp parameters bound via the Calendar-aware PreparedStatement overload, which breaks batch inserts mixing null and non-null rows for the same column.") + public static final String HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE = + "HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE"; + /** * You can use this variable to speed up hostname lookup. Hostname lookup is performed by Hop so * that it is capable of logging the server on which a workflow or pipeline is executed. diff --git a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java index 83b18073cdc..9f585d422aa 100644 --- a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java +++ b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java @@ -221,6 +221,7 @@ public class ValueMetaBase implements IValueMeta { protected boolean dateFormatLenient; protected boolean lenientStringToNumber; protected boolean emptyStringAndNullAreDifferent; + protected boolean ignoreTimezone; @JsonIgnore protected SimpleDateFormat dateFormat; @JsonIgnore protected boolean dateFormatChanged; @@ -335,6 +336,9 @@ public ValueMetaBase( this.emptyStringAndNullAreDifferent = convertStringToBoolean( Const.NVL(System.getProperty(Const.HOP_EMPTY_STRING_DIFFERS_FROM_NULL, "N"), "N")); + this.ignoreTimezone = + convertStringToBoolean( + Const.NVL(System.getProperty(Const.HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE, "N"), "N")); this.comparator = comparator; determineSingleByteEncoding(); @@ -481,6 +485,7 @@ && getPrecision() == that.getPrecision() && dateFormatLenient == that.dateFormatLenient && lenientStringToNumber == that.lenientStringToNumber && emptyStringAndNullAreDifferent == that.emptyStringAndNullAreDifferent + && ignoreTimezone == that.ignoreTimezone && Objects.equals(name, that.name) && Objects.equals(conversionMask, that.conversionMask) && Objects.equals(stringEncoding, that.stringEncoding) @@ -522,7 +527,8 @@ public int hashCode() { dateFormatTimeZone, dateFormatLenient, lenientStringToNumber, - emptyStringAndNullAreDifferent); + emptyStringAndNullAreDifferent, + ignoreTimezone); result = 31 * result + Arrays.hashCode(index); return result; } @@ -5851,7 +5857,7 @@ public void setPreparedStatementValue( // is not yet implemented preparedStatement.setDate(index, ddate); } else { - if (this.getDateFormatTimeZone() == null) { + if (ignoreTimezone || this.getDateFormatTimeZone() == null) { preparedStatement.setDate(index, ddate); } else { preparedStatement.setDate( @@ -5868,7 +5874,7 @@ public void setPreparedStatementValue( // is not yet implemented preparedStatement.setTimestamp(index, timestamp); } else { - if (this.getDateFormatTimeZone() == null) { + if (ignoreTimezone || this.getDateFormatTimeZone() == null) { preparedStatement.setTimestamp(index, timestamp); } else { preparedStatement.setTimestamp( @@ -5884,7 +5890,7 @@ public void setPreparedStatementValue( // is not yet implemented preparedStatement.setTimestamp(index, sdate); } else { - if (this.getDateFormatTimeZone() == null) { + if (ignoreTimezone || this.getDateFormatTimeZone() == null) { preparedStatement.setTimestamp(index, sdate); } else { preparedStatement.setTimestamp( diff --git a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseSetPreparedStmntValueTest.java b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseSetPreparedStmntValueTest.java index 37bd8667011..069feea21c2 100644 --- a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseSetPreparedStmntValueTest.java +++ b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaBaseSetPreparedStmntValueTest.java @@ -20,6 +20,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -30,6 +31,7 @@ import java.util.Arrays; import java.util.Calendar; import java.util.Date; +import org.apache.hop.core.Const; import org.apache.hop.core.database.BaseDatabaseMeta; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.database.IDatabase; @@ -37,6 +39,7 @@ import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowMeta; import org.apache.hop.junit.rules.RestoreHopEnvironment; +import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; @@ -61,6 +64,11 @@ public void setUp() { ts = new Timestamp(System.currentTimeMillis()); } + @After + public void tearDown() { + System.clearProperty(Const.HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE); + } + @Test public void testXMLParsingWithNoDataFormatLocale() throws IOException { IValueMeta r1 = new ValueMetaString("value"); @@ -99,4 +107,27 @@ public void testConvertedTimestampRegular() throws Exception { verify(ps).setTimestamp(eq(1), any(Timestamp.class), any(Calendar.class)); } + + @Test + public void testTimestampWithIgnoreTimezoneSkipsCalendarOverload() throws Exception { + System.setProperty(Const.HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE, "Y"); + + ValueMetaBase valueMeta = new ValueMetaDate(""); + valueMeta.setPreparedStatementValue(dbMeta, ps, 1, ts); + + verify(ps).setTimestamp(eq(1), any(Timestamp.class)); + verify(ps, never()).setTimestamp(eq(1), any(Timestamp.class), any(Calendar.class)); + } + + @Test + public void testDateWithIgnoreTimezoneSkipsCalendarOverload() throws Exception { + System.setProperty(Const.HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE, "Y"); + + ValueMetaBase valueMeta = new ValueMetaDate(""); + valueMeta.setPrecision(1); + valueMeta.setPreparedStatementValue(dbMeta, ps, 1, date); + + verify(ps).setDate(eq(1), any(java.sql.Date.class)); + verify(ps, never()).setDate(eq(1), any(java.sql.Date.class), any(Calendar.class)); + } } diff --git a/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironment.java b/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironment.java index c9c40be3466..25a79e9ee6c 100644 --- a/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironment.java +++ b/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironment.java @@ -119,6 +119,7 @@ protected void before() throws Throwable { System.clearProperty(Const.HOP_DEFAULT_DATE_FORMAT); System.clearProperty(Const.HOP_DEFAULT_TIMESTAMP_FORMAT); System.clearProperty(Const.HOP_EMPTY_STRING_DIFFERS_FROM_NULL); + System.clearProperty(Const.HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE); defaultInit(); } diff --git a/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironmentExtension.java b/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironmentExtension.java index 8dee942c4a9..4dd10bcd490 100644 --- a/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironmentExtension.java +++ b/core/src/test/java/org/apache/hop/junit/rules/RestoreHopEnvironmentExtension.java @@ -126,6 +126,7 @@ public void beforeAll(ExtensionContext context) throws Exception { System.clearProperty(Const.HOP_DEFAULT_DATE_FORMAT); System.clearProperty(Const.HOP_DEFAULT_TIMESTAMP_FORMAT); System.clearProperty(Const.HOP_EMPTY_STRING_DIFFERS_FROM_NULL); + System.clearProperty(Const.HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE); try { defaultInit();