DLPX-99529 Masking : Create new branch and upgrade to version 2.17 for forked repo Apache hop

.gitignore

@@ -59,3 +59,5 @@ integration-tests/mdi/files/excelwriter-mdi-test.xlsx
 integration-tests/spreadsheet/files/sample-file-append.xlsx
 
 assemblies/debug/audit
+
+local-snapshots-dir/

assemblies/static/src/main/resources/hop-gui.sh

@@ -22,6 +22,9 @@ ORIGINDIR=$(pwd)
 BASEDIR=$(dirname "$0")
 cd "${BASEDIR}" || exit 1
 
+# Compliance Engine customization to add MASKING HOME lib dir to classpath
+ADDITIONAL_CLASSPATH=":lib"
+
 # set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
 if [ -n "${HOP_JAVA_HOME}" ]; then
   _HOP_JAVA="${HOP_JAVA_HOME}/bin/java"
@@ -109,7 +112,7 @@ Darwin)
 esac
 CLASSPATH="lib/core/*:lib/beam/*:lib/swt/$os_path/$arch_path/*"
 
-"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}" org.apache.hop.ui.hopgui.HopGui "$@"
+"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}${ADDITIONAL_CLASSPATH}" org.apache.hop.ui.hopgui.HopGui "$@"
 EXITCODE=$?
 
 cd "${ORIGINDIR}" || exit 1

assemblies/static/src/main/resources/hop-run.sh

@@ -22,6 +22,9 @@ ORIGINDIR=$(pwd)
 BASEDIR=$(dirname "$0")
 cd "${BASEDIR}" || exit 1
 
+# Compliance Engine customization to add MASKING HOME lib dir to classpath
+ADDITIONAL_CLASSPATH=":lib:$MASKING_DATA/resources/jarfiles/db2jcc_license_cisuz.jar"
+
 # set java primary is HOP_JAVA_HOME fallback to JAVA_HOME or default java
 if [ -n "${HOP_JAVA_HOME}" ]; then
   _HOP_JAVA="${HOP_JAVA_HOME}/bin/java"
@@ -91,7 +94,7 @@ Darwin)
   ;;
 esac
 
-"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}" org.apache.hop.run.HopRun "$@"
+"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}${ADDITIONAL_CLASSPATH}" org.apache.hop.run.HopRun "$@"
 EXITCODE=$?
 
 cd "${ORIGINDIR}" || exit 1

core/src/main/java/org/apache/hop/core/database/Database.java

@@ -119,6 +119,8 @@ public class Database implements IVariables, ILoggingObject, AutoCloseable {
 
   private int rowlimit;
   private int commitsize;
+  private boolean delphixSpecial = false;
+  private int fetchSize = 0;
 
   private Connection connection;
 
@@ -655,6 +657,23 @@ public void cancelStatement(Statement statement) throws HopDatabaseException {
    * @param commsize The number of rows to wait before doing a commit on the connection.
    */
   public void setCommit(int commsize) {
+    /*
+     * We override this method to set fetch size in the case where it was called
+     * previously with a "magic" value. It would be much cleaner to just add a
+     * setFetchSize() method, but the way we build dms-core-gate, changes here
+     * aren't visible to link against when building our code.
+     */
+    if (commsize == Integer.MIN_VALUE + 42) {
+      delphixSpecial = true;
+      return;
+    }
+    if (delphixSpecial) {
+      fetchSize = commsize;
+      delphixSpecial = false;
+      return;
+    }
+    // End Delphix extra-special case.
+
     commitsize = commsize;
     String onOff = (commitsize <= 0 ? "on" : "off");
     try {
@@ -1119,7 +1138,8 @@ && getDatabaseMetaData().supportsBatchUpdates()
           && databaseMeta.supportsBatchUpdates()
           && Utils.isEmpty(connectionGroup);
     } catch (SQLException e) {
-      throw createHopDatabaseBatchException("Error determining whether to use batch", e);
+      throw createHopDatabaseBatchException(
+          "Error determining whether to use batch", e, this.log.getLogLevel());
     }
   }
 
@@ -1186,10 +1206,11 @@ public boolean insertRow(PreparedStatement ps, boolean batch, boolean handleComm
 
       return rowsAreSafe;
     } catch (BatchUpdateException ex) {
-      throw createHopDatabaseBatchException(CONST_ERROR_UPDATING_BATCH, ex);
+      throw createHopDatabaseBatchException(CONST_ERROR_UPDATING_BATCH, ex, this.log.getLogLevel());
     } catch (SQLException ex) {
       if (isBatchUpdate) {
-        throw createHopDatabaseBatchException(CONST_ERROR_UPDATING_BATCH, ex);
+        throw createHopDatabaseBatchException(
+            CONST_ERROR_UPDATING_BATCH, ex, this.log.getLogLevel());
       } else {
         throw new HopDatabaseException("Error inserting/updating row", ex);
       }
@@ -1299,7 +1320,12 @@ public void emptyAndCommit(
 
   public static HopDatabaseBatchException createHopDatabaseBatchException(
       String message, SQLException ex) {
-    HopDatabaseBatchException kdbe = new HopDatabaseBatchException(message, ex);
+    return createHopDatabaseBatchException(message, ex, DefaultLogLevel.getLogLevel());
+  }
+
+  public static HopDatabaseBatchException createHopDatabaseBatchException(
+      String message, SQLException ex, LogLevel logLevel1) {
+    HopDatabaseBatchException kdbe = new HopDatabaseBatchException(message, ex, logLevel1);
     if (ex instanceof BatchUpdateException batchUpdateException) {
       kdbe.setUpdateCounts(batchUpdateException.getUpdateCounts());
     } else {
@@ -1546,13 +1572,13 @@ public ResultSet openQuery(
 
         if (canWeSetFetchSize(pstmt)) {
           int maxRows = pstmt.getMaxRows();
-          int fs = Const.FETCH_SIZE <= maxRows ? maxRows : Const.FETCH_SIZE;
+          int fs = Math.max(maxRows, fetchSize > 0 ? fetchSize : Const.FETCH_SIZE);
           if (databaseMeta.isMySqlVariant()) {
             setMysqlFetchSize(pstmt, fs, maxRows);
           } else {
             pstmt.setFetchSize(fs);
           }
-
+          log.logBasic("Statement fetch size set to " + fs);
           pstmt.setFetchDirection(fetchMode);
         }
 
@@ -1568,13 +1594,14 @@ public ResultSet openQuery(
         selStmt = connection.createStatement();
         log.snap(Metrics.METRIC_DATABASE_CREATE_SQL_STOP, databaseMeta.getName());
         if (canWeSetFetchSize(selStmt)) {
-          int fs =
-              Const.FETCH_SIZE <= selStmt.getMaxRows() ? selStmt.getMaxRows() : Const.FETCH_SIZE;
+          int fs = Math.max(selStmt.getMaxRows(), fetchSize > 0 ? fetchSize : Const.FETCH_SIZE);
           if (databaseMeta.getIDatabase().isMySqlVariant() && databaseMeta.isStreamingResults()) {
             selStmt.setFetchSize(Integer.MIN_VALUE);
           } else {
             selStmt.setFetchSize(fs);
           }
+          log.logBasic("Statement fetch size set to " + fs);
+
           selStmt.setFetchDirection(fetchMode);
         }
         if (rowlimit > 0 && databaseMeta.supportsSetMaxRows()) {
@@ -1604,10 +1631,16 @@ public ResultSet openQuery(
   }
 
   private boolean canWeSetFetchSize(Statement statement) throws SQLException {
-    return databaseMeta.isFetchSizeSupported()
-        && (statement.getMaxRows() > 0
-            || databaseMeta.getIDatabase().isPostgresVariant()
-            || (databaseMeta.isMySqlVariant() && databaseMeta.isStreamingResults()));
+    if (!databaseMeta.isFetchSizeSupported()) {
+      return false;
+    }
+    // Override for delphix
+    if (fetchSize > 0) {
+      return true;
+    }
+    return statement.getMaxRows() > 0
+        || databaseMeta.getIDatabase().isPostgresVariant()
+        || (databaseMeta.isMySqlVariant() && databaseMeta.isStreamingResults());
   }
 
   public ResultSet openQuery(PreparedStatement ps, IRowMeta params, Object[] data)
@@ -1622,24 +1655,25 @@ public ResultSet openQuery(PreparedStatement ps, IRowMeta params, Object[] data)
       setValues(params, data, ps); // set the parameters!
       log.snap(Metrics.METRIC_DATABASE_SQL_VALUES_STOP, databaseMeta.getName());
 
+      if (rowlimit > 0 && databaseMeta.supportsSetMaxRows()) {
+        ps.setMaxRows(rowlimit);
+      }
+
       if (canWeSetFetchSize(ps)) {
         int maxRows = ps.getMaxRows();
-        int fs = Const.FETCH_SIZE <= maxRows ? maxRows : Const.FETCH_SIZE;
+        int fs = Math.max(maxRows, fetchSize > 0 ? fetchSize : Const.FETCH_SIZE);
         // mysql have some restriction on fetch size assignment
         if (databaseMeta.isMySqlVariant()) {
           setMysqlFetchSize(ps, fs, maxRows);
         } else {
           // other databases seems not.
           ps.setFetchSize(fs);
         }
+        log.logBasic("Statement fetch size set to " + fs);
 
         ps.setFetchDirection(ResultSet.FETCH_FORWARD);
       }
 
-      if (rowlimit > 0 && databaseMeta.supportsSetMaxRows()) {
-        ps.setMaxRows(rowlimit);
-      }
-
       log.snap(Metrics.METRIC_DATABASE_EXECUTE_SQL_START, databaseMeta.getName());
       res = ps.executeQuery();
       log.snap(Metrics.METRIC_DATABASE_EXECUTE_SQL_STOP, databaseMeta.getName());

core/src/main/java/org/apache/hop/core/exception/HopDatabaseBatchException.java

@@ -18,7 +18,11 @@
 package org.apache.hop.core.exception;
 
 import java.io.Serial;
+import java.sql.SQLException;
 import java.util.List;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.logging.DefaultLogLevel;
+import org.apache.hop.core.logging.LogLevel;
 
 /** This exception is used by the Database class. */
 public class HopDatabaseBatchException extends HopDatabaseException {
@@ -28,6 +32,8 @@ public class HopDatabaseBatchException extends HopDatabaseException {
 
   private List<Exception> exceptionsList;
 
+  private LogLevel logLevel = DefaultLogLevel.getLogLevel();
+
   /** Constructs a new throwable with null as its detail message. */
   public HopDatabaseBatchException() {
     super();
@@ -66,6 +72,19 @@ public HopDatabaseBatchException(String message, Throwable cause) {
     super(message, cause);
   }
 
+  /**
+   * Constructs a new throwable with the specified detail message and cause.
+   *
+   * @param message the detail message (which is saved for later retrieval by the getMessage()
+   *     method).
+   * @param cause the cause (which is saved for later retrieval by the getCause() method). (A null
+   *     value is permitted, and indicates that the cause is nonexistent or unknown.)
+   */
+  public HopDatabaseBatchException(String message, Throwable cause, LogLevel logLevel) {
+    super(message, cause);
+    this.logLevel = logLevel;
+  }
+
   /**
    * @return Returns the updateCounts.
    */
@@ -87,4 +106,45 @@ public void setExceptionsList(List<Exception> exceptionsList) {
   public List<Exception> getExceptionsList() {
     return exceptionsList;
   }
+
+  /*
+   * This method exists to fix DLPX-56268 (Masking does not see Hop getNextException to surface the cause of a SQLException).
+   *
+   * The HopException class overrides this method of Exception class to get the root cause. This
+   * class is a subclass of HopException class. It has an additional member field to store list
+   * of exceptions. The static method createHopDatabaseBatchException in Database class is used
+   * to create an instance of this class. In that method all the exceptions associated with SQLException
+   * are retrieved using the getNextException and stored in this list.
+   *
+   * While logging the error message, the getMessage method is used. Since it is a subclass of HopException class, it
+   * uses the method defined there. That definition ignores this list of exceptions. Thus these messages are supressed.
+   *
+   * We fix this by overriding the getMessage method here and log all the exceptions present in the exceptionList variable.
+   * It also calls the HopException's getMessage method so that the previous behavior is maintained while adding
+   * additional log messages.
+   */
+  @Override
+  public String getMessage() {
+    String retval = Const.CR;
+    retval += super.getMessage() + Const.CR;
+
+    for (Exception exc : exceptionsList) {
+      if (exc instanceof SQLException) {
+        SQLException sqlException = (SQLException) exc;
+        retval +=
+            "Next Exception: SQLState( "
+                + sqlException.getSQLState()
+                + ") ErrorCode("
+                + sqlException.getErrorCode()
+                + ")";
+        if (this.logLevel.isDetailed()) {
+          retval += " Message: " + sqlException.getMessage();
+        }
+        retval += Const.CR;
+      } else {
+        retval += "Next Exception: " + exc.getClass() + Const.CR;
+      }
+    }
+    return retval;
+  }
 }