Switch the Spark JDBC driver from Simba to Databricks JDBC 3.4.2

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: DatabaseConnector
 Type: Package
 Title: Connecting to Various Database Platforms
-Version: 8.0.0
-Date: 2026-10-01
+Version: 8.1.0
+Date: 2026-10-12
 Authors@R: c(
     person("Martijn", "Schuemie", email = "schuemie@ohdsi.org", role = c("aut", "cre")),
     person("Marc", "Suchard", role = c("aut")),

NEWS.md

@@ -1,3 +1,15 @@
+DatabaseConnector 8.1.0
+=======================
+
+Changes:
+
+1. Documented the Spark (Databricks) JDBC driver downloaded by `downloadJdbcDrivers()` as V2.6.40.
+
+2. `connectSpark()` no longer appends `UseNativeQuery`. That property belongs to the legacy Simba driver and is not supported by Databricks JDBC driver 3.x, which already sends native Databricks SQL. `AuthMech`, `ssl`, `httpPath`, and the user and password are unchanged. `transportMode` from older connection strings is ignored.
+
+3. Spark ODBC connections default to the `Databricks ODBC Driver` instead of hardcoding `Simba Spark ODBC Driver`. Pass `extraSettings = list(Driver = "Simba Spark ODBC Driver", ...)` to keep using the legacy driver.
+
+
 DatabaseConnector 8.0.0
 =======================
 

R/Connect.R

@@ -649,20 +649,44 @@ connectBigQuery <- function(connectionDetails) {
   return(connection)
 }
 
-connectSpark <- function(connectionDetails) {
-  inform("Connecting using Spark JDBC driver")
-  jarPath <- findPathToJar("^[Dd]atabricks.*\\.jar$", connectionDetails$pathToDriver)
-  driver <- getJbcDriverSingleton("com.databricks.client.jdbc.Driver", jarPath)
-  connectionString <- connectionDetails$connectionString()
+# Databricks JDBC driver 3.x (com.databricks.client.jdbc.Driver) sends SQL as native
+# Databricks SQL. UseNativeQuery is a legacy Simba property and is not part of this
+# driver, so it is not added. AuthMech, ssl, httpPath, user, and password are valid.
+# transportMode from older Simba URLs is ignored.
+prepareSparkJdbcConnectionString <- function(connectionString) {
   if (is.null(connectionString) || connectionString == "") {
     abort("Error: Connection string required for connecting to Spark.")
   }
-  if (!grepl("UseNativeQuery", connectionString)) {
-    if (!endsWith(connectionString, ";")) {
-      connectionString <- paste0(connectionString, ";")
+  connectionString
+}
+
+resolveSparkOdbcDriver <- function(extraSettings) {
+  driver <- NULL
+  if (is.list(extraSettings)) {
+    driver <- extraSettings$Driver
+  }
+  if (is.null(driver) || !is.character(driver) || length(driver) != 1 || !nzchar(driver)) {
+    "Databricks ODBC Driver"
+  } else {
+    driver
+  }
+}
+
+dropSparkOdbcDriverSetting <- function(extraSettings) {
+  if (is.list(extraSettings)) {
+    extraSettings$Driver <- NULL
+    if (length(extraSettings) == 0) {
+      extraSettings <- NULL
     }
-    connectionString <- paste0(connectionString, "UseNativeQuery=1")
   }
+  extraSettings
+}
+
+connectSpark <- function(connectionDetails) {
+  inform("Connecting using Spark JDBC driver")
+  jarPath <- findPathToJar("^[Dd]atabricks.*\\.jar$", connectionDetails$pathToDriver)
+  driver <- getJbcDriverSingleton("com.databricks.client.jdbc.Driver", jarPath)
+  connectionString <- prepareSparkJdbcConnectionString(connectionDetails$connectionString())
   if (is.null(connectionDetails$user())) {
     connection <- connectUsingJdbcDriver(driver, connectionString, dbms = connectionDetails$dbms)
   } else {
@@ -680,20 +704,22 @@ connectSpark <- function(connectionDetails) {
 connectSparkUsingOdbc <- function(connectionDetails) {
   inform("Connecting using Spark ODBC driver")
   ensure_installed("odbc")
+  # Default is the renamed Databricks ODBC driver. Callers that still have the
+  # legacy driver installed can pass Driver in extraSettings.
+  extraSettings <- dropSparkOdbcDriverSetting(connectionDetails$extraSettings)
   dbiConnectionDetails <- createDbiConnectionDetails(
     dbms = connectionDetails$dbms,
     drv = odbc::odbc(),
-    Driver = "Simba Spark ODBC Driver",
+    Driver = resolveSparkOdbcDriver(connectionDetails$extraSettings),
     Host = connectionDetails$server(),
     uid = connectionDetails$user(),
     pwd = connectionDetails$password(),
     Port = connectionDetails$port()
-    # UseNativeQuery=1
   )
-  if (!is.null(connectionDetails$extraSettings)) {
+  if (!is.null(extraSettings)) {
     dbiConnectionDetails <- append(
       dbiConnectionDetails,
-      connectionDetails$extraSettings
+      extraSettings
     )
   }
   connection <- connectUsingDbi(dbiConnectionDetails)

R/Drivers.R

@@ -44,7 +44,7 @@ jdbcDrivers <- new.env()
 #' - RedShift: V2.1.0.20
 #' - SQL Server: V9.2.0
 #' - Oracle: V19.8
-#' - Spark (Databricks): V2.6.36
+#' - Spark (Databricks): V2.6.40
 #' - Snowflake: V4.3.4
 #' - BigQuery: v0.10.0
 #' - InterSystems IRIS: v3.11.0
@@ -94,7 +94,7 @@ downloadJdbcDrivers <- function(dbms, pathToDriver = Sys.getenv("DATABASECONNECT
     2,redshift,redshift-jdbc42-2.1.0.20.zip,https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/2.1.0.20/
     3,sql server,sqlServerV9.2.0.zip,https://ohdsi.github.io/DatabaseConnectorJars/
     4,oracle,oracleV19.8.zip,https://ohdsi.github.io/DatabaseConnectorJars/
-    5,spark,DatabricksJDBC-3.4.2.zip,https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/3.4.2/
+    5,spark,databricks-jdbc-2.6.40.jar,https://repo1.maven.org/maven2/com/databricks/databricks-jdbc/2.6.40/
     6,snowflake,snowflake-jdbc-4.3.4.jar,https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/4.3.4/
     7,bigquery,google-cloud-bigquery-jdbc-0.10.0-all.jar,https://repo1.maven.org/maven2/com/google/cloud/google-cloud-bigquery-jdbc/0.10.0/
     8,iris,intersystems-jdbc-3.11.0.jar,https://repo1.maven.org/maven2/com/intersystems/intersystems-jdbc/3.11.0/"