Skip to content

Commit e3d4ca0

Browse files
treysptreysp
committed
Expose HTTPS/SSL config parameters
1 parent 7b1b78a commit e3d4ca0

3 files changed

Lines changed: 68 additions & 2 deletions

File tree

‎docs/integrations/engines/clickhouse.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,4 +421,29 @@ If a model has many records in each partition, you may see additional performanc
421421
Choose a model's time partitioning granularity based on the characteristics of the data it will process, making sure the total number of partitions is 1000 or fewer.
422422

423423
## Local/Built-in Scheduler
424+
424425
**Engine Adapter Type**: `clickhouse`
426+
427+
| Option | Description | Type | Required |
428+
| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----: | :------: |
429+
| `type` | Engine type name - must be `clickhouse` | string | Y |
430+
| `host` | ClickHouse server hostname or IP address | string | Y |
431+
| `username` | ClickHouse user name | string | Y |
432+
| `password` | ClickHouse user password | string | N |
433+
| `port` | The ClickHouse HTTP or HTTPS port (Default: `8123`) | int | N |
434+
| `cluster` | ClickHouse cluster name | string | N |
435+
| `connect_timeout` | Connection timeout in seconds (Default: `10`) | int | N |
436+
| `send_receive_timeout` | Send/receive timeout in seconds (Default: `300`) | int | N |
437+
| `query_limit` | Query result limit (Default: `0` - no limit) | int | N |
438+
| `use_compression` | Whether to use compression (Default: `True`) | bool | N |
439+
| `compression_method` | Compression method to use | string | N |
440+
| `http_proxy` | HTTP proxy address (equivalent to setting the HTTP_PROXY environment variable) | string | N |
441+
| `verify` | Verify server TLS/SSL certificate (Default: `True`) | bool | N |
442+
| `ca_cert` | Ignored if verify is `False`. If verify is `True`, the file path to Certificate Authority root to validate ClickHouse server certificate, in .pem format. Not necessary if the ClickHouse server certificate is a globally trusted root as verified by the operating system. | string | N |
443+
| `client_cert` | File path to a TLS Client certificate in .pem format (for mutual TLS authentication). The file should contain a full certificate chain, including any intermediate certificates. | string | N |
444+
| `client_cert_key` | File path to the private key for the Client Certificate. Required if the private key is not included the Client Certificate key file. | string | N |
445+
| `https_proxy` | HTTPS proxy address (equivalent to setting the HTTPS_PROXY environment variable) | string | N |
446+
| `server_host_name` | The ClickHouse server hostname as identified by the CN or SNI of its TLS certificate. Set this to avoid SSL errors when connecting through a proxy or tunnel with a different hostname. | string | N |
447+
| `tls_mode` | Controls advanced TLS behavior. proxy and strict do not invoke ClickHouse mutual TLS connection, but do send client cert and key. mutual assumes ClickHouse mutual TLS auth with a client certificate. | string | N |
448+
| `connection_settings` | Additional [connection settings](https://clickhouse.com/docs/integrations/python#settings-argument) | dict | N |
449+
| `connection_pool_options` | Additional [options](https://clickhouse.com/docs/integrations/python#customizing-the-http-connection-pool) for the HTTP connection pool | dict | N |

‎sqlmesh/core/config/connection.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1734,11 +1734,19 @@ class ClickhouseConnectionConfig(ConnectionConfig):
17341734
cluster: t.Optional[str] = None
17351735
connect_timeout: int = 10
17361736
send_receive_timeout: int = 300
1737-
verify: bool = True
17381737
query_limit: int = 0
17391738
use_compression: bool = True
17401739
compression_method: t.Optional[str] = None
17411740
connection_settings: t.Optional[t.Dict[str, t.Any]] = None
1741+
http_proxy: t.Optional[str] = None
1742+
# HTTPS/TLS settings
1743+
verify: bool = True
1744+
ca_cert: t.Optional[str] = None
1745+
client_cert: t.Optional[str] = None
1746+
client_cert_key: t.Optional[str] = None
1747+
https_proxy: t.Optional[str] = None
1748+
server_host_name: t.Optional[str] = None
1749+
tls_mode: t.Optional[str] = None
17421750

17431751
concurrent_tasks: int = 1
17441752
register_comments: bool = True
@@ -1763,8 +1771,15 @@ def _connection_kwargs_keys(self) -> t.Set[str]:
17631771
"password",
17641772
"connect_timeout",
17651773
"send_receive_timeout",
1766-
"verify",
17671774
"query_limit",
1775+
"http_proxy",
1776+
"verify",
1777+
"ca_cert",
1778+
"client_cert",
1779+
"client_cert_key",
1780+
"https_proxy",
1781+
"server_host_name",
1782+
"tls_mode",
17681783
}
17691784
return kwargs
17701785

@@ -1783,7 +1798,18 @@ def _connection_factory(self) -> t.Callable:
17831798
maxsize=self.concurrent_tasks,
17841799
# Block if there are no free connections
17851800
block=True,
1801+
verify=self.verify,
1802+
ca_cert=self.ca_cert,
1803+
client_cert=self.client_cert,
1804+
client_cert_key=self.client_cert_key,
1805+
https_proxy=self.https_proxy,
17861806
)
1807+
# this doesn't happen automatically because we always supply our own pool manager to the connection
1808+
# https://github.com/ClickHouse/clickhouse-connect/blob/3a7f4b04cad29c7c2536661b831fb744248e2ec0/clickhouse_connect/driver/httpclient.py#L109
1809+
if self.server_host_name:
1810+
pool_manager_options["server_hostname"] = self.server_host_name
1811+
if self.verify:
1812+
pool_manager_options["assert_hostname"] = self.server_host_name
17871813
if self.connection_pool_options:
17881814
pool_manager_options.update(self.connection_pool_options)
17891815
pool_mgr = httputil.get_pool_manager(**pool_manager_options)

‎tests/core/test_connection_config.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -899,6 +899,13 @@ def test_clickhouse(make_config):
899899
cluster="default",
900900
use_compression=True,
901901
connection_settings={"this_setting": "1"},
902+
server_host_name="server_host_name",
903+
verify=True,
904+
ca_cert="ca_cert",
905+
client_cert="client_cert",
906+
client_cert_key="client_cert_key",
907+
https_proxy="https://proxy",
908+
connection_pool_options={"pool_option": "value"},
902909
)
903910
assert isinstance(config, ClickhouseConnectionConfig)
904911
assert config.cluster == "default"
@@ -909,6 +916,14 @@ def test_clickhouse(make_config):
909916
assert config.is_recommended_for_state_sync is False
910917
assert config.is_forbidden_for_state_sync
911918

919+
pool = config._connection_factory.keywords["pool_mgr"]
920+
assert pool.connection_pool_kw["server_hostname"] == "server_host_name"
921+
assert pool.connection_pool_kw["assert_hostname"] == "server_host_name" # because verify=True
922+
assert pool.connection_pool_kw["ca_certs"] == "ca_cert"
923+
assert pool.connection_pool_kw["cert_file"] == "client_cert"
924+
assert pool.connection_pool_kw["key_file"] == "client_cert_key"
925+
assert pool.connection_pool_kw["pool_option"] == "value"
926+
912927
config2 = make_config(
913928
type="clickhouse",
914929
host="localhost",

0 commit comments

Comments
 (0)