Skip to content

Commit 485452b

Browse files
treysptreysp
committed
Expose HTTPS/SSL config parameters
1 parent 405e218 commit 485452b

3 files changed

Lines changed: 68 additions & 2 deletions

File tree

‎docs/integrations/engines/clickhouse.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,4 +421,29 @@ If a model has many records in each partition, you may see additional performanc
421421
Choose a model's time partitioning granularity based on the characteristics of the data it will process, making sure the total number of partitions is 1000 or fewer.
422422

423423
## Local/Built-in Scheduler
424+
424425
**Engine Adapter Type**: `clickhouse`
426+
427+
| Option | Description | Type | Required |
428+
| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----: | :------: |
429+
| `type` | Engine type name - must be `clickhouse` | string | Y |
430+
| `host` | ClickHouse server hostname or IP address | string | Y |
431+
| `username` | ClickHouse user name | string | Y |
432+
| `password` | ClickHouse user password | string | N |
433+
| `port` | The ClickHouse HTTP or HTTPS port (Default: `8123`) | int | N |
434+
| `cluster` | ClickHouse cluster name | string | N |
435+
| `connect_timeout` | Connection timeout in seconds (Default: `10`) | int | N |
436+
| `send_receive_timeout` | Send/receive timeout in seconds (Default: `300`) | int | N |
437+
| `query_limit` | Query result limit (Default: `0` - no limit) | int | N |
438+
| `use_compression` | Whether to use compression (Default: `True`) | bool | N |
439+
| `compression_method` | Compression method to use | string | N |
440+
| `http_proxy` | HTTP proxy address (equivalent to setting the HTTP_PROXY environment variable) | string | N |
441+
| `verify` | Verify server TLS/SSL certificate (Default: `True`) | bool | N |
442+
| `ca_cert` | Ignored if verify is `False`. If verify is `True`, the file path to Certificate Authority root to validate ClickHouse server certificate, in .pem format. Not necessary if the ClickHouse server certificate is a globally trusted root as verified by the operating system. | string | N |
443+
| `client_cert` | File path to a TLS Client certificate in .pem format (for mutual TLS authentication). The file should contain a full certificate chain, including any intermediate certificates. | string | N |
444+
| `client_cert_key` | File path to the private key for the Client Certificate. Required if the private key is not included the Client Certificate key file. | string | N |
445+
| `https_proxy` | HTTPS proxy address (equivalent to setting the HTTPS_PROXY environment variable) | string | N |
446+
| `server_host_name` | The ClickHouse server hostname as identified by the CN or SNI of its TLS certificate. Set this to avoid SSL errors when connecting through a proxy or tunnel with a different hostname. | string | N |
447+
| `tls_mode` | Controls advanced TLS behavior. proxy and strict do not invoke ClickHouse mutual TLS connection, but do send client cert and key. mutual assumes ClickHouse mutual TLS auth with a client certificate. | string | N |
448+
| `connection_settings` | Additional [connection settings](https://clickhouse.com/docs/integrations/python#settings-argument) | dict | N |
449+
| `connection_pool_options` | Additional [options](https://clickhouse.com/docs/integrations/python#customizing-the-http-connection-pool) for the HTTP connection pool | dict | N |

‎sqlmesh/core/config/connection.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1722,11 +1722,19 @@ class ClickhouseConnectionConfig(ConnectionConfig):
17221722
cluster: t.Optional[str] = None
17231723
connect_timeout: int = 10
17241724
send_receive_timeout: int = 300
1725-
verify: bool = True
17261725
query_limit: int = 0
17271726
use_compression: bool = True
17281727
compression_method: t.Optional[str] = None
17291728
connection_settings: t.Optional[t.Dict[str, t.Any]] = None
1729+
http_proxy: t.Optional[str] = None
1730+
# HTTPS/TLS settings
1731+
verify: bool = True
1732+
ca_cert: t.Optional[str] = None
1733+
client_cert: t.Optional[str] = None
1734+
client_cert_key: t.Optional[str] = None
1735+
https_proxy: t.Optional[str] = None
1736+
server_host_name: t.Optional[str] = None
1737+
tls_mode: t.Optional[str] = None
17301738

17311739
concurrent_tasks: int = 1
17321740
register_comments: bool = True
@@ -1751,8 +1759,15 @@ def _connection_kwargs_keys(self) -> t.Set[str]:
17511759
"password",
17521760
"connect_timeout",
17531761
"send_receive_timeout",
1754-
"verify",
17551762
"query_limit",
1763+
"http_proxy",
1764+
"verify",
1765+
"ca_cert",
1766+
"client_cert",
1767+
"client_cert_key",
1768+
"https_proxy",
1769+
"server_host_name",
1770+
"tls_mode",
17561771
}
17571772
return kwargs
17581773

@@ -1771,7 +1786,18 @@ def _connection_factory(self) -> t.Callable:
17711786
maxsize=self.concurrent_tasks,
17721787
# Block if there are no free connections
17731788
block=True,
1789+
verify=self.verify,
1790+
ca_cert=self.ca_cert,
1791+
client_cert=self.client_cert,
1792+
client_cert_key=self.client_cert_key,
1793+
https_proxy=self.https_proxy,
17741794
)
1795+
# this doesn't happen automatically because we always supply our own pool manager to the connection
1796+
# https://github.com/ClickHouse/clickhouse-connect/blob/3a7f4b04cad29c7c2536661b831fb744248e2ec0/clickhouse_connect/driver/httpclient.py#L109
1797+
if self.server_host_name:
1798+
pool_manager_options["server_hostname"] = self.server_host_name
1799+
if self.verify:
1800+
pool_manager_options["assert_hostname"] = self.server_host_name
17751801
if self.connection_pool_options:
17761802
pool_manager_options.update(self.connection_pool_options)
17771803
pool_mgr = httputil.get_pool_manager(**pool_manager_options)

‎tests/core/test_connection_config.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -875,6 +875,13 @@ def test_clickhouse(make_config):
875875
cluster="default",
876876
use_compression=True,
877877
connection_settings={"this_setting": "1"},
878+
server_host_name="server_host_name",
879+
verify=True,
880+
ca_cert="ca_cert",
881+
client_cert="client_cert",
882+
client_cert_key="client_cert_key",
883+
https_proxy="https://proxy",
884+
connection_pool_options={"pool_option": "value"},
878885
)
879886
assert isinstance(config, ClickhouseConnectionConfig)
880887
assert config.cluster == "default"
@@ -885,6 +892,14 @@ def test_clickhouse(make_config):
885892
assert config.is_recommended_for_state_sync is False
886893
assert config.is_forbidden_for_state_sync
887894

895+
pool = config._connection_factory.keywords["pool_mgr"]
896+
assert pool.connection_pool_kw["server_hostname"] == "server_host_name"
897+
assert pool.connection_pool_kw["assert_hostname"] == "server_host_name" # because verify=True
898+
assert pool.connection_pool_kw["ca_certs"] == "ca_cert"
899+
assert pool.connection_pool_kw["cert_file"] == "client_cert"
900+
assert pool.connection_pool_kw["key_file"] == "client_cert_key"
901+
assert pool.connection_pool_kw["pool_option"] == "value"
902+
888903
config2 = make_config(
889904
type="clickhouse",
890905
host="localhost",

0 commit comments

Comments
 (0)