Skip to content

Commit f6f441a

Browse files
treysptreysp
authored
Feat: Clickhouse HTTPS config parameters (#4571)
1 parent 0d0baf1 commit f6f441a

3 files changed

Lines changed: 68 additions & 2 deletions

File tree

‎docs/integrations/engines/clickhouse.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,4 +421,29 @@ If a model has many records in each partition, you may see additional performanc
421421
Choose a model's time partitioning granularity based on the characteristics of the data it will process, making sure the total number of partitions is 1000 or fewer.
422422

423423
## Local/Built-in Scheduler
424+
424425
**Engine Adapter Type**: `clickhouse`
426+
427+
| Option | Description | Type | Required |
428+
| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----: | :------: |
429+
| `type` | Engine type name - must be `clickhouse` | string | Y |
430+
| `host` | ClickHouse server hostname or IP address | string | Y |
431+
| `username` | ClickHouse user name | string | Y |
432+
| `password` | ClickHouse user password | string | N |
433+
| `port` | The ClickHouse HTTP or HTTPS port (Default: `8123`) | int | N |
434+
| `cluster` | ClickHouse cluster name | string | N |
435+
| `connect_timeout` | Connection timeout in seconds (Default: `10`) | int | N |
436+
| `send_receive_timeout` | Send/receive timeout in seconds (Default: `300`) | int | N |
437+
| `query_limit` | Query result limit (Default: `0` - no limit) | int | N |
438+
| `use_compression` | Whether to use compression (Default: `True`) | bool | N |
439+
| `compression_method` | Compression method to use | string | N |
440+
| `http_proxy` | HTTP proxy address (equivalent to setting the HTTP_PROXY environment variable) | string | N |
441+
| `verify` | Verify server TLS/SSL certificate (Default: `True`) | bool | N |
442+
| `ca_cert` | Ignored if verify is `False`. If verify is `True`, the file path to Certificate Authority root to validate ClickHouse server certificate, in .pem format. Not necessary if the ClickHouse server certificate is a globally trusted root as verified by the operating system. | string | N |
443+
| `client_cert` | File path to a TLS Client certificate in .pem format (for mutual TLS authentication). The file should contain a full certificate chain, including any intermediate certificates. | string | N |
444+
| `client_cert_key` | File path to the private key for the Client Certificate. Required if the private key is not included the Client Certificate key file. | string | N |
445+
| `https_proxy` | HTTPS proxy address (equivalent to setting the HTTPS_PROXY environment variable) | string | N |
446+
| `server_host_name` | The ClickHouse server hostname as identified by the CN or SNI of its TLS certificate. Set this to avoid SSL errors when connecting through a proxy or tunnel with a different hostname. | string | N |
447+
| `tls_mode` | Controls advanced TLS behavior. proxy and strict do not invoke ClickHouse mutual TLS connection, but do send client cert and key. mutual assumes ClickHouse mutual TLS auth with a client certificate. | string | N |
448+
| `connection_settings` | Additional [connection settings](https://clickhouse.com/docs/integrations/python#settings-argument) | dict | N |
449+
| `connection_pool_options` | Additional [options](https://clickhouse.com/docs/integrations/python#customizing-the-http-connection-pool) for the HTTP connection pool | dict | N |

‎sqlmesh/core/config/connection.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1737,11 +1737,19 @@ class ClickhouseConnectionConfig(ConnectionConfig):
17371737
cluster: t.Optional[str] = None
17381738
connect_timeout: int = 10
17391739
send_receive_timeout: int = 300
1740-
verify: bool = True
17411740
query_limit: int = 0
17421741
use_compression: bool = True
17431742
compression_method: t.Optional[str] = None
17441743
connection_settings: t.Optional[t.Dict[str, t.Any]] = None
1744+
http_proxy: t.Optional[str] = None
1745+
# HTTPS/TLS settings
1746+
verify: bool = True
1747+
ca_cert: t.Optional[str] = None
1748+
client_cert: t.Optional[str] = None
1749+
client_cert_key: t.Optional[str] = None
1750+
https_proxy: t.Optional[str] = None
1751+
server_host_name: t.Optional[str] = None
1752+
tls_mode: t.Optional[str] = None
17451753

17461754
concurrent_tasks: int = 1
17471755
register_comments: bool = True
@@ -1766,8 +1774,15 @@ def _connection_kwargs_keys(self) -> t.Set[str]:
17661774
"password",
17671775
"connect_timeout",
17681776
"send_receive_timeout",
1769-
"verify",
17701777
"query_limit",
1778+
"http_proxy",
1779+
"verify",
1780+
"ca_cert",
1781+
"client_cert",
1782+
"client_cert_key",
1783+
"https_proxy",
1784+
"server_host_name",
1785+
"tls_mode",
17711786
}
17721787
return kwargs
17731788

@@ -1786,7 +1801,18 @@ def _connection_factory(self) -> t.Callable:
17861801
maxsize=self.concurrent_tasks,
17871802
# Block if there are no free connections
17881803
block=True,
1804+
verify=self.verify,
1805+
ca_cert=self.ca_cert,
1806+
client_cert=self.client_cert,
1807+
client_cert_key=self.client_cert_key,
1808+
https_proxy=self.https_proxy,
17891809
)
1810+
# this doesn't happen automatically because we always supply our own pool manager to the connection
1811+
# https://github.com/ClickHouse/clickhouse-connect/blob/3a7f4b04cad29c7c2536661b831fb744248e2ec0/clickhouse_connect/driver/httpclient.py#L109
1812+
if self.server_host_name:
1813+
pool_manager_options["server_hostname"] = self.server_host_name
1814+
if self.verify:
1815+
pool_manager_options["assert_hostname"] = self.server_host_name
17901816
if self.connection_pool_options:
17911817
pool_manager_options.update(self.connection_pool_options)
17921818
pool_mgr = httputil.get_pool_manager(**pool_manager_options)

‎tests/core/test_connection_config.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,6 +902,13 @@ def test_clickhouse(make_config):
902902
cluster="default",
903903
use_compression=True,
904904
connection_settings={"this_setting": "1"},
905+
server_host_name="server_host_name",
906+
verify=True,
907+
ca_cert="ca_cert",
908+
client_cert="client_cert",
909+
client_cert_key="client_cert_key",
910+
https_proxy="https://proxy",
911+
connection_pool_options={"pool_option": "value"},
905912
)
906913
assert isinstance(config, ClickhouseConnectionConfig)
907914
assert config.cluster == "default"
@@ -912,6 +919,14 @@ def test_clickhouse(make_config):
912919
assert config.is_recommended_for_state_sync is False
913920
assert config.is_forbidden_for_state_sync
914921

922+
pool = config._connection_factory.keywords["pool_mgr"]
923+
assert pool.connection_pool_kw["server_hostname"] == "server_host_name"
924+
assert pool.connection_pool_kw["assert_hostname"] == "server_host_name" # because verify=True
925+
assert pool.connection_pool_kw["ca_certs"] == "ca_cert"
926+
assert pool.connection_pool_kw["cert_file"] == "client_cert"
927+
assert pool.connection_pool_kw["key_file"] == "client_cert_key"
928+
assert pool.connection_pool_kw["pool_option"] == "value"
929+
915930
config2 = make_config(
916931
type="clickhouse",
917932
host="localhost",

0 commit comments

Comments
 (0)