From be236e76a5bf9903c782551e640dee3ee816bef3 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 10:50:09 -0300 Subject: [PATCH 01/37] fix: enhance TLS configuration to support 'skip-verify' option --- core/dbio/database/database.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 06f6904d6..f7802d59f 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -3151,8 +3151,11 @@ func (conn *BaseConn) credsProvided(provider string) bool { } func (conn *BaseConn) makeTlsConfig() (tlsConfig *tls.Config, err error) { - if cast.ToBool(conn.GetProp("tls")) { + if val := strings.ToLower(conn.GetProp("tls")); g.In(val, "true", "skip-verify") { tlsConfig = &tls.Config{} + if val == "skip-verify" { + tlsConfig.InsecureSkipVerify = true + } } // legacy only handles files From 13e5d2a7a0956ecf911174a3917a9da0bf6f0fee Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 10:54:57 -0300 Subject: [PATCH 02/37] fix: enhance ClickHouse connection handling with TLS and SSH tunnel support --- core/dbio/database/database_clickhouse.go | 201 ++++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/core/dbio/database/database_clickhouse.go b/core/dbio/database/database_clickhouse.go index 1687aea21..7404d102b 100755 --- a/core/dbio/database/database_clickhouse.go +++ b/core/dbio/database/database_clickhouse.go @@ -2,17 +2,23 @@ package database import ( "context" + "crypto/tls" "database/sql" "fmt" + "net/url" + "os" "strings" "time" + "github.com/jmoiron/sqlx" "github.com/shopspring/decimal" + "github.com/slingdata-io/sling-cli/core" "github.com/slingdata-io/sling-cli/core/dbio" "github.com/slingdata-io/sling-cli/core/dbio/iop" "github.com/slingdata-io/sling-cli/core/env" "github.com/spf13/cast" + "github.com/ClickHouse/clickhouse-go/v2" _ "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/chcol" @@ -38,6 +44,201 @@ func (conn *ClickhouseConn) Init() error { func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { + // use base connect for HTTP + if url := conn.GetProp("http_url"); url != "" { + return conn.BaseConn.Connect(timeOut...) + } + + // build chOptions + tlsConfig, err := conn.makeTlsConfig() + if err != nil { + return g.Error(err, "could not make tls config") + } + + // Handle ClickHouse specific TLS settings + if cast.ToBool(conn.GetProp("secure")) { + if tlsConfig == nil { + tlsConfig = &tls.Config{} + } + if cast.ToBool(conn.GetProp("skip_verify")) { + tlsConfig.InsecureSkipVerify = true + } + } + + settings := clickhouse.Settings{} + g.JSONConvert(conn.GetProp("extra_settings"), &settings) + + // Build address from host and port + host := conn.GetProp("host") + if host == "" { + host = "127.0.0.1" + } + port := conn.GetProp("port") + if port == "" { + port = "9000" + } + addr := host + ":" + port + + chOptions := clickhouse.Options{ + Addr: []string{addr}, + Auth: clickhouse.Auth{ + Database: conn.GetProp("database"), + Username: conn.GetProp("username"), + Password: conn.GetProp("password"), + }, + TLS: tlsConfig, + Settings: settings, + DialTimeout: time.Second * 30, + Compression: &clickhouse.Compression{ + Method: clickhouse.CompressionZSTD, + }, + Debug: cast.ToBool(conn.GetProp("debug")), + BlockBufferSize: 10, + MaxCompressionBuffer: 10240, + ClientInfo: clickhouse.ClientInfo{ + Products: []struct { + Name string + Version string + }{ + {Name: "sling", Version: core.Version}, + }, + }, + } + + // for debugging + chOptionsMap := g.M( + "addr", chOptions.Addr, + "auth", g.M( + "database", chOptions.Auth.Database, + "username", chOptions.Auth.Username, + "password", chOptions.Auth.Password, + ), + "tls", g.M( + "enabled", chOptions.TLS != nil, + "insecure_skip_verify", chOptions.TLS != nil && chOptions.TLS.InsecureSkipVerify, + ), + "settings", chOptions.Settings, + "dial_timeout", chOptions.DialTimeout.String(), + "compression", g.M( + "method", chOptions.Compression.Method.String(), + ), + "debug", chOptions.Debug, + "block_buffer_size", chOptions.BlockBufferSize, + "max_compression_buffer", chOptions.MaxCompressionBuffer, + "client_info", g.M( + "products", chOptions.ClientInfo.Products, + ), + ) + + var tryNum int + + to := 15 + if len(timeOut) > 0 && timeOut[0] != 0 { + to = timeOut[0] + } + + usePool = os.Getenv("USE_POOL") == "TRUE" + // g.Trace("conn.Type: %s", conn.Type) + // g.Trace("conn.URL: " + conn.Self().GetURL()) + if conn.Type == "" { + return g.Error("Invalid URL? conn.Type needs to be specified") + } + + connURL := conn.Self().ConnString() + + // start SSH Tunnel with SSH_TUNNEL prop + if sshURL := conn.GetProp("SSH_TUNNEL"); sshURL != "" { + + connU, err := url.Parse(connURL) + if err != nil { + return g.Error(err, "could not parse connection URL for SSH forwarding") + } + + connHost := connU.Hostname() + connPort := cast.ToInt(connU.Port()) + if connPort == 0 { + connPort = conn.defaultPort + connURL = strings.ReplaceAll( + connURL, g.F("@%s", connHost), + g.F("@%s:%d", connHost, connPort), + ) + } + + localPort, err := iop.OpenTunnelSSH(connHost, connPort, sshURL, conn.GetProp("SSH_PRIVATE_KEY"), conn.GetProp("SSH_PASSPHRASE")) + if err != nil { + return g.Error(err, "could not connect to ssh tunnel server") + } + + connURL = strings.ReplaceAll( + connURL, g.F("@%s:%d", connHost, connPort), + g.F("@127.0.0.1:%d", localPort), + ) + g.Trace("new connection URL: " + conn.Self().GetURL(connURL)) + conn.SetProp("ssh_url", connURL) // set ssh url for 3rd party bulk loading + } + + if conn.db == nil { + connURL = conn.Self().GetURL(connURL) + connPool.Mux.Lock() + db, poolOk := connPool.Dbs[connURL] + connPool.Mux.Unlock() + + driver := getDriverName(conn) + g.Trace("driver=%s conn_options=%s", driver, g.Marshal(chOptionsMap)) + + if !usePool || !poolOk { + + dbConn := clickhouse.OpenDB(&chOptions) + dbConn.SetMaxIdleConns(5) + dbConn.SetMaxOpenConns(10) + + db = sqlx.NewDb(dbConn, driver) + } else { + conn.SetProp("POOL_USED", cast.ToString(poolOk)) + } + + conn.db = db + + tryNum++ + + // 15 sec timeout + pingCtx, cancel := context.WithTimeout(conn.Context().Ctx, time.Duration(to)*time.Second) + _ = cancel // lint complaint + + err = conn.db.PingContext(pingCtx) + if err != nil { + if strings.Contains(err.Error(), "unexpected packet") { + g.Info(env.MagentaString("Try using the `http_url` instead to connect to Clickhouse via HTTP. See https://docs.slingdata.io/connections/database-connections/clickhouse")) + } + return g.Error(err, "could not connect to database") + } + + if !cast.ToBool(conn.GetProp("silent")) { + g.Debug(`opened "%s" connection (%s)`, conn.Type, conn.GetProp("sling_conn_id")) + } + + // add to pool after successful connection + if usePool && !poolOk { + connPool.Mux.Lock() + connPool.Dbs[connURL] = db + connPool.Mux.Unlock() + + // expire the connection from pool after 10 minutes of + timer := time.NewTimer(time.Duration(10*60) * time.Second) + go func() { + select { + case <-timer.C: + connPool.Mux.Lock() + delete(connPool.Dbs, connURL) + connPool.Mux.Unlock() + } + }() + } + } + + conn.SetProp("connected", "true") + conn.SetProp("connect_time", cast.ToString(time.Now())) + err = conn.BaseConn.Connect(timeOut...) if err != nil { if strings.Contains(err.Error(), "unexpected packet") { From fad8bfa1fb67594bd91fbdca407d2ee7a675227e Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 10:55:02 -0300 Subject: [PATCH 03/37] fix: improve SSH tunnel handling by extracting host and port from connection options --- core/dbio/database/database_clickhouse.go | 38 +++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/core/dbio/database/database_clickhouse.go b/core/dbio/database/database_clickhouse.go index 7404d102b..cb6c3df5c 100755 --- a/core/dbio/database/database_clickhouse.go +++ b/core/dbio/database/database_clickhouse.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "database/sql" "fmt" - "net/url" "os" "strings" "time" @@ -148,20 +147,21 @@ func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { // start SSH Tunnel with SSH_TUNNEL prop if sshURL := conn.GetProp("SSH_TUNNEL"); sshURL != "" { - - connU, err := url.Parse(connURL) - if err != nil { - return g.Error(err, "could not parse connection URL for SSH forwarding") + // Extract host and port from chOptions.Addr + if len(chOptions.Addr) == 0 { + return g.Error("no addresses configured for SSH tunnel") } - - connHost := connU.Hostname() - connPort := cast.ToInt(connU.Port()) + + originalAddr := chOptions.Addr[0] // use first address + parts := strings.Split(originalAddr, ":") + if len(parts) != 2 { + return g.Error("invalid address format for SSH tunnel: %s", originalAddr) + } + + connHost := parts[0] + connPort := cast.ToInt(parts[1]) if connPort == 0 { - connPort = conn.defaultPort - connURL = strings.ReplaceAll( - connURL, g.F("@%s", connHost), - g.F("@%s:%d", connHost, connPort), - ) + connPort = conn.GetType().DefPort() } localPort, err := iop.OpenTunnelSSH(connHost, connPort, sshURL, conn.GetProp("SSH_PRIVATE_KEY"), conn.GetProp("SSH_PASSPHRASE")) @@ -169,12 +169,12 @@ func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { return g.Error(err, "could not connect to ssh tunnel server") } - connURL = strings.ReplaceAll( - connURL, g.F("@%s:%d", connHost, connPort), - g.F("@127.0.0.1:%d", localPort), - ) - g.Trace("new connection URL: " + conn.Self().GetURL(connURL)) - conn.SetProp("ssh_url", connURL) // set ssh url for 3rd party bulk loading + // Update chOptions.Addr to use localhost with the local tunnel port + newAddr := g.F("127.0.0.1:%d", localPort) + chOptions.Addr[0] = newAddr + + g.Trace("SSH tunnel established: %s -> %s", originalAddr, newAddr) + conn.SetProp("ssh_tunnel_local_addr", newAddr) // store for potential 3rd party bulk loading } if conn.db == nil { From 6189d7b847ec6c389bf2b2c0ae756c4cfa93bbcf Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 11:03:22 -0300 Subject: [PATCH 04/37] fix: improve ClickHouse connection handling for HTTP and SSH tunnel scenarios --- core/dbio/database/database_clickhouse.go | 25 ++++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/core/dbio/database/database_clickhouse.go b/core/dbio/database/database_clickhouse.go index cb6c3df5c..451e86dec 100755 --- a/core/dbio/database/database_clickhouse.go +++ b/core/dbio/database/database_clickhouse.go @@ -42,18 +42,23 @@ func (conn *ClickhouseConn) Init() error { } func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { - - // use base connect for HTTP - if url := conn.GetProp("http_url"); url != "" { - return conn.BaseConn.Connect(timeOut...) - } - - // build chOptions tlsConfig, err := conn.makeTlsConfig() if err != nil { return g.Error(err, "could not make tls config") } + // use base connect for HTTP and non-TLS connection + if tlsConfig == nil { + err = conn.BaseConn.Connect(timeOut...) + if err != nil { + if strings.Contains(err.Error(), "unexpected packet") { + g.Info(env.MagentaString("Try using the `http_url` instead to connect to Clickhouse via HTTP. See https://docs.slingdata.io/connections/database-connections/clickhouse")) + } + } + return err + } + + // build chOptions // Handle ClickHouse specific TLS settings if cast.ToBool(conn.GetProp("secure")) { if tlsConfig == nil { @@ -151,13 +156,13 @@ func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { if len(chOptions.Addr) == 0 { return g.Error("no addresses configured for SSH tunnel") } - + originalAddr := chOptions.Addr[0] // use first address parts := strings.Split(originalAddr, ":") if len(parts) != 2 { return g.Error("invalid address format for SSH tunnel: %s", originalAddr) } - + connHost := parts[0] connPort := cast.ToInt(parts[1]) if connPort == 0 { @@ -172,7 +177,7 @@ func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { // Update chOptions.Addr to use localhost with the local tunnel port newAddr := g.F("127.0.0.1:%d", localPort) chOptions.Addr[0] = newAddr - + g.Trace("SSH tunnel established: %s -> %s", originalAddr, newAddr) conn.SetProp("ssh_tunnel_local_addr", newAddr) // store for potential 3rd party bulk loading } From 10b6bf0aab5c6f732ec390d98d336a0cee15a1fe Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 11:05:21 -0300 Subject: [PATCH 05/37] fix: remove redundant TLS config initialization in ClickHouse connection --- core/dbio/database/database_clickhouse.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/dbio/database/database_clickhouse.go b/core/dbio/database/database_clickhouse.go index 451e86dec..85ba7b744 100755 --- a/core/dbio/database/database_clickhouse.go +++ b/core/dbio/database/database_clickhouse.go @@ -2,7 +2,6 @@ package database import ( "context" - "crypto/tls" "database/sql" "fmt" "os" @@ -58,12 +57,8 @@ func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { return err } - // build chOptions // Handle ClickHouse specific TLS settings if cast.ToBool(conn.GetProp("secure")) { - if tlsConfig == nil { - tlsConfig = &tls.Config{} - } if cast.ToBool(conn.GetProp("skip_verify")) { tlsConfig.InsecureSkipVerify = true } @@ -83,6 +78,7 @@ func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { } addr := host + ":" + port + // build chOptions chOptions := clickhouse.Options{ Addr: []string{addr}, Auth: clickhouse.Auth{ From 5010f437df064adbdf05c49b7f9784a1f43f4f95 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 11:18:20 -0300 Subject: [PATCH 06/37] fix: extend TLS configuration to support 'custom' option in ClickHouse connection --- core/dbio/database/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index f7802d59f..9872be0d8 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -3151,7 +3151,7 @@ func (conn *BaseConn) credsProvided(provider string) bool { } func (conn *BaseConn) makeTlsConfig() (tlsConfig *tls.Config, err error) { - if val := strings.ToLower(conn.GetProp("tls")); g.In(val, "true", "skip-verify") { + if val := strings.ToLower(conn.GetProp("tls")); g.In(val, "true", "skip-verify", "custom") { tlsConfig = &tls.Config{} if val == "skip-verify" { tlsConfig.InsecureSkipVerify = true From 100dd4d0c41bfaa1bea0243c062a88661ae6bfa6 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 18:36:56 -0300 Subject: [PATCH 07/37] fix: add 'secure=true' query parameter for HTTPS URLs in Clickhouse connection --- core/dbio/database/database_clickhouse.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/dbio/database/database_clickhouse.go b/core/dbio/database/database_clickhouse.go index 85ba7b744..43e8a3daa 100755 --- a/core/dbio/database/database_clickhouse.go +++ b/core/dbio/database/database_clickhouse.go @@ -21,6 +21,7 @@ import ( "github.com/ClickHouse/clickhouse-go/v2/lib/chcol" "github.com/flarco/g" + "github.com/flarco/g/net" ) // ClickhouseConn is a Clikchouse connection @@ -253,6 +254,19 @@ func (conn *ClickhouseConn) Connect(timeOut ...int) (err error) { func (conn *ClickhouseConn) ConnString() string { if url := conn.GetProp("http_url"); url != "" { + if strings.HasPrefix(url, "https://") && !strings.Contains(url, "secure=true") { + // let's add `secure=true` query param + parsedURL, err := net.NewURL(url) + if err != nil { + return url // fallback to original URL if parsing fails + } + + query := parsedURL.U.Query() + query.Set("secure", "true") + parsedURL.U.RawQuery = query.Encode() + + url = parsedURL.String() + } return url } From 4612052050123c2342cb3b00c4f4cf9327098624 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 20 Aug 2025 22:58:59 -0300 Subject: [PATCH 08/37] fix: enhance EnsureBinDuckDB to validate DUCKDB_PATH as a file and support additional ARM architectures --- core/dbio/iop/duckdb.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index ff07f9729..5c0bcd56e 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -986,7 +986,11 @@ func EnsureBinDuckDB(version string) (binPath string, err error) { if !g.PathExists(envPath) { return "", g.Error("duckdb binary not found: %s", envPath) } - return envPath, nil + if stat, _ := os.Stat(envPath); stat.IsDir() { + return "", g.Error("DUCKDB_PATH provided is a directory, should be a file: %s", envPath) + } else { + return envPath, nil + } } if useTempFile := os.Getenv("DUCKDB_USE_TMP_FILE"); useTempFile != "" { @@ -1048,7 +1052,7 @@ func EnsureBinDuckDB(version string) (binPath string, err error) { case "linux/amd64": downloadURL = "https://github.com/duckdb/duckdb/releases/download/v{version}/duckdb_cli-linux-amd64.zip" - case "linux/aarch64": + case "linux/aarch64", "linux/arm64", "linux/arm": downloadURL = "https://github.com/duckdb/duckdb/releases/download/v{version}/duckdb_cli-linux-arm64.zip" default: From 69a94f476b1540713a2636a96c6bfab46c518444 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 21 Aug 2025 10:52:51 -0300 Subject: [PATCH 09/37] fix: update mcp.yaml to enhance documentation and clarify connection and API spec actions --- cmd/sling/resource/mcp.yaml | 478 +++++++++++------------------------- 1 file changed, 143 insertions(+), 335 deletions(-) diff --git a/cmd/sling/resource/mcp.yaml b/cmd/sling/resource/mcp.yaml index cafe1609b..64b2bfc9d 100644 --- a/cmd/sling/resource/mcp.yaml +++ b/cmd/sling/resource/mcp.yaml @@ -1,11 +1,14 @@ tools: connection: | **Main connection tool that provides comprehensive connection management capabilities. This is a parent tool that orchestrates various connection operations through sub-actions.** + + **IMPORTANT**: To fully understand how to use connections, first call the `connection/docs` tool. **Parameters**: - `action` (string, required): The connection action to perform. Valid values: + - `"docs"` - Fetch documentation on how to use connections - `"list"` - List all available connections - - `"discover"` - Discover streams/tables/files in a connection + - `"discover"` - Discover tables/files in a connection - `"test"` - Test an existing connection - `"set"` - Create or update a connection - `"database_get_schemata"` - Get database schemata (schemas, tables, columns) @@ -59,7 +62,7 @@ tools: - `limit` (number, optional): The limit of rows to return (defaults to 100) - `transient` (boolean, optional): Whether to use a transient connection (default: false) - This aciton executes a SQL query on a database connection and return the results. **WARNING: Only use this tool for SELECT queries and other read-only operations. Never execute destructive queries such as DELETE, DROP, TRUNCATE, ALTER, UPDATE, INSERT, or any other data modification operations.** + This action executes a SQL query on a database connection and return the results. **WARNING: Only use this tool for SELECT queries and other read-only operations. Never execute destructive queries such as DELETE, DROP, TRUNCATE, ALTER, UPDATE, INSERT, or any other data modification operations.** * If a destructive operation (e.g., dropping an object, deleting significant data, altering table structures) is deemed necessary, **DO NOT execute it directly**. Instead, formulate the required SQL query/statement and **return it to the USER for manual review and execution**. * Always use the least number of columns, to minimize computation and the amount of data being retrieved. @@ -126,294 +129,81 @@ tools: } } ``` - - conns_list: | - List the connections detected and available for use. - - **Parameters**: None. - - **Output (for User)**: A CSV string listing connections with columns: `name`, `description`, `type`. The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string representing a list of connections, each with `name`, `description`, `source`, and `type`. - - conns_discover: | - Discover available streams/tables/files in a connection. - - **Parameters**: - - `connection` (string, required): The name of the connection to discover. - - `pattern` (string, required): Filter stream name by glob pattern (e.g. schema.prefix_*, dir/*.csv, dir/**/*.json, */*/*.parquet). - - `recursive` (boolean, required): List all files recursively. - - `columns` (boolean, required): Show column level metadata. - - **Output (for User)**: A CSV string with varying columns depending on discovery type: - - For database columns: `database`, `schema`, `table`, `name`, `type`, `position` - - For database tables: `database`, `schema`, `name`, `type` - - For database schemas: `database`, `name` - - For file columns: `file`, `name`, `type`, `position` - - For files: `name`, `type`, `size`, `updated`, `is_dir` - - For API endpoints: `name` - The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string detailing the discovered items. The structure varies based on connection type and discovery level. - - conns_database_get_schemata: | - Obtains schemata (databases, schemas, tables, columns) from a database connection. - - **Parameters**: - - `connection` (string, required): The name of the connection to get schemata from. - - `level` (string, required): The level of schemata to retrieve: 'schema', 'table', or 'column'. - - `schema_name` (string, optional): The specific schema name to inspect. Required for 'table' and 'column' levels if not fetching all schemas. - - `table_names` (array, optional): Optional list of table names to focus on. Relevant for 'table' and 'column' levels. - - **Output**: A JSON string representing the database schemata (databases, schemas, tables, columns) according to the specified `level`, `schema_name`, and `table_names`. - - conns_database_get_schemas: | - Retrieves a list of schema names from a database connection. - - **Parameters**: - - `connection` (string, required): The name of the connection to get schemata from. - - **Output**: A JSON string representing an `iop.Dataset` where each row contains a schema name from the specified database connection. - - conns_database_query: | - Execute a SQL query on a database connection and return the results. **WARNING: Only use this tool for SELECT queries and other read-only operations. Never execute destructive queries such as DELETE, DROP, TRUNCATE, ALTER, UPDATE, INSERT, or any other data modification operations.** - - * If a destructive operation (e.g., dropping an object, deleting significant data, altering table structures) is deemed necessary, **DO NOT execute it directly**. Instead, formulate the required SQL query/statement and **return it to the USER for manual review and execution**. - * Always use the least number of columns, to minimize computation and the amount of data being retrieved. - * Always use the LIMIT clause to minimize the number of rows returned. - - **Parameters**: - - `connection` (string, required): The name of the database connection to execute the query on. - - `query` (string, required): The SQL query to execute. - - `limit` (number, optional): The limit of rows to return (defaults to 100). - - `transient` (boolean, optional): Whether to use a transient connection (default: false). - - **Output (for User)**: A CSV formatted string of the query results. The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string containing the `query` executed, `connection` name, an array of `columns` names, an array of `rows` (each row being an array of values), and the `row_count`. - - conns_database_get_columns: | - Get column metadata for a specific table in a database connection. - - **Parameters**: - - `connection` (string, required): The name of the database connection. - - `table_name` (string, required): The fully qualified table name (e.g. 'schema.table' or just 'table'). - - **Output (for User)**: A CSV string listing columns with: `name`, `type`, `position`. The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string with `type: "columns"`, `connection` name, `table_name`, `full_table_name`, `column_count`, and an array of `columns` with detailed metadata. - - conns_test: | - Test a database, API or file system connection to verify it works correctly. - - **Parameters**: - - `debug` (boolean, optional): Enable debug logging during the test (default: false). - - `trace` (boolean, optional): Enable trace logging during the test (default: false). - - `endpoints` (array, optional): Specify the endpoints to test (only for API connections) - - **Output (for User)**: A simple text message: "Connection test for '[connection_name]': SUCCESS" or "Connection test for '[connection_name]': FAILED - [error_message]". - - **Output (for Assistant)**: A JSON string with `type: "connection_test"`, `connection` name, `success` (boolean), `error` (string, null if success), and captured `logs` from the test. - - conns_set: | - Create or update a connection by setting its properties. For API Specs, if secrets are required, put in a nested object under the `secrets` key. - - **Parameters**: - - `name` (string, required): The name of the connection to set (will be converted to uppercase). - - `properties` (object, required): Object containing the connection properties as key-value pairs (e.g. `{"type": "postgres", "host": "localhost", "user": "myuser"}` or for APIs `{"type": "api", "spec": "file:///path/to/spec.yaml", "secrets": {"token", "mytoken"}}`). - - **Output (for User)**: A text message: "Connection '[NAME]' has been set in [env_file_path]. Please test with `sling conns test [NAME]`". - - **Output (for Assistant)**: A JSON string with `type: "connection_set"`, `connection` name, the `properties` that were set (keys normalized to lowercase), `env_file_path`, and `success: true`. - - conns_file_list: | - List files and directories in a file system connection path. - - **Parameters**: - - `connection` (string, required): The name of the file system connection to list from. - - `path` (string, required): The path to list files from. - - `recursive` (boolean, optional): Whether to list files recursively (default: false). - - `only` (string, optional): Filter results to only 'files' or 'folders'. - - **Output (for User)**: A CSV string listing items with columns: `path`, `type` (directory/file), `size` (human-readable), `last_updated` (formatted timestamp and time ago). The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string with `type: "file_list"`, `connection` name, `path` queried, `recursive` flag, `only` filter, `count` of items, and a `files` array with detailed metadata for each item. - - conns_file_copy: | - Copy files from one file system connection to another. - - **Parameters**: - - `target_location` (string, required): The target location in format 'connection_name/path' (e.g. 's3/folder/file.txt' or 'local/path/to/file'). - - `recursive` (boolean, optional): Whether to copy files recursively (default: false). - - **Output**: A JSON string with `type: "file_copy"`, details of `source_location`, `source_connection`, `source_path`, `source_uri`, `target_location`, `target_connection`, `target_path`, `target_uri`, `bytes_written`, and `recursive` flag. - - conns_file_inspect: | - Inspect a file or directory in a file system connection to get metadata. - - **Parameters**: - - `path` (string, required): The path to inspect. - - `recursive` (boolean, optional): Whether to get recursive statistics for directories (default: false). - - **Output**: A JSON string with `type: "file_inspect"`, `connection` name, `path` inspected, `recursive` flag, and a `metadata` object containing detailed file/directory information. api_spec: | **Main API specification tool that provides comprehensive API spec management capabilities. This is a parent tool that orchestrates various API specification operations through sub-actions.** + + **IMPORTANT**: To fully understand how to use replications, first call the `api_spec/docs` tool. **Parameters**: - `action` (string, required): The API spec action to perform. Valid values: - - `"save"` - Validate and save an API specification to the specs folder - - `"list"` - List all API specification files in the specs folder - - `"get"` - Get the content of an API specification file - - `"load"` - Load the YAML content of an API specification - - `"edit"` - Edit an API specification file with search and replace + - `"parse"` - Load and parse the API specification file, will return various details - `"docs"` - Fetch the Sling API specification documentation + - `"test"` - Test an existing API Spec connection - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: - **For `action: "save"`**: - - `name` (string, required): The name of the API specification (will be used as filename without .yaml extension). Can also be the full file path - - `spec_content` (string, required): The YAML content of the API specification to validate and save - - **For `action: "list"`**: - - No parameters required (empty object) - - **For `action: "get"`**: - - `name` (string, required): The name of the API specification to get (without .yaml extension) - - **For `action: "load"`**: - - `name` (string, required): The name of the API specification (without .yaml extension) or full file path - - **For `action: "edit"`**: - - `name` (string, required): The name of the API specification (without .yaml extension) or full file path - - `old_string` (string, required): The exact text to find and replace - - `new_string` (string, required): The replacement text - - `expected_replacements` (number, optional): Expected number of replacements (default: 1) + **For `action: "parse"`**: + - `file_path` (string, required): The file path of the API specification. **For `action: "docs"`**: - No parameters required (empty object) + **For `action: "test"`**: + - `connection` (string, required): The name of the connection to test + - `debug` (boolean, optional): Enable debug logging during the test (default: false) + - `trace` (boolean, optional): Enable trace logging during the test (default: false) + - `endpoints` (array, optional): Specify the endpoints to test (only for API connections) + - `limit` (integer, optional): Specify the maximum number of records to process. + **Output**: The output format depends on the action and audience: - **For User**: Human-readable messages, CSV tables, or raw content - **For Assistant**: JSON strings with detailed metadata and operation results **Example Usage**: ``` - # List all API specifications + # Obtain API specification Documentation { - "action": "list", + "action": "docs", "input": {} } - # Get API specification content + # Test an API Spec connection { - "action": "get", - "input": { - "name": "github_api" - } - } - - # Save a new API specification - { - "action": "save", + "action": "test", "input": { - "name": "my_api", - "spec_content": "name: my_api\nbase_url: https://api.example.com\n..." + "connection": "MY_API", + "debug": true, + "limit": 15 } } - # Edit an API specification + # Parse API specification content { - "action": "edit", + "action": "parse", "input": { - "name": "my_api", - "old_string": "old_endpoint", - "new_string": "new_endpoint" + "file_path": "path/to/github_api.yaml" } } ``` - api_spec_save: | - Validate and save an API specification YAML content to the specs folder in sling home directory. - - **Parameters**: - - `name` (string, required): The name of the API specification (will be used as filename without .yaml extension). Can also be the full file path. - - `spec_content` (string, required): The YAML content of the API specification to validate and save. - - **Output (for User)**: A text message: "API specification '[name]' validated and saved to [file_path]". - - **Output (for Assistant)**: A JSON string with `type: "api_spec_save"`, `name` of the spec, `file_path` where it was saved, `specs_dir` (sling home specs folder), and `success: true`. - - api_spec_list: | - List all YAML API specification files in the sling home specs folder. - - **Parameters**: None. - - **Output (for User)**: A CSV string listing `spec_file_name` for each API specification found. The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string with `type: "api_spec_list"`, `specs_dir`, `count` of specs found, and a `specs` array with detailed metadata for each specification. - - api_spec_get: | - Get the content of an API specification by name from the sling home specs folder. - - **Parameters**: - - `name` (string, required): The name of the API specification to get (without .yaml extension). - - **Output (for User)**: If parsing is successful, a CSV string listing endpoints with columns: `endpoint_name`, `endpoint_description`, `endpoint_method`. If parsing fails, a text message: "Error parsing API spec: [error_message]". - - **Output (for Assistant)**: A JSON string with `type: "api_spec_get"`, `name` of the spec, `file_path`, and raw `content` (string). If parsing was successful, includes additional parsed metadata. - - api_spec_load: | - Load the YAML content of an API specification by name or file path. - - **Parameters**: - - `name` (string, required): The name of the API specification (without .yaml extension) or full file path. - - **Output (for User)**: The raw YAML content of the API specification. - - **Output (for Assistant)**: A JSON string with `type: "api_spec_load"`, `name` parameter, `file_path` of the loaded file, and the raw `content` (string) of the YAML specification. - - api_spec_edit: | - Edit an API specification file by performing a search and replace operation. - - **Parameters**: - - `name` (string, required): The name of the API specification (without .yaml extension) or full file path. - - `old_string` (string, required): The exact text to find and replace. - - `new_string` (string, required): The replacement text. - - `expected_replacements` (number, optional): Expected number of replacements (default: 1). - - **Output (for User)**: A text message: "API specification '[name]' edited successfully - made [count] replacements". - - **Output (for Assistant)**: A JSON string with `type: "api_spec_edit"`, `name` parameter, `file_path` of the edited file, `old_string`, `new_string`, `expected_replacements`, `actual_replacements`, and `success: true`. - - api_spec_docs: | - Fetch the Sling API specification documentation from the official repository. - - **Parameters**: None. - - **Output**: The raw markdown content of the API specification documentation from https://raw.githubusercontent.com/slingdata-io/sling-cli/refs/heads/main/api/llms.md. Both user and assistant audiences receive the same raw content. - replication: | **Main replication tool that provides comprehensive data replication management capabilities. This is a parent tool that orchestrates various replication operations through sub-actions.** + + **IMPORTANT**: To fully understand how to use replications, first call the `replication/docs` tool. **Parameters**: - `action` (string, required): The replication action to perform. Valid values: - - `"save"` - Save/create a replication configuration file - - `"list"` - List all replication configuration files in a directory - - `"get"` - Get the content of a replication configuration file + - `"docs"` - Fetch documentation on how to use replications + - `"parse"` - Parse the content of a replication configuration file (non-compile validation) - `"compile"` - Compile/validate a replication configuration - `"run"` - Execute a replication configuration - - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: - **For `action: "save"`**: - - `file_path` (string, required): File path to save the replication configuration - - `config_content` (string, required): YAML content of the replication configuration - - `working_dir` (string, optional): Working directory to change to before saving + - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: - **For `action: "list"`**: - - `directory` (string, optional): Directory to list files from (defaults to current working directory) - - `working_dir` (string, optional): Working directory to change to before listing + **For `action: "docs"`**: + - No parameters required (empty object) - **For `action: "get"`**: + **For `action: "parse"`**: - `file_path` (string, required): Path to the replication configuration file to read - `working_dir` (string, optional): Working directory to change to before reading @@ -436,20 +226,25 @@ tools: **Example Usage**: ``` - # Save a replication configuration + # Obtain replication Documentation + { + "action": "docs", + "input": {} + } + + # Parse a replication file { - "action": "save", + "action": "parse", "input": { - "file_path": "my_replication.yaml", - "config_content": "source: postgres://...\ntarget: s3://...\nstreams:\n my_table: {}" + "file_path": "path/to/my_replication.yaml", } } - - # List all replication files + + # Compile a replication file { - "action": "list", + "action": "compile", "input": { - "directory": "./replications" + "file_path": "path/to/my_replication.yaml", } } @@ -457,69 +252,57 @@ tools: { "action": "run", "input": { - "file_path": "my_replication.yaml", + "file_path": "path/to/my_replication.yaml", "select_streams": ["table1", "table2"], "mode": "incremental" } } ``` + + pipeline: | + **Main pipeline tool that provides comprehensive data pipeline management capabilities. This is a parent tool that orchestrates various pipeline operations through sub-actions.** - replication_save: | - Save a replication configuration to a specified file path. - - **Parameters**: - - `file_path` (string, required): The file path where to save the replication configuration (can be absolute or relative to current working directory). - - `config_content` (string, required): The YAML content of the replication configuration to validate and save. - - `working_dir` (string, optional): Working directory to change to before saving replication configuration. - - **Output (for User)**: A text message: "Replication configuration validated and saved to [absFilePath]". - - **Output (for Assistant)**: A JSON string with `type: "replication_save"`, `file_path` (absolute), `directory` (absolute path to dir of file), `working_dir` (if provided), and `success: true`. - - replication_list: | - List all YAML replication configuration files in a directory. - - **Parameters**: - - `directory` (string, optional): Directory to list replication files from (defaults to current working directory). - - `working_dir` (string, optional): Working directory to change to before listing replication files. - - **Output (for User)**: A CSV string listing `config_file_name` for each replication configuration found. The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string with `type: "replication_list"`, `directory` (absolute path searched), `working_dir` (if provided), `count` of files found, and a `files` array with detailed metadata. - - replication_get: | - Get the content of a replication configuration file. + **IMPORTANT**: To fully understand how to use pipelines, first call the `pipeline/docs` tool. **Parameters**: - - `file_path` (string, required): The file path of the replication configuration to get (can be absolute or relative to current working directory). - - `working_dir` (string, optional): Working directory to change to before getting replication configuration. - - **Output (for User)**: If parsing is successful, a CSV string listing streams with columns: `stream_name`, `object`, `mode`. If parsing fails, a text message: "Error parsing replication config: [error_message]". The assistant should display this as a pretty table if possible. - - **Output (for Assistant)**: A JSON string with `type: "replication_get"`, `file_path` (absolute), `working_dir` (if provided), and raw `content` (string). If parsing was successful, includes additional parsed metadata. - - replication_compile: | - Compile a replication configuration to validate and prepare it for execution. + - `action` (string, required): The pipeline action to perform. Valid values: + - `"docs"` - Fetch documentation on how to use pipelines + - `"parse"` - Parse and validate the content of a pipeline configuration file + - `"run"` - Execute a pipeline configuration + - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: - **Parameters**: - - `select_streams` (array, optional): Optional list of specific streams to compile (default: all streams). - - `working_dir` (string, optional): Working directory to change to before compiling replication configuration. + **For `action: "docs"`**: + - No parameters required (empty object) - **Output (for User)**: A text message: "Replication configuration '[absFilePath]' compiled successfully with [task_count] tasks". + **Output**: The output format depends on the action and audience: + - **For User**: Human-readable messages, CSV tables, or status updates + - **For Assistant**: JSON strings with detailed metadata and operation results - **Output (for Assistant)**: A JSON string with `type: "replication_compile"`, `file_path` (absolute), `working_dir` (if provided), `success: true`, `task_count`, `compiled` (boolean status), `streams_selected` (array), and a `tasks` array with detailed task metadata. + **Example Usage**: + ``` + # Obtain pipeline Documentation + { + "action": "docs", + "input": {} + } - replication_run: | - Run a replication configuration. - - **Parameters**: - - `range` (string, optional): Backfill range for source options. - - `mode` (string, optional): Mode override for replication (e.g. full-refresh, incremental). - - `env` (object, optional): Environment variables to set for the replication run. + # Parse a pipeline file + { + "action": "parse", + "input": { + "file_path": "path/to/my_pipeline.yaml", + } + } - **Output (for User)**: A simple text message: "Replication '[absFilePath]': SUCCESS" or "Replication '[absFilePath]': FAILED - [error_message]". + # Run a pipeline + { + "action": "run", + "input": { + "file_path": "path/to/my_pipeline.yaml" + } + } + ``` - **Output (for Assistant)**: A JSON string with `type: "replication_run"`, `file_path` (absolute), `success` (boolean), `streams_selected` (array), `working_dir` (if provided), `range` (if provided), `mode` (if provided), `env` variables (if provided), and captured `logs`. If an error occurred, includes error details. resources: "docs://API_SPEC": | @@ -535,7 +318,7 @@ prompts: inputs: spec_name: "Name for the API specification (will be used as filename)" spec_file_path: "Optional full file path for the spec file" # optional - connection_name: "Name for the API connection to create and test" + connection_name: "Name for the API connection to create (if needed) and test" api_docs_url: "URL to the API documentation website" endpoint_names: "Comma-separated list of endpoint names to include in the spec" additional_info: "Additional instructions or requirements for the API spec" # optional @@ -543,26 +326,32 @@ prompts: You are an expert at creating Sling API specifications. Your task is to create a complete, working API spec by following this workflow: ## Workflow Steps: - 1. **Get Documentation**: Use `api_spec_docs` to access the Sling API specification guide - 2. **Analyze API Documentation**: Navigate to the provided API docs URL using the browser to understand: + 1. **Get Documentation**: Call the tool api_spec/docs to obtain the latest Sling API specification guide + + 2. **Analyze API Documentation**: Navigate to the provided API docs URL using the browser to fully understand: - Authentication methods and requirements - Endpoint structures and parameters - Response formats and data structures - Pagination patterns - Rate limiting constraints - Determine what capabilities are available for incremental updates (Typically for a date/ID anchor/watermark, or range) in the filters - 3. **Create Specification**: Use `api_spec_save` to save the complete YAML spec in the specs directory - 4. **Create Connection**: Use `conns_set` to create a connection using the spec. When using the `conns_set` tools, the `properties` parameters must contain `type` (which is always "api") and `spec` (which is the local path of the spec file with prefix `file://`) keys. + + The assistant should first attempt to use a browser mcp tool to fetch the documentation, as that will yield better content. If unable to use the browser mcp tool, the assistant should next attempt to directly fetch from the URL links (without a browser). + + If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user for further instruction. Recommend the user to download and activate the `browsermcp` (https://browsermcp.io/) which will provide tooling to access the browser. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. + + 3. **Create Specification**: Now that the API is greatly understood, formulate and create a fully functional Sling API Spec in file path {spec_file_path}. If the assistant sees obstacles or issues in developing the spec, or does not have sufficient information, this is the time to alert the user with any concerns or questions for clarification. + + 4. **Create Connection**: Use tool connection/list to determine if connection `{connection_name}` exists. If it exists, proceed to the next step, we will use the existing connection to test our API Spec. If it does not exist, use the connection/set tool to create a connection using the spec. When using the connection/set tool, the `properties` parameters must contain `type` (which is always "api") and `spec` (which is the local path of `file://{spec_file_path}`) keys. If the API connection requires authentication credentials (such as token or client ID/Secret), they should be put under the `secrets` key as a nested object. - Furthermore, if the API connection requires authentication credentials, STOP and ask the user to open their env.yaml file (in folder ~/.sling/env.yaml), and to manually set the sensitive values. Assistant should have already ran `conns_set` with placeholder values (under the `secrets` key). When the user is done setting the creds, to mention so that the assistant can continue with testing. If Authentication fails, and the assistant believes there may be a error with the sensitive values, to STOP and mention so the user can correct the inputs. + Furthermore, if the API connection requires authentication credentials, STOP and ask the user to open their env.yaml file (in folder ~/.sling/env.yaml), and to manually set the sensitive values. Assistant should have already ran connection/set with placeholder values (under the `secrets` key). When the user is done setting the creds, to mention so that the assistant can continue with testing. If Authentication fails, and the assistant believes there may be a error with the sensitive values, to STOP and mention so the user can correct the inputs. - 5. **Test and Iterate**: Use `conn_test` with debug enabled to test the connection - - If test fails, analyze errors and modify the spec - - Repeat testing until all endpoints return data without errors + 5. **Test and Iterate**: Use api_spec/test with debug enabled to test the API connection + - If test fails, analyze errors and modify the spec accordingly + - Repeat testing until all endpoints return data without errors or determining that no progress can be made - Success means no errors and records are returned - - IMPORTANT: Use a built-in tool to directly edit the file (or use `api_spec_edit`), in order to edit the spec file (instead of rewriting the whole file with `api_spec_save`) ## Key Requirements: - Follow the Sling API specification format exactly @@ -574,9 +363,11 @@ prompts: - Add primary keys for deduplication when applicable - Use appropriate state management for incremental loading - When using functions, always use double quotes (`"`) for string literals in expressions, never single quotes (`'`). + - If having access to VS Code, recommend to the user to download the VS Code extension (share with user the url https://docs.slingdata.io/sling-cli/vscode). The extension will evaluate the YAML for any formatting errors. This will help the assistant with a feedback loop to better design the API Spec. ## Inputs Provided: - **API Spec Name**: {spec_name} + - **API Spec Path**: {spec_file_path} - **Connection Name**: {connection_name} - **API Documentation URL**: {api_docs_url} - **Endpoints to Include**: {endpoint_names} @@ -587,31 +378,40 @@ prompts: api_spec_add_endpoint: description: "Add a new endpoint to an existing Sling API specification by analyzing endpoint documentation and implementing proper configuration" inputs: - spec_name: "Name of the existing API specification" - spec_file_path: "Full file path to the existing spec file (must exist)" # optional + spec_file_path: "Full file path to the existing spec file (must exist)" endpoint_name: "Name of the new endpoint to add" endpoint_docs_url: "URL to the specific endpoint documentation" # optional additional_info: "Additional instructions for the endpoint implementation" # optional message: | - You are an expert at extending Sling API specifications. Your task is to add a new endpoint to an existing API spec by following this workflow: + You are an expert at extending Sling API specifications. Learn about the Sling API specifications here: https://raw.githubusercontent.com/slingdata-io/sling-cli/refs/heads/main/cmd/sling/resource/llm_API_SPEC.md + + Your task is to add a new endpoint to an existing API spec by following this workflow: ## Workflow Steps: - 1. **Get Documentation**: Use `api_spec_docs` to access the Sling API specification guide - 2. **Load Existing Spec**: Use `api_spec_load` to load the current specification content - 3. **Read Raw Spec**: Examine the existing spec structure and patterns - 4. **Analyze Endpoint Documentation**: Navigate to the endpoint docs URL (if provided) or the main API docs to understand: + + 1. **Get Documentation**: Call the tool `api_spec/docs` to obtain the latest Sling API specification guide + + 2. **Load Existing Spec**: Use `api_spec/parse` to load the current specification content. + + 3. **Read Raw Spec**: Examine the existing spec structure and patterns. If there were any parsing errors from the previous step, attempt to fix the structure of the file, and continue using `api_spec/parse` until there are no errors or determining that no progress can be made. + + 4. **Analyze Endpoint Documentation**: Navigate to the endpoint docs URL (if provided) or the main API docs, using the browser to fully understand: - Endpoint URL structure and parameters - HTTP method and headers required - Request/response format - Pagination requirements - Authentication needs - Rate limiting considerations - - When using functions, always use double quotes (`"`) for string literals in expressions, never single quotes (`'`). - 5. **Implement Endpoint**: Add the new endpoint configuration following existing patterns - 6. **Test Endpoint**: Use `conn_test` with debug enabled, specifying the endpoint name + + The assistant should first attempt to use a browser mcp tool to fetch the documentation, as that will yield better content. If unable to use the browser mcp tool, the assistant should next attempt to directly fetch from the URL links (without a browser). + + If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user for further instruction. Recommend the user to download and activate the `browsermcp` (https://browsermcp.io/) which will provide tooling to access the browser. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. + + 5. **Implement Endpoint**: Add the new endpoint configuration to the spec file following existing patterns. + + 6. **Test Endpoint**: Use tool `api_spec/test` with debug enabled, specifying the endpoint name - If test fails, analyze errors and modify the endpoint configuration - - IMPORTANT: Use a built-in tool to directly edit the file (or use `api_spec_edit`), in order to edit the spec file (instead of rewriting the whole file with `api_spec_save`) - - Repeat testing until the endpoint returns data without errors + - Repeat testing until the endpoint returns data without errors or determining that no progress can be made - Success means no errors and records are returned ## Key Requirements: @@ -622,9 +422,10 @@ prompts: - Include error handling rules - Add primary keys if applicable - Use consistent naming conventions + - When using functions, always use double quotes (`"`) for string literals in expressions, never single quotes (`'`). + - If having access to VS Code, recommend to the user to download the VS Code extension (share with user the url https://docs.slingdata.io/sling-cli/vscode). The extension will evaluate the YAML for any formatting errors. This will help the assistant with a feedback loop to better design the API Spec. ## Inputs Provided: - - **Spec Name**: {spec_name} - **Spec File Path**: {spec_file_path} - **New Endpoint Name**: {endpoint_name} - **Endpoint Documentation URL**: {endpoint_docs_url} @@ -635,7 +436,6 @@ prompts: api_spec_debug_endpoint: description: "Debug and fix issues with an existing endpoint in a Sling API specification by analyzing errors and adjusting configuration" inputs: - spec_name: "Name of the API specification containing the endpoint" spec_file_path: "Full file path to the spec file (must exist)" # optional endpoint_name: "Name of the endpoint to debug and fix" additional_info: "Additional context about the issues or specific areas to focus on" # optional @@ -643,28 +443,37 @@ prompts: You are an expert at debugging Sling API specifications. Your task is to identify and fix issues with an existing endpoint by following this workflow: ## Workflow Steps: - 1. **Get Documentation**: Use `api_spec_docs` to access the Sling API specification guide - 2. **Load Current Spec**: Use `api_spec_load` to examine the current specification - 3. **Read Raw Spec**: Analyze the endpoint configuration in detail - 4. **Analyze API Documentation**: Navigate to the API docs to verify: + 1. **Get Documentation**: Use `api_spec/docs` to access the Sling API specification guide + + 2. **Load Existing Spec**: Use `api_spec/parse` to load the current specification content. + + 3. **Read Raw Spec**: Examine the existing spec structure and patterns. If there were any parsing errors from the previous step, attempt to fix the structure of the file, and continue using `api_spec/parse` until there are no errors or determining that no progress can be made. + + 4. **Analyze API Documentation**: Navigate to the API docs using the browser to fully understand and verify: - Correct endpoint URL and parameters - Required headers and authentication - Expected request/response format - Pagination implementation - Rate limiting compliance - 5. **Test and Diagnose**: Use `conn_test` with debug enabled, specifying the endpoint name + + The assistant should first attempt to use a browser mcp tool to fetch the documentation, as that will yield better content. If unable to use the browser mcp tool, the assistant should next attempt to directly fetch from the URL links (without a browser). + + If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user for further instruction. Recommend the user to download and activate the `browsermcp` (https://browsermcp.io/) which will provide tooling to access the browser. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. + + 5. **Test and Diagnose**: Use `api_spec/test` with debug enabled, specifying the endpoint name - Examine error messages and response details - Identify configuration mismatches - Check authentication issues - Verify data extraction paths + 6. **Fix and Iterate**: Modify the endpoint specification based on findings - - IMPORTANT: Use a built-in tool to directly edit the file (or use `api_spec_edit`), in order to edit the spec file (instead of rewriting the whole file with `api_spec_save`) - Update URLs, parameters, or headers - Fix JMESPath expressions for data extraction - When using functions, always use double quotes (`"`) for string literals in expressions, never single quotes (`'`). - Adjust pagination logic - Correct authentication setup - - Repeat testing until successful + - Repeat testing until successful or determining that no progress can be made + - If having access to VS Code, recommend to the user to download the VS Code extension (share with user the url https://docs.slingdata.io/sling-cli/vscode). The extension will evaluate the YAML for any formatting errors. This will help the assistant with a feedback loop to better design the API Spec. ## Common Issues to Check: - **Authentication**: Token format, header names, authentication type @@ -683,7 +492,6 @@ prompts: - Proper error handling ## Inputs Provided: - - **Spec Name**: {spec_name} - **Spec File Path**: {spec_file_path} - **Endpoint Name**: {endpoint_name} - **Issue Context**: {additional_info} From 7fdba6f864035c2c976ffe97e59b780dd2fb96fc Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Fri, 22 Aug 2025 10:05:46 -0300 Subject: [PATCH 10/37] fix: update GetNativeType to use BIGNUMERIC for BigQuery when scale > 9 or precision > 38 --- core/dbio/iop/datatype.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/dbio/iop/datatype.go b/core/dbio/iop/datatype.go index e8f6be8f7..f207438e9 100755 --- a/core/dbio/iop/datatype.go +++ b/core/dbio/iop/datatype.go @@ -1308,6 +1308,13 @@ func (col *Column) GetNativeType(t dbio.Type, ct ColumnTyping) (nativeType strin "(,)", fmt.Sprintf("(%d,%d)", precision, scale), ) + + // BigQuery: use BIGNUMERIC if scale > 9 or precision > 38 + if t == dbio.TypeDbBigQuery && strings.EqualFold(nativeType, "numeric") && + (scale > 9 || precision > 38) { + nativeType = "bignumeric" + } + } else if col.Type.IsJSON() { if ct.JSON != nil { ct.JSON.Apply(col) From 34fab37ca0bc6fd66d7876b22667b39cbd9067fa Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Fri, 22 Aug 2025 14:34:20 -0300 Subject: [PATCH 11/37] fix: add InliningLimit to DuckLakeConn and initialize from properties --- core/dbio/database/database_ducklake.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/dbio/database/database_ducklake.go b/core/dbio/database/database_ducklake.go index ecb50efc3..ee31b073a 100644 --- a/core/dbio/database/database_ducklake.go +++ b/core/dbio/database/database_ducklake.go @@ -29,6 +29,7 @@ type DuckLakeConn struct { DataPath string // path to data files (local or cloud storage) Database string // the database name to attached Encrypted bool // whether data is written using Parquet encryption + InliningLimit int // where to enable data inlining when attaching } // Init initiates the object @@ -37,6 +38,7 @@ func (conn *DuckLakeConn) Init() error { // Extract data path from properties conn.CatalogType = conn.GetProp("catalog_type") conn.CatalogSchema = conn.GetProp("catalog_schema") + conn.InliningLimit = cast.ToInt(conn.GetProp("data_inlining_limit")) conn.CatalogConnStr = conn.GetProp("catalog_conn_string") conn.DataPath = conn.GetProp("data_path") conn.Database = "ducklake" // for hard-coded '__ducklake_metadata_ducklake' @@ -164,6 +166,9 @@ func (conn *DuckLakeConn) buildAttachSQL() string { if conn.CatalogSchema != "" { metaParts = append(metaParts, g.F("META_SCHEMA '%s'", conn.CatalogSchema)) } + if conn.InliningLimit > 0 { + metaParts = append(metaParts, g.F("DATA_INLINING_ROW_LIMIT %d", conn.InliningLimit)) + } if conn.Encrypted { metaParts = append(metaParts, "ENCRYPTED") } From 5a9c297f4bce231c9af38708799939a51ee52304 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Fri, 22 Aug 2025 17:13:38 -0300 Subject: [PATCH 12/37] fix: clean up Clickhouse subquery by removing FORMAT CSVWithNames --- core/dbio/database/database.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 9872be0d8..afc0bd1aa 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -1512,6 +1512,11 @@ func (conn *BaseConn) GetSQLColumns(table Table) (columns iop.Columns, err error return conn.GetColumns(table.FullName()) } + // clean up for clickhouse subquery + if conn.Type == dbio.TypeDbClickhouse { + table.SQL = strings.ReplaceAll(table.SQL, "FORMAT CSVWithNames", "") + } + limitSQL := table.Select(SelectOptions{Limit: 1}) if table.IsProcedural() { limitSQL = table.Raw // don't wrap in limit From ae2ff19bb972a8752eb24b273c087a00a7dc007d Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Fri, 22 Aug 2025 18:18:48 -0300 Subject: [PATCH 13/37] fix: remove FORMAT CSVWithNames from Clickhouse subquery and add export stream format option --- core/dbio/database/database.go | 5 ----- core/dbio/database/database_clickhouse.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index afc0bd1aa..9872be0d8 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -1512,11 +1512,6 @@ func (conn *BaseConn) GetSQLColumns(table Table) (columns iop.Columns, err error return conn.GetColumns(table.FullName()) } - // clean up for clickhouse subquery - if conn.Type == dbio.TypeDbClickhouse { - table.SQL = strings.ReplaceAll(table.SQL, "FORMAT CSVWithNames", "") - } - limitSQL := table.Select(SelectOptions{Limit: 1}) if table.IsProcedural() { limitSQL = table.Raw // don't wrap in limit diff --git a/core/dbio/database/database_clickhouse.go b/core/dbio/database/database_clickhouse.go index 43e8a3daa..1bbb71dae 100755 --- a/core/dbio/database/database_clickhouse.go +++ b/core/dbio/database/database_clickhouse.go @@ -502,6 +502,18 @@ func (conn *ClickhouseConn) BulkImportStream(tableFName string, ds *iop.Datastre return count, nil } +// BulkExportStream streams the rows in bulk +func (conn *ClickhouseConn) BulkExportStream(table Table) (ds *iop.Datastream, err error) { + // append SELECT FORMAT if specified (such as CSVWithNames) + // low-memory streaming! + sql := table.Select() + if val := conn.GetProp("export_stream_format"); val != "" { + sql = sql + " FORMAT " + val + } + + return conn.BaseConn.StreamRows(sql, g.M("columns", table.Columns)) +} + // GenerateInsertStatement returns the proper INSERT statement func (conn *ClickhouseConn) GenerateInsertStatement(tableName string, cols iop.Columns, numRows int) string { values := make([]string, len(cols)) From d4bf06099955ef9e286d4993a36a3f9aac7069b4 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Fri, 22 Aug 2025 18:19:13 -0300 Subject: [PATCH 14/37] fix: allow custom delimiter for CSV output in WriteCsv method --- core/dbio/iop/dataset.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/dbio/iop/dataset.go b/core/dbio/iop/dataset.go index dc69c295c..ab87f1088 100644 --- a/core/dbio/iop/dataset.go +++ b/core/dbio/iop/dataset.go @@ -204,6 +204,10 @@ func (data *Dataset) PrettyTable(fields ...string) (output string) { // WriteCsv writes to a writer func (data *Dataset) WriteCsv(dest io.Writer) (tbw int, err error) { w := csv.NewWriter(dest) + if val := data.Sp.Config.Delimiter; val != "" { + w.Comma = val + } + defer w.Flush() tbw, err = w.Write(data.GetFields()) From ba389519b8758a791da07c49030f530981fa116e Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 23 Aug 2025 07:18:56 -0300 Subject: [PATCH 15/37] fix: update defaultCsvConfig to set FileMaxRows from properties if specified --- core/dbio/database/database_duckdb.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/dbio/database/database_duckdb.go b/core/dbio/database/database_duckdb.go index 18711fdc9..790852cbf 100644 --- a/core/dbio/database/database_duckdb.go +++ b/core/dbio/database/database_duckdb.go @@ -292,7 +292,11 @@ func (conn *DuckDbConn) importViaHTTP(tableFName string, df *iop.Dataflow, forma } func (conn *DuckDbConn) defaultCsvConfig() (config iop.StreamConfig) { - return conn.duck.DefaultCsvConfig() + config = conn.duck.DefaultCsvConfig() + if val := conn.GetProp("file_max_rows"); val != "" { + config.FileMaxRows = cast.ToInt64(val) + } + return config } func (conn *DuckDbConn) generateCsvColumns(columns iop.Columns) (colStr string) { @@ -417,11 +421,11 @@ func MakeDuckDbSecretProps(conn Connection, secretType iop.DuckDbSecretType) (se "provider": "PROVIDER", }) // If user supplied CHAIN but no PROVIDER, default to credential_chain - if _, hasChain := secretProps["CHAIN"]; hasChain { - if _, hasProv := secretProps["PROVIDER"]; !hasProv { - secretProps["PROVIDER"] = "credential_chain" - } - } + if _, hasChain := secretProps["CHAIN"]; hasChain { + if _, hasProv := secretProps["PROVIDER"]; !hasProv { + secretProps["PROVIDER"] = "credential_chain" + } + } case iop.DuckDbSecretTypeAzure: fillSecretProps(map[string]string{ "azure_connection_string": "CONNECTION_STRING", From bb8a290f231dc8190b84f82c45eb08d77d5a00e3 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 23 Aug 2025 09:51:24 -0300 Subject: [PATCH 16/37] fix: update TimeType mapping to use TimeFieldType in BigQuery schema --- core/dbio/database/database_bigquery.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/dbio/database/database_bigquery.go b/core/dbio/database/database_bigquery.go index 62f91701b..d41a400d5 100755 --- a/core/dbio/database/database_bigquery.go +++ b/core/dbio/database/database_bigquery.go @@ -529,7 +529,8 @@ func getBqSchema(columns iop.Columns) (schema bigquery.Schema) { // https://stackoverflow.com/questions/55904464/big-query-does-now-cast-automatically-long-decimal-values-to-numeric-when-runni iop.DecimalType: bigquery.NumericFieldType, // "decimal": bigquery.FloatFieldType, - iop.TimeType: bigquery.StringFieldType, + iop.TimeType: bigquery.TimeFieldType, + iop.TimezType: bigquery.TimeFieldType, iop.TimestampType: bigquery.TimestampFieldType, iop.TimestampzType: bigquery.TimestampFieldType, } From 5d4909752898f511f2d1407bbd37186057ea05ca Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 23 Aug 2025 14:57:23 -0300 Subject: [PATCH 17/37] fix: enhance debug logging for DuckDb direct stream to include stream format --- core/dbio/iop/duckdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index 5c0bcd56e..3031595a1 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -1201,7 +1201,7 @@ func (duck *DuckDb) DataflowToHttpStream(df *Dataflow, sc StreamConfig) (streamP // start server in background, wait for it to start importContext.Wg.Read.Add() go func() { - g.Debug("started %s for duckdb direct stream", httpURL) + g.Debug("started %s for duckdb direct %s stream", httpURL, sc.Format) importContext.Wg.Read.Done() if err := server.Start(g.F("localhost:%d", port)); err != http.ErrServerClosed { g.Error(err, "duckdb import http server error") From 912e97bd4a6a9af4d0b0cdeb97315f7d5715f68a Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 23 Aug 2025 19:33:00 -0300 Subject: [PATCH 18/37] fix: update BulkImportFlow to use FileTypeArrow for default and arrow_http cases --- core/dbio/database/database_duckdb_unix.go | 2 +- core/dbio/database/database_duckdb_windows.go | 4 +++- core/dbio/filesys/fs.go | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/dbio/database/database_duckdb_unix.go b/core/dbio/database/database_duckdb_unix.go index 56f5c60a4..8aab5e2dd 100644 --- a/core/dbio/database/database_duckdb_unix.go +++ b/core/dbio/database/database_duckdb_unix.go @@ -28,7 +28,7 @@ func (conn *DuckDbConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (cou case "arrow_http": return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) default: - return conn.importViaHTTP(tableFName, df, dbio.FileTypeCsv) + return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) } } diff --git a/core/dbio/database/database_duckdb_windows.go b/core/dbio/database/database_duckdb_windows.go index 52324dba1..e8f50dba8 100644 --- a/core/dbio/database/database_duckdb_windows.go +++ b/core/dbio/database/database_duckdb_windows.go @@ -13,7 +13,9 @@ func (conn *DuckDbConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (cou return conn.importViaTempCSVs(tableFName, df) case "csv_http": return conn.importViaHTTP(tableFName, df, dbio.FileTypeCsv) + case "arrow_http": + return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) default: - return conn.importViaHTTP(tableFName, df, dbio.FileTypeCsv) + return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) } } diff --git a/core/dbio/filesys/fs.go b/core/dbio/filesys/fs.go index 150e1cfad..a4434ef0f 100755 --- a/core/dbio/filesys/fs.go +++ b/core/dbio/filesys/fs.go @@ -1289,6 +1289,7 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw // merge into single stream to push into duckdb duckSc := duck.DefaultCsvConfig() duckSc.FileMaxRows = sc.FileMaxRows + duckSc.Format = dbio.FileTypeArrow streamPartChn, err := duck.DataflowToHttpStream(df, duckSc) if err != nil { return bw, g.Error(err) From 2f0334470dbcd8baefca34e9a868a53c6296e90f Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 23 Aug 2025 19:40:35 -0300 Subject: [PATCH 19/37] fix: update flarco/g dependency to v0.1.156 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4bbc7d8a7..478c91971 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/exasol/exasol-driver-go v1.0.14 github.com/fatih/color v1.18.0 github.com/flarco/bigquery v0.0.9 - github.com/flarco/g v0.1.155 + github.com/flarco/g v0.1.156 github.com/getsentry/sentry-go v0.27.0 github.com/go-sql-driver/mysql v1.9.1 github.com/gobwas/glob v0.2.3 From 6b153e9fd4c3b3009c48893809ad999657711a00 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 23 Aug 2025 19:58:32 -0300 Subject: [PATCH 20/37] fix: update DuckDB extension handling for Arrow support --- core/dbio/database/database_duckdb.go | 2 +- core/dbio/filesys/fs.go | 1 + core/dbio/iop/duckdb.go | 35 +++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/core/dbio/database/database_duckdb.go b/core/dbio/database/database_duckdb.go index 790852cbf..bfb6a3888 100644 --- a/core/dbio/database/database_duckdb.go +++ b/core/dbio/database/database_duckdb.go @@ -133,7 +133,7 @@ func (conn *DuckDbConn) Connect(timeOut ...int) (err error) { // add extensions if conn.GetProp("copy_method") == "arrow_http" { - conn.duck.AddExtension("arrow from community") + conn.duck.AddExtension("arrow") } return nil diff --git a/core/dbio/filesys/fs.go b/core/dbio/filesys/fs.go index a4434ef0f..ebe96d3b6 100755 --- a/core/dbio/filesys/fs.go +++ b/core/dbio/filesys/fs.go @@ -1268,6 +1268,7 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw props := g.MapToKVArr(fs.Props()) duck := iop.NewDuckDb(context.Background(), props...) + duck.AddExtension("arrow from community") duckURI := duck.PrepareFsSecretAndURI(uri) sp := iop.NewStreamProcessor() diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index 3031595a1..bfac738a9 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -95,6 +95,33 @@ func (duck *DuckDb) AddExtension(extension string) { } } +// CheckExtension checks if an extension is installed in DuckDB +func (duck *DuckDb) CheckExtension(extension string) (bool, error) { + if !cast.ToBool(duck.GetProp("connected")) { + if err := duck.Open(); err != nil { + return false, g.Error(err, "Could not open DuckDB connection") + } + } + + duck.Query("select 1 as a" + env.NoDebugKey) // installs pending extensions + + sql := fmt.Sprintf("SELECT extension_name, loaded, installed FROM duckdb_extensions() WHERE extension_name = '%s'", extension) + data, err := duck.Query(sql + env.NoDebugKey) + if err != nil { + return false, g.Error(err, "could not check extension status") + } + + // Check if extension exists in the results and is installed + for _, row := range data.Rows { + if len(row) >= 3 { + installed := cast.ToBool(row[2]) // installed column + return installed, nil + } + } + + return false, nil // extension not found or not installed +} + type DuckDbSecret struct { Type DuckDbSecretType `json:"type"` Name string `json:"name"` @@ -1172,8 +1199,12 @@ func (duck *DuckDb) DataflowToHttpStream(df *Dataflow, sc StreamConfig) (streamP contentType := "text/csv" format := dbio.FileTypeCsv // default to CSV if sc.Format == dbio.FileTypeArrow { - contentType = "application/vnd.apache.arrow.stream" - format = dbio.FileTypeArrow + if installed, _ := duck.CheckExtension("arrow"); installed { + contentType = "application/vnd.apache.arrow.stream" + format = dbio.FileTypeArrow + } else { + g.Warn("duckdb extension arrow is not installed") + } } // create http server to serve data From 191f8b5e11aa83b824a817afd525163c6c56d039 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 23 Aug 2025 20:34:51 -0300 Subject: [PATCH 21/37] fix: simplify DuckDB extension name for clarity --- core/dbio/filesys/fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbio/filesys/fs.go b/core/dbio/filesys/fs.go index ebe96d3b6..6778b2590 100755 --- a/core/dbio/filesys/fs.go +++ b/core/dbio/filesys/fs.go @@ -1268,7 +1268,7 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw props := g.MapToKVArr(fs.Props()) duck := iop.NewDuckDb(context.Background(), props...) - duck.AddExtension("arrow from community") + duck.AddExtension("arrow") duckURI := duck.PrepareFsSecretAndURI(uri) sp := iop.NewStreamProcessor() From 69fb2b9fa9d2c8ea188a9d679879e52c99b93910 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 07:55:11 -0300 Subject: [PATCH 22/37] fix: add retry logic for EnsureBinDuckDB version check on "text file busy" error --- core/dbio/iop/duckdb.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index bfac738a9..4f9589e12 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -1035,8 +1035,14 @@ func EnsureBinDuckDB(version string) (binPath string, err error) { checkVersion := func() (bool, error) { + retry: out, err := exec.Command(binPath, "-version").Output() if err != nil { + if strings.Contains(err.Error(), "text file busy") { + g.Warn("could not get version for duckdb (%s), retrying...", err.Error()) + time.Sleep(1 * time.Second) + goto retry + } return false, g.Error(err, "could not get version for duckdb") } From 07f14c9dd24d2b0b5df050ffa84210cb13fab7e2 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 07:58:58 -0300 Subject: [PATCH 23/37] fix: add retry logic for handling "text file busy" errors in Oracle, SQLite, and SQL Server import functions --- core/dbio/database/database_oracle.go | 7 +++++++ core/dbio/database/database_sqlite.go | 6 ++++++ core/dbio/database/database_sqlserver.go | 7 +++++++ 3 files changed, 20 insertions(+) diff --git a/core/dbio/database/database_oracle.go b/core/dbio/database/database_oracle.go index 5520aedab..54befb01b 100755 --- a/core/dbio/database/database_oracle.go +++ b/core/dbio/database/database_oracle.go @@ -345,6 +345,7 @@ func (conn *OracleConn) SQLLoad(tableFName string, ds *iop.Datastream) (count ui defer func() { env.RemoveLocalTempFile(logPath) }() } +retry: proc := exec.Command( conn.sqlldrPath(), "'"+credHost+"'", @@ -373,6 +374,12 @@ func (conn *OracleConn) SQLLoad(tableFName string, ds *iop.Datastream) (count ui defer func() { env.RemoveLocalTempFile(ctlPath) }() if err != nil { + if strings.Contains(err.Error(), "text file busy") { + g.Warn("could not start sqlldr (%s), retrying...", err.Error()) + time.Sleep(1 * time.Second) + goto retry + } + err = g.Error( err, fmt.Sprintf( diff --git a/core/dbio/database/database_sqlite.go b/core/dbio/database/database_sqlite.go index 53e2e6e9c..8a05cfeb8 100644 --- a/core/dbio/database/database_sqlite.go +++ b/core/dbio/database/database_sqlite.go @@ -148,6 +148,7 @@ func (conn *SQLiteConn) BulkImportStream(tableFName string, ds *iop.Datastream) } } + retry: cmd := exec.Command(bin) sameCols := g.Marshal(ds.Columns.Names(true, true)) == g.Marshal(columns.Names(true, true)) @@ -236,6 +237,11 @@ func (conn *SQLiteConn) BulkImportStream(tableFName string, ds *iop.Datastream) out, err := cmd.Output() stderrVal := stderr.String() if err != nil { + if strings.Contains(err.Error(), "text file busy") { + g.Warn("could not start sqlite (%s), retrying...", err.Error()) + time.Sleep(1 * time.Second) + goto retry + } return 0, g.Error(err, "could not ingest csv file: %s\n%s", string(out), stderrVal) } } diff --git a/core/dbio/database/database_sqlserver.go b/core/dbio/database/database_sqlserver.go index 097047181..2174bb440 100755 --- a/core/dbio/database/database_sqlserver.go +++ b/core/dbio/database/database_sqlserver.go @@ -600,6 +600,7 @@ func (conn *MsSQLServerConn) BcpImportFile(tableFName, filePath string) (count u bcpArgs = append(bcpArgs, bcpExtraParts...) } +retry: proc := exec.Command(conn.bcpPath(), bcpArgs...) proc.Stderr = &stderr proc.Stdout = &stdout @@ -636,6 +637,12 @@ func (conn *MsSQLServerConn) BcpImportFile(tableFName, filePath string) (count u } if err != nil { + if strings.Contains(err.Error(), "text file busy") { + g.Warn("could not start bcp (%s), retrying...", err.Error()) + time.Sleep(1 * time.Second) + goto retry + } + errOut := stderr.String() if errPath != "/dev/stderr" { errOutB, _ := os.ReadFile(errPath) From 2cb515510e40d3b2ce1a2d73165d76eb413163e3 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 16:18:45 -0300 Subject: [PATCH 24/37] fix: update type mappings --- .../r.48.bigquery_time_datatype.yaml | 63 +++++++++++++++++++ cmd/sling/tests/suite.cli.yaml | 24 +++++-- core/dbio/database/database_bigquery.go | 4 +- core/dbio/iop/stream_processor.go | 18 +++--- .../templates/types_general_to_native.tsv | 4 +- 5 files changed, 94 insertions(+), 19 deletions(-) create mode 100644 cmd/sling/tests/replications/r.48.bigquery_time_datatype.yaml diff --git a/cmd/sling/tests/replications/r.48.bigquery_time_datatype.yaml b/cmd/sling/tests/replications/r.48.bigquery_time_datatype.yaml new file mode 100644 index 000000000..540615d38 --- /dev/null +++ b/cmd/sling/tests/replications/r.48.bigquery_time_datatype.yaml @@ -0,0 +1,63 @@ +source: POSTGRES +target: BIGQUERY + +defaults: + mode: truncate + +hooks: + start: + - type: query + connection: '{source.name}' + query: | + DROP TABLE IF EXISTS public.test_time_datatype; + CREATE TABLE public.test_time_datatype ( + id INTEGER, + time_col TIME, + time_tz_col TIME WITH TIME ZONE, + time_precision_col TIME(3) + ); + INSERT INTO public.test_time_datatype VALUES + (1, '08:30:45', '08:31:45+02:00', '08:30:45.123'), + (2, '12:15:30', '12:11:30-05:00', '12:15:30.456'), + (3, '23:59:59', '23:51:59+00:00', '23:59:59.789'); + + end: + - type: query + connection: '{target.name}' + query: | + SELECT + column_name, + data_type + FROM `public`.INFORMATION_SCHEMA.COLUMNS + WHERE table_name = 'test_time_datatype2' + ORDER BY ordinal_position + into: column_info + + - type: log + message: | + Target table column info: + { pretty_table(store.column_info) } + + - type: query + connection: '{target.name}' + query: SELECT * FROM public.test_time_datatype2 ORDER BY id + into: result_data + + - type: log + message: | + Target table data: + { pretty_table(store.result_data) } + + # Clean up + - type: query + connection: '{source.name}' + query: DROP TABLE IF EXISTS public.test_time_datatype + + - type: query + connection: '{target.name}' + query: DROP TABLE IF EXISTS public.test_time_datatype2 + +streams: + public.test_time_datatype: + object: public.test_time_datatype2 + mode: full-refresh \ No newline at end of file diff --git a/cmd/sling/tests/suite.cli.yaml b/cmd/sling/tests/suite.cli.yaml index 8f76633cb..60f699fc7 100644 --- a/cmd/sling/tests/suite.cli.yaml +++ b/cmd/sling/tests/suite.cli.yaml @@ -788,13 +788,29 @@ - 'execution succeeded' - id: 104 - name: Test PostgreSQL TIME datatype preservation - run: sling run -d -r cmd/sling/tests/replications/r.46.postgres_time_datatype.yaml - streams: 1 - rows: 3 + name: Test PostgreSQL & BigQury TIME datatype preservation + run: | + sling run -d -r cmd/sling/tests/replications/r.46.postgres_time_datatype.yaml + sling run -d -r cmd/sling/tests/replications/r.48.bigquery_time_datatype.yaml output_contains: - '| time without time zone |' - '| time with time zone |' + - | # pg output + +----+----------+--------------------+----------------+ + | ID | TIME_COL | TIME_PRECISION_COL | TIME_TZ_COL | + +----+----------+--------------------+----------------+ + | 1 | 08:30:45 | 08:30:45.123 | 08:30:45+02:00 | + | 2 | 12:15:30 | 12:15:30.456 | 12:15:30-05:00 | + | 3 | 23:59:59 | 23:59:59.789 | 23:59:59+00:00 | + +----+----------+--------------------+----------------+ + - | # bq output + +----+----------+--------------------+----------------+ + | ID | TIME_COL | TIME_PRECISION_COL | TIME_TZ_COL | + +----+----------+--------------------+----------------+ + | 1 | 08:30:45 | 08:30:45.123 | 08:31:45+02:00 | + | 2 | 12:15:30 | 12:15:30.456 | 12:11:30-05:00 | + | 3 | 23:59:59 | 23:59:59.789 | 23:51:59+00:00 | + +----+----------+--------------------+----------------+ - id: 105 name: Test MSSQL uniqueidentifier datatype preservation diff --git a/core/dbio/database/database_bigquery.go b/core/dbio/database/database_bigquery.go index d41a400d5..eea77d15e 100755 --- a/core/dbio/database/database_bigquery.go +++ b/core/dbio/database/database_bigquery.go @@ -529,8 +529,8 @@ func getBqSchema(columns iop.Columns) (schema bigquery.Schema) { // https://stackoverflow.com/questions/55904464/big-query-does-now-cast-automatically-long-decimal-values-to-numeric-when-runni iop.DecimalType: bigquery.NumericFieldType, // "decimal": bigquery.FloatFieldType, - iop.TimeType: bigquery.TimeFieldType, - iop.TimezType: bigquery.TimeFieldType, + iop.TimeType: bigquery.StringFieldType, + iop.TimezType: bigquery.StringFieldType, iop.TimestampType: bigquery.TimestampFieldType, iop.TimestampzType: bigquery.TimestampFieldType, } diff --git a/core/dbio/iop/stream_processor.go b/core/dbio/iop/stream_processor.go index 150afce82..720731105 100644 --- a/core/dbio/iop/stream_processor.go +++ b/core/dbio/iop/stream_processor.go @@ -900,17 +900,13 @@ func (sp *StreamProcessor) CastVal(i int, val any, col *Column) any { case col.Type.IsTime(): dVal, err := sp.CastToTime(val) if err != nil { - sp.ds.ChangeColumn(i, StringType) - cs.StringCnt++ - sVal = cast.ToString(val) - sp.rowChecksum[i] = uint64(len(sVal)) - nVal = sVal - } - - if col.Type == TimezType { - sVal = dVal.Format("15:04:05.999999-07:00") - } else if isUTC(&dVal) { - sVal = dVal.Format("15:04:05.999999") + sVal = sp.CastToString(val) + } else { + if col.Type == TimezType { + sVal = dVal.Format("15:04:05.999999-07:00") + } else if isUTC(&dVal) { + sVal = dVal.Format("15:04:05.999999") + } } nVal = sVal sp.rowChecksum[i] = uint64(len(sVal)) diff --git a/core/dbio/templates/types_general_to_native.tsv b/core/dbio/templates/types_general_to_native.tsv index 103ebb2dd..df5990ba3 100755 --- a/core/dbio/templates/types_general_to_native.tsv +++ b/core/dbio/templates/types_general_to_native.tsv @@ -13,6 +13,6 @@ text clob text mediumtext mediumtext nvarchar(max) nvarchar(max) nvarchar(max) v timestamp timestamp(9) timestamp datetime(6) datetime(6) datetime2 datetime2 datetime2 timestamp timestamp_ntz text text timestamp Nullable(DateTime64(6)) timestamp timestamp timestamp datetime timestamp nullable(datetime64(6)) timestamp timestamp_ntz timestamp timestampz timestamp(9) with time zone timestamptz datetime(6) datetime(6) datetimeoffset datetimeoffset datetimeoffset timestamptz timestamp_tz text text timestamp Nullable(DateTime64(6)) timestamptz timestamptz timestamptz datetime timestamp with time zone nullable(datetime64(6)) timestamp timestamp timestamp with local time zone float number double precision double double float float float double precision float real real float64 Nullable(Float64) double double double double double nullable(float64) double double double precision -time varchar(22) time(6) time(6) time(6) time(7) time(7) time(7) varchar(22) time text text time varchar(22) time time time varchar(22) time(12) varchar(22) varchar(22) varchar(22) varchar(22) -timez varchar(22) time with time zone time(6) time(6) time(7) time(7) time(7) varchar(22) time text text time varchar(22) time time time varchar(22) time with time zone varchar(22) varchar(22) varchar(22) varchar(22) +time varchar() time(6) varchar() varchar() varchar() varchar() varchar() varchar(65535) varchar text text string Nullable(String) time time time varchar() varchar nullable(string) string string varchar(100) +timez varchar() time with time zone varchar() varchar() varchar() varchar() varchar() varchar(65535) varchar text text string Nullable(String) time time time varchar() varchar nullable(string) string string varchar(100) uuid varchar(36) uuid varchar(36) varchar(36) uniqueidentifier uniqueidentifier uniqueidentifier varchar(36) varchar(36) text text string Nullable(UUID) uuid uuid uuid varchar(36) uuid nullable(string) string string varchar(36) From b151908c7e427f1e91e4fbcc52a1dae4a6633f6d Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 16:18:55 -0300 Subject: [PATCH 25/37] fix: add time package import for improved time handling --- core/dbio/database/database_sqlserver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbio/database/database_sqlserver.go b/core/dbio/database/database_sqlserver.go index 2174bb440..582b871b4 100755 --- a/core/dbio/database/database_sqlserver.go +++ b/core/dbio/database/database_sqlserver.go @@ -9,6 +9,7 @@ import ( "regexp" "runtime" "strings" + "time" "github.com/samber/lo" "github.com/slingdata-io/sling-cli/core/dbio" From a5dad0e84d4e7610185a1ab5c6ef8cdaf74a1ce0 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 16:21:51 -0300 Subject: [PATCH 26/37] feat: add TODO for Arrows output in StreamContext function --- core/dbio/iop/duckdb.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index 4f9589e12..b12f0dc51 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -798,6 +798,9 @@ func (duck *DuckDb) StreamContext(ctx context.Context, sql string, options ...ma duck.Context.Unlock() // release lock }) + // TODO: Add Arrows output + // COPY (SELECT extension_name, loaded, installed FROM duckdb_extensions()) TO '/dev/stdout' (FORMAT ARROWS, BATCH_SIZE 100); + err = ds.ConsumeCsvReader(dq.reader) if err != nil { ds.Close() From c7dd450bb8ea570c1baab4c7392641bc14f38dca Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 16:22:07 -0300 Subject: [PATCH 27/37] fix: update connection tool documentation and examples for clarity and consistency --- cmd/sling/resource/mcp.yaml | 234 +++++++++++++++++++++++++++++++----- 1 file changed, 202 insertions(+), 32 deletions(-) diff --git a/cmd/sling/resource/mcp.yaml b/cmd/sling/resource/mcp.yaml index 64b2bfc9d..40b1572cb 100644 --- a/cmd/sling/resource/mcp.yaml +++ b/cmd/sling/resource/mcp.yaml @@ -2,22 +2,15 @@ tools: connection: | **Main connection tool that provides comprehensive connection management capabilities. This is a parent tool that orchestrates various connection operations through sub-actions.** - **IMPORTANT**: To fully understand how to use connections, first call the `connection/docs` tool. + **IMPORTANT**: Call the `connection/docs` tool to fully understand how to use connections tool. **Parameters**: - `action` (string, required): The connection action to perform. Valid values: - - `"docs"` - Fetch documentation on how to use connections + - `"docs"` - Fetch documentation on how to use connections. Call this tool first before interacting with the `connection` tools. - `"list"` - List all available connections - `"discover"` - Discover tables/files in a connection - `"test"` - Test an existing connection - `"set"` - Create or update a connection - - `"database_get_schemata"` - Get database schemata (schemas, tables, columns) - - `"database_get_schemas"` - Get list of schema names - - `"database_query"` - Execute SQL queries (read-only) - - `"database_get_columns"` - Get column metadata for a table - - `"file_list"` - List files in a file system connection - - `"file_copy"` - Copy files between connections - - `"file_inspect"` - Inspect file/directory metadata - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: **For `action: "list"`**: @@ -47,16 +40,64 @@ tools: - Before setting a new connection, check if the connection already exists with the `list` action. If a connection exists with the same name, get permission from the user before overwriting the existing connection. - If the connection is set, the user will manually need to put the sensitive credentials in the env.yaml file. Make sure to mention this to the user, along with the file path of the `env.yaml` file. - **For `action: "database_get_schemata"`**: + **Example Usage**: + ``` + # Fetch documentation + { + "action": "docs", + "input": {} + } + + # List all connections + { + "action": "list", + "input": {} + } + + # Test a connection + { + "action": "test", + "input": { + "connection": "MY_PG", + "debug": true + } + } + + # Discover tables in a connection + { + "action": "discover", + "input": { + "connection": "MY_PG", + "pattern": "public.*", + "recursive": false, + "columns": false + } + } + ``` + + database: | + **Main database tool that provides database interaction capabilities. This is a parent tool that orchestrates various database operations through sub-actions.** + + **IMPORTANT**: To fully understand how to use the `database` tool, first call the `database/docs` tool. + + **Parameters**: + - `action` (string, required): The database action to perform. Valid values: + - `"get_schemata"` - Get database schemata (schemas, tables, columns) + - `"get_schemas"` - Get list of schema names + - `"query"` - Execute SQL queries (read-only) + - `"get_columns"` - Get column metadata for a table + - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: + + **For `action: "get_schemata"`**: - `connection` (string, required): The name of the connection to get schemata from - `level` (string, required): The level of schemata to retrieve: 'schema', 'table', or 'column' - `schema_name` (string, optional): The specific schema name to inspect. Required for 'table' and 'column' levels if not fetching all schemas - `table_names` (array, optional): Optional list of table names to focus on. Relevant for 'table' and 'column' levels - **For `action: "database_get_schemas"`**: + **For `action: "get_schemas"`**: - `connection` (string, required): The name of the connection to get schemata from - **For `action: "database_query"`**: + **For `action: "query"`**: - `connection` (string, required): The name of the database connection to execute the query on - `query` (string, required): The SQL query to execute - `limit` (number, optional): The limit of rows to return (defaults to 100) @@ -68,22 +109,120 @@ tools: * Always use the least number of columns, to minimize computation and the amount of data being retrieved. * Always use the LIMIT clause to minimize the number of rows returned. - **For `action: "database_get_columns"`**: + **For `action: "get_columns"`**: - `connection` (string, required): The name of the database connection - `table_name` (string, required): The fully qualified table name (e.g. 'schema.table' or just 'table') + + **Example Usage**: + ``` + # Fetch documentation + { + "action": "docs", + "input": {} + } + + # Query a database + { + "action": "query", + "input": { + "connection": "MY_PG", + "query": "SELECT * FROM users LIMIT 10" + } + } - **For `action: "file_list"`**: + # Get all schemas in a database + { + "action": "get_schemas", + "input": { + "connection": "MY_PG" + } + } + + # Get schema-level schemata (list of schemas) + { + "action": "get_schemata", + "input": { + "connection": "MY_PG", + "level": "schema" + } + } + + # Get table-level schemata for a specific schema + { + "action": "get_schemata", + "input": { + "connection": "MY_PG", + "level": "table", + "schema_name": "public" + } + } + + # Get column-level schemata for specific tables + { + "action": "get_schemata", + "input": { + "connection": "MY_PG", + "level": "column", + "schema_name": "public", + "table_names": ["users", "orders"] + } + } + + # Get column metadata for a specific table + { + "action": "get_columns", + "input": { + "connection": "MY_PG", + "table_name": "public.users" + } + } + + # Execute a complex analytical query + { + "action": "query", + "input": { + "connection": "MY_PG", + "query": "SELECT status, COUNT(*) as count FROM orders WHERE created_date >= '2024-01-01' GROUP BY status ORDER BY count DESC LIMIT 50", + "limit": 50 + } + } + + # Use transient connection for one-off query + { + "action": "query", + "input": { + "connection": "MY_PG", + "query": "SELECT version()", + "transient": true + } + } + ``` + + file_system: | + **Main file_system system tool that provides file system interaction capabilities. This is a parent tool that orchestrates various file operations through sub-actions.** + + **IMPORTANT**: To fully understand how to use the `file_system` tool, first call the `file_system/docs` tool. + + **Parameters**: + - `action` (string, required): The connection action to perform. Valid values: + - `"docs"` - Fetch documentation on how to use file + - `"list"` - List files in a file system connection + - `"copy"` - Copy files between connections + - `"inspect"` - Inspect file/directory metadata + - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: + + **For `action: "list"`**: - `connection` (string, required): The name of the file system connection to list from - `path` (string, required): The path to list files from - `recursive` (boolean, optional): Whether to list files recursively (default: false) - `only` (string, optional): Filter results to only 'files' or 'folders' - **For `action: "file_copy"`**: + **For `action: "copy"`**: - `source_location` (string, required): The source location in format 'connection_name/path' - `target_location` (string, required): The target location in format 'connection_name/path' (e.g. 's3/folder/file.txt' or 'local//absolute/path/to/file' or 'local/relative/path/to/file') - `recursive` (boolean, optional): Whether to copy files recursively (default: false) - **For `action: "file_inspect"`**: + **For `action: "inspect"`**: - `connection` (string, required): The name of the file system connection - `path` (string, required): The path to inspect - `recursive` (boolean, optional): Whether to get recursive statistics for directories (default: false) @@ -94,38 +233,69 @@ tools: **Example Usage**: ``` - # List all connections + # Fetch documentation { - "action": "list", + "action": "docs", "input": {} } + + # List files in a directory + { + "action": "list", + "input": { + "connection": "MY_S3", + "path": "data/csv_files/", + "recursive": false, + "only": "files" + } + } - # Test a connection + # List all files recursively in a folder { - "action": "test", + "action": "list", "input": { - "connection": "MY_PG", - "debug": true + "connection": "LOCAL_FS", + "path": "/home/user/documents", + "recursive": true } } - # Query a database + # Copy a file between connections { - "action": "database_query", + "action": "copy", "input": { - "connection": "MY_PG", - "query": "SELECT * FROM users LIMIT 10" + "source_location": "local/path/to/source.csv", + "target_location": "s3/bucket/folder/destination.csv", + "recursive": false } } - # Discover tables in a connection + # Copy entire directory recursively { - "action": "discover", + "action": "copy", "input": { - "connection": "MY_PG", - "pattern": "public.*", - "recursive": false, - "columns": false + "source_location": "s3/bucket/source_folder/", + "target_location": "local/backup/target_folder/", + "recursive": true + } + } + + # Inspect file metadata + { + "action": "inspect", + "input": { + "connection": "MY_S3", + "path": "data/large_file.parquet" + } + } + + # Get recursive directory statistics + { + "action": "inspect", + "input": { + "connection": "LOCAL_FS", + "path": "/var/log/", + "recursive": true } } ``` From 5b82ac11db77f1c20c5b77438e24b90a18ea06cd Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 16:54:05 -0300 Subject: [PATCH 28/37] Add comprehensive Sling Replication Guide for LLMs - Introduced a new markdown file detailing the Sling replication process. - Covered topics including introduction, quick start, replication structure, modes, source and target configurations, column management, data transformations, and more. - Provided examples for various replication scenarios such as database to database, file to database, and incremental loading. - Included best practices and troubleshooting tips to enhance user experience and efficiency. --- cmd/sling/resource/llm_CONNECTION.md | 765 +++++++ cmd/sling/resource/llm_CONNECTION_DATABASE.md | 693 ++++++ cmd/sling/resource/llm_CONNECTION_FILE.md | 848 ++++++++ cmd/sling/resource/llm_REPLICATION.md | 1909 +++++++++++++++++ 4 files changed, 4215 insertions(+) create mode 100644 cmd/sling/resource/llm_CONNECTION.md create mode 100644 cmd/sling/resource/llm_CONNECTION_DATABASE.md create mode 100644 cmd/sling/resource/llm_CONNECTION_FILE.md create mode 100644 cmd/sling/resource/llm_REPLICATION.md diff --git a/cmd/sling/resource/llm_CONNECTION.md b/cmd/sling/resource/llm_CONNECTION.md new file mode 100644 index 000000000..2bae51daf --- /dev/null +++ b/cmd/sling/resource/llm_CONNECTION.md @@ -0,0 +1,765 @@ +# Sling Connection Guide for LLMs + +## Table of Contents + +1. [Introduction and Overview](#1-introduction-and-overview) +2. [MCP Tool Integration](#2-mcp-tool-integration) +3. [Connection Types Overview](#3-connection-types-overview) +4. [Database Connections](#4-database-connections) +5. [File/Storage Connections](#5-filestorage-connections) +6. [Datalake Connections](#6-datalake-connections) +7. [API Connections](#7-api-connections) +8. [Connection Management](#8-connection-management) +9. [Discovery Operations](#9-discovery-operations) +10. [Environment and Configuration](#10-environment-and-configuration) +11. [Integration with Replications](#11-integration-with-replications) +12. [Best Practices](#12-best-practices) +13. [Examples and Common Patterns](#13-examples-and-common-patterns) +14. [Troubleshooting](#14-troubleshooting) + +--- + +## 1. Introduction and Overview + +### What are Sling Connections? + +Sling connections are configurations that define how to access and interact with various data sources and destinations. They provide a unified interface for connecting to over 40 different systems including databases, file systems, cloud storage, and APIs. + +### Key Benefits + +- **Unified Interface**: Single API for all connection types +- **Security**: Centralized credential management +- **Flexibility**: Support for multiple authentication methods +- **Discovery**: Built-in schema and metadata discovery +- **Testing**: Connection validation and health checks +- **Integration**: Seamless use with replications and pipelines + +### When to Use Connections + +- Setting up data sources and destinations for replications +- Exploring database schemas and file systems +- Running queries and data operations +- Managing credentials securely +- Discovering available data streams + +### Core Concepts + +- **Connection Name**: Unique identifier for the connection (typically UPPERCASE) +- **Connection Type**: The specific system type (e.g., postgres, s3, snowflake) +- **Properties**: Configuration parameters specific to each connection type +- **Environment**: Where credentials and configurations are stored +- **Location String**: Format for referencing connection paths (`CONNECTION_NAME/path/to/resource`) + +--- + +## 2. MCP Tool Integration + +The Sling MCP provides three main tools for working with connections: + +- **`connection`** - General connection management operations +- **`database`** - Database-specific operations (requires separate documentation call) +- **`file_system`** - File system operations (requires separate documentation call) + +### Connection Tool + +The `connection` tool provides core connection management through various actions: + +#### Available Actions + +- `docs` - Get comprehensive connection documentation +- `list` - List all available connections +- `set` - Create or update a connection +- `test` - Test connection validity +- `discover` - Discover tables/files in a connection + +#### Basic Tool Usage + +```json +{ + "tool": "connection", + "action": "action_name", + "input": { + "parameter1": "value1", + "parameter2": "value2" + } +} +``` + +#### Getting Documentation + +```json +{ + "tool": "connection", + "action": "docs", + "input": {} +} +``` + +### Database and File System Tools + +For database operations (queries, schema exploration) and file system operations (file listing, copying), use separate tools: + +- **Database Operations**: Use `database` tool - call `database` with `action: "docs"` for full documentation +- **File System Operations**: Use `file_system` tool - call `file_system` with `action: "docs"` for full documentation + +--- + +## 3. Connection Types Overview + +Sling supports 41+ different connection types across four categories: + +### Database Connections (24 types) +- **Relational**: PostgreSQL, MySQL, MariaDB, SQLServer, Oracle, SQLite +- **Cloud Warehouses**: Snowflake, BigQuery, Redshift, Databricks +- **Analytics**: ClickHouse, DuckDB, MotherDuck, StarRocks, Trino, Proton +- **NoSQL**: MongoDB, ElasticSearch, Prometheus +- **Specialized**: Azure Table, BigTable, Exasol, D1 + +### File/Storage Connections (13 types) +- **Cloud Storage**: AWS S3, Google Cloud Storage, Azure Blob +- **S3-Compatible**: MinIO, DigitalOcean Spaces, Cloudflare R2, Wasabi, B2 +- **File Transfer**: FTP, SFTP +- **Cloud Drives**: Google Drive +- **Local**: Local file system + +### Datalake Connections (4 types) +- **Query Engines**: Athena, DuckLake +- **Table Formats**: Iceberg + +### API Connections +- **Custom APIs**: User-defined API specifications in YAML format + +--- + +## 4. Database Connections + +Navigate or fetch the content of the connector from the below respective URL to learn more about it. + +**PostgreSQL (`postgres`)** -> https://docs.slingdata.io/connections/database-connections/postgres + +**MySQL (`mysql`)** -> https://docs.slingdata.io/connections/database-connections/mysql + +**Snowflake (`snowflake`)** -> https://docs.slingdata.io/connections/database-connections/snowflake + +**BigQuery (`bigquery`)** -> https://docs.slingdata.io/connections/database-connections/bigquery + +**SQL Server (`sqlserver`)** -> https://docs.slingdata.io/connections/database-connections/sqlserver + +**Oracle (`oracle`)** -> https://docs.slingdata.io/connections/database-connections/oraclese + +**ClickHouse (`clickhouse`)** -> https://docs.slingdata.io/connections/database-connections/clickhouse + +**DuckDB (`duckdb`)** -> https://docs.slingdata.io/connections/database-connections/duckdb + +**Redshift (`redshift`)** -> https://docs.slingdata.io/connections/database-connections/redshift + +**Databricks (`databricks`)** -> https://docs.slingdata.io/connections/database-connections/databricks + +**MongoDB (`mongodb`)** -> https://docs.slingdata.io/connections/database-connections/mongodb + +**MariaDB (`mariadb`)** -> https://docs.slingdata.io/connections/database-connections/mariadb + +**SQLite (`sqlite`)** -> https://docs.slingdata.io/connections/database-connections/sqlite + +**MotherDuck (`motherduck`)** -> https://docs.slingdata.io/connections/database-connections/motherduck + +**ElasticSearch (`elasticsearch`)** -> https://docs.slingdata.io/connections/database-connections/elasticsearch + +**Prometheus (`prometheus`)** -> https://docs.slingdata.io/connections/database-connections/prometheus + +**StarRocks (`starrocks`)** -> https://docs.slingdata.io/connections/database-connections/starrocks + +**Trino (`trino`)** -> https://docs.slingdata.io/connections/database-connections/trino + +**Proton (`proton`)** -> https://docs.slingdata.io/connections/database-connections/proton + +**Azure Table (`azuretable`)** -> https://docs.slingdata.io/connections/database-connections/azuretable + +**BigTable (`bigtable`)** -> https://docs.slingdata.io/connections/database-connections/bigtable + +**Exasol (`exasol`)** -> https://docs.slingdata.io/connections/database-connections/exasol + +**D1 (`d1`)** -> https://docs.slingdata.io/connections/database-connections/d1 + +--- + +## 5. File/Storage Connections + +Navigate or fetch the content of the connector from the below respective URL to learn more about it. + +**AWS S3 (`s3`)** -> https://docs.slingdata.io/connections/file-connections/s3 + +**Google Cloud Storage (`gs`)** -> https://docs.slingdata.io/connections/file-connections/gs + +**Azure Blob Storage (`azure`)** -> https://docs.slingdata.io/connections/file-connections/azure + +**MinIO (`minio`)** -> https://docs.slingdata.io/connections/file-connections/minio + +**DigitalOcean Spaces (`dospaces`)** -> https://docs.slingdata.io/connections/file-connections/dospaces + +**Cloudflare R2 (`r2`)** -> https://docs.slingdata.io/connections/file-connections/r2 + +**FTP (`ftp`)** -> https://docs.slingdata.io/connections/file-connections/ftp + +**SFTP (`sftp`)** -> https://docs.slingdata.io/connections/file-connections/sftp + +**Google Drive (`gdrive`)** -> https://docs.slingdata.io/connections/file-connections/gdrive + +**Wasabi (`wasabi`)** -> https://docs.slingdata.io/connections/file-connections/wasabi + +**Backblaze B2 (`b2`)** -> https://docs.slingdata.io/connections/file-connections/b2 + +**Local File System (`local`)** -> https://docs.slingdata.io/connections/file-connections/local + +--- + +## 6. Datalake Connections + +Navigate or fetch the content of the connector from the below respective URL to learn more about it. + +**Amazon Athena (`athena`)** -> https://docs.slingdata.io/connections/datalake-connections/athena + +**Apache Iceberg (`iceberg`) & AWS S3 Tables (`s3tables`)** -> https://docs.slingdata.io/connections/datalake-connections/iceberg + +**DuckLake (`ducklake`)** -> https://docs.slingdata.io/connections/datalake-connections/ducklake + +--- + +## 7. API Connections + +API connections in Sling use custom YAML specifications to define how to interact with REST APIs. + +### API Connection Properties +**Properties:** +- `type` (required) - Must be "api" +- `spec` (required) - Path to API specification file (`file:///path/to/spec.yaml`) +- `secrets` - Object containing API credentials and secrets + +### API Specification Structure + +API specifications define: +- **Authentication**: Basic, OAuth2, API keys, custom headers +- **Endpoints**: URL patterns, methods, parameters +- **Pagination**: Cursor, offset, or page-based pagination +- **Response Processing**: JSON path extraction, data transformation +- **Rate Limiting**: Request throttling and retry logic + +### Example API Connection Setup + +```json +{ + "action": "set", + "input": { + "name": "MY_API", + "properties": { + "type": "api", + "spec": "file:///path/to/my_api_spec.yaml", + "secrets": { + "api_key": "your-api-key", + "client_id": "your-client-id", + "client_secret": "your-client-secret" + } + } + } +} +``` + +--- + +## 8. Connection Management + +### Listing Connections + +Get all available connections across all sources (env.yaml, dbt profiles, environment variables): + +```json +{ + "action": "list", + "input": {} +} +``` + +**Returns:** JSON array with connection details including name, type, and source. + +### Setting Connections + +Create or update a connection in the Sling environment file: + +```json +{ + "action": "set", + "input": { + "name": "MY_POSTGRES", + "properties": { + "type": "postgres", + "host": "localhost", + "user": "myuser", + "database": "mydb", + "password": "mypass", + "port": 5432 + } + } +} +``` + +**Important Notes:** +- Check existing connections with `list` before overwriting +- Sensitive credentials should be manually set in `~/.sling/env.yaml` +- The tool will provide the env.yaml file path after setting + +### Testing Connections + +Validate that a connection works properly: + +```json +{ + "action": "test", + "input": { + "connection": "MY_POSTGRES", + "debug": true, + "trace": false + } +} +``` + +**Parameters:** +- `connection` (required) - Connection name to test +- `debug` - Enable debug logging +- `trace` - Enable trace logging +- `endpoints` - Array of specific endpoints to test (API connections only) +- `limit` - Maximum records to process during test + +--- + +## 9. Discovery Operations + +### Basic Discovery + +Discover tables, files, or endpoints in a connection: + +```json +{ + "action": "discover", + "input": { + "connection": "MY_POSTGRES", + "pattern": "public.*", + "recursive": false, + "columns": false + } +} +``` + +**Parameters:** +- `connection` (required) - Connection name +- `pattern` - Filter pattern (e.g., `schema.prefix_*`, `dir/*.csv`, `*/*/*.parquet`) +- `recursive` - List files recursively (file connections) +- `columns` - Include column-level metadata + +**Pattern Examples:** +- Database: `public.*`, `schema.table_name`, `*.users` +- Files: `dir/*.csv`, `data/**/*.json`, `logs/2024/*.parquet` + +### Column-Level Discovery + +Get detailed column information: + +```json +{ + "action": "discover", + "input": { + "connection": "MY_POSTGRES", + "pattern": "public.users", + "columns": true + } +} +``` + +**Returns:** Detailed column metadata including names, types, and positions. + +--- + +## 10. Environment and Configuration + +### Sling Environment File + +Connections are primarily stored in `~/.sling/env.yaml`: + +```yaml +connections: + MY_POSTGRES: + type: postgres + host: localhost + user: myuser + database: mydb + password: ${PG_PASSWORD} + port: 5432 + + MY_S3: + type: s3 + bucket: my-bucket + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_SECRET_ACCESS_KEY} + +variables: + global_var1: "value1" + global_var2: "value2" +``` + +### Environment Variables + +Connections can also be defined as environment variables: + +```bash +# URL format +export MY_POSTGRES='postgresql://user:pass@host:5432/db' + +# YAML format +export MY_S3='{type: s3, bucket: my-bucket, access_key_id: key, secret_access_key: secret}' +``` + +### DBT Profiles Integration + +Sling automatically reads DBT profiles from `~/.dbt/profiles.yml`: + +```yaml +my_project: + target: dev + outputs: + dev: + type: postgres + host: localhost + user: myuser + password: mypass + port: 5432 + dbname: mydb + schema: public +``` + +### Credential Precedence + +1. Sling env.yaml file (`~/.sling/env.yaml`) +2. Environment variables +3. DBT profiles (`~/.dbt/profiles.yml`) + +### Location String Format + +The location string format is used throughout Sling for referencing connection resources: + +**Database Objects:** +- `CONNECTION_NAME/schema.table` +- `CONNECTION_NAME/database.schema.table` + +**File System Objects:** +- `CONNECTION_NAME/path/to/file.txt` +- `CONNECTION_NAME/folder/` (for directories) +- `local/relative/path` (relative paths) +- `local//absolute/path` (absolute paths - note double slash) + +--- + +## 11. Integration with Replications + +Connections are the foundation of Sling replications, defining source and target systems. + +### Basic Replication Structure + +```yaml +source: MY_POSTGRES +target: MY_S3 + +defaults: + mode: full-refresh + +streams: + public.users: + object: users.csv + public.orders: + object: orders.parquet +``` + +### Connection Variables in Replications + +Access connection properties in replication configurations: + +```yaml +source: MY_POSTGRES +target: MY_S3 + +defaults: + target_options: + file_path: "exports/{source.schema}.{stream.object}.csv" + +streams: + public.*: +``` + +--- + +## 12. Best Practices + +### Security + +1. **Never hardcode credentials** in connection configurations +2. **Use environment variables** for sensitive values (`${PASSWORD}`) +3. **Rotate credentials regularly** and update connections +4. **Use least-privilege access** - only grant necessary permissions +5. **Enable SSL/TLS** for database connections where available +6. **Use SSH tunnels** for secure database access over untrusted networks + +### Connection Naming + +1. **Use UPPERCASE names** for consistency (automatic conversion) +2. **Use descriptive names** that indicate purpose: `PROD_POSTGRES`, `STAGING_S3` +3. **Include environment** in the name: `DEV_DB`, `PROD_DW` +4. **Group related connections** with prefixes: `SALES_DB`, `SALES_S3` + +### Performance + +1. **Test connections** before using in replications +2. **Use appropriate connection pooling** settings for databases +3. **Configure timeouts** appropriately for your network +4. **Use regions closest** to your data for cloud storage +5. **Enable compression** where supported + +### Maintenance + +1. **Document connection purposes** and owners +2. **Regular testing** of critical connections +3. **Monitor connection health** in production +4. **Version control** env.yaml file (without secrets) +5. **Backup connection configurations** + +--- + +## 13. Examples and Common Patterns + +### Database to Database Replication + +```json +// Set source database +{ + "action": "set", + "input": { + "name": "SOURCE_PG", + "properties": { + "type": "postgres", + "host": "source-db.com", + "user": "reader", + "database": "production" + } + } +} + +// Set target warehouse +{ + "action": "set", + "input": { + "name": "TARGET_SNOWFLAKE", + "properties": { + "type": "snowflake", + "account": "myaccount", + "user": "loader", + "database": "analytics", + "warehouse": "LOAD_WH" + } + } +} +``` + +### File to Database Loading + +```json +// Set S3 source +{ + "action": "set", + "input": { + "name": "DATA_S3", + "properties": { + "type": "s3", + "bucket": "data-lake", + "use_environment": true + } + } +} + +// Discover available files +{ + "action": "discover", + "input": { + "connection": "DATA_S3", + "pattern": "raw/2024/*/*.parquet" + } +} +``` + +### API Data Extraction + +```json +// Set up API connection +{ + "action": "set", + "input": { + "name": "SALESFORCE_API", + "properties": { + "type": "api", + "spec": "file:///configs/salesforce_spec.yaml", + "secrets": { + "client_id": "your-client-id", + "client_secret": "your-client-secret", + "username": "api-user@company.com", + "password": "password-with-token" + } + } + } +} + +// Test API endpoints +{ + "action": "test", + "input": { + "connection": "SALESFORCE_API", + "endpoints": ["accounts", "contacts"], + "limit": 10, + "debug": true + } +} +``` + +### SSH Tunnel Database Access + +```json +{ + "action": "set", + "input": { + "name": "SECURE_DB", + "properties": { + "type": "postgres", + "host": "internal-db.local", + "user": "appuser", + "database": "production", + "ssh_tunnel": "ssh://tunnel-user@bastion.company.com:22", + "ssh_private_key": "/path/to/private/key" + } + } +} +``` + +--- + +## 14. Troubleshooting + +### Connection Test Failures + +**Symptom:** Connection test fails with timeout or connection refused + +**Solutions:** +1. Verify host and port are correct +2. Check network connectivity and firewall rules +3. Confirm the service is running +4. Test with debug/trace logging enabled +5. Verify SSL/TLS configuration + +**Debug Example:** +```json +{ + "action": "test", + "input": { + "connection": "MY_DB", + "debug": true, + "trace": true + } +} +``` + +### Authentication Failures + +**Symptom:** "Access denied" or "Authentication failed" + +**Solutions:** +1. Verify username and password +2. Check if account is locked or expired +3. Confirm user has necessary permissions +4. For cloud services, verify region settings +5. Check if MFA is required + +### SSL/TLS Issues + +**Symptom:** "SSL connection error" or certificate warnings + +**Solutions:** +1. Set appropriate `sslmode` for database connections +2. Verify certificate chain is complete +3. Check if self-signed certificates are being used +4. Update connection to use correct SSL settings + +### File System Access Issues + +**Symptom:** "Access denied" or "Path not found" + +**Solutions:** +1. Verify bucket/container exists +2. Check IAM permissions and access policies +3. Confirm region settings for cloud storage +4. Test with anonymous access if appropriate +5. Verify file/directory paths are correct + +### Discovery Returns No Results + +**Symptom:** Empty results from discovery operations + +**Solutions:** +1. Check pattern syntax and wildcards +2. Verify schema/path exists +3. Confirm user has read permissions +4. Try broader patterns (e.g., `*.*` instead of specific schemas) + +### Performance Issues + +**Symptom:** Slow connection tests or operations + +**Solutions:** +1. Check network latency to target system +2. Adjust timeout settings +3. Verify system isn't under heavy load +4. Consider using connection pooling +5. Use regional endpoints for cloud services + +### Environment Configuration Issues + +**Symptom:** Connections not found or credentials not working + +**Solutions:** +1. Check `~/.sling/env.yaml` file exists and has correct format +2. Verify environment variables are set correctly +3. Check file permissions on configuration files +4. Ensure DBT profiles are in the correct location +5. Use `connection` tool with `list` action to see which connections are detected + +### Common Error Messages + +**"Invalid Connection name: X. Make sure it is created."** +- Connection doesn't exist or name is wrong +- Use `connection` tool with `list` action to see available connections +- Create the connection with `connection` tool and `set` action + +**"could not init database connection"** +- Database-specific connection error +- Enable debug logging to see detailed error +- Verify connection properties are correct + +**"could not init file connection - connection must be a file/storage type"** +- Trying to use file operations on a database connection +- Verify connection type matches the operation + +### Getting Help + +When troubleshooting: +1. Use debug/trace logging to get detailed error information +2. Test connections individually before using in replications +3. Verify credentials and permissions outside of Sling first +4. Check official documentation for specific connection types +5. Use the `connection` tool with `docs` action for the latest information + +--- + +This comprehensive guide covers all aspects of Sling connection management for LLM/AI usage. For database operations, use the `database` tool with the `docs` action. For file system operations, use the `file_system` tool with the `docs` action. \ No newline at end of file diff --git a/cmd/sling/resource/llm_CONNECTION_DATABASE.md b/cmd/sling/resource/llm_CONNECTION_DATABASE.md new file mode 100644 index 000000000..d34cdd127 --- /dev/null +++ b/cmd/sling/resource/llm_CONNECTION_DATABASE.md @@ -0,0 +1,693 @@ +# Sling Database Connection MCP Guide for LLMs + +## Table of Contents + +1. [Introduction and Overview](#1-introduction-and-overview) +2. [MCP Database Tool Integration](#2-mcp-database-tool-integration) +3. [Database Operations](#3-database-operations) +4. [Schema and Metadata Operations](#4-schema-and-metadata-operations) +5. [Query Operations](#5-query-operations) +6. [Common Usage Patterns](#6-common-usage-patterns) +7. [Database-Specific Examples](#7-database-specific-examples) +8. [Best Practices](#8-best-practices) +9. [Troubleshooting Database Operations](#9-troubleshooting-database-operations) + +--- + +## 1. Introduction and Overview + +### What are Database Operations? + +Database operations in Sling provide comprehensive functionality for interacting with database connections, including schema exploration, metadata retrieval, and read-only query execution. These operations work with all supported database types through a unified interface. + +### Key Capabilities + +- **Schema Discovery**: Explore database structure at schema, table, and column levels +- **Metadata Retrieval**: Get detailed information about tables, columns, and data types +- **Query Execution**: Run read-only SQL queries with results in structured format +- **Cross-Database Support**: Works with 24+ database types with consistent interface +- **Safety**: Built-in protection against destructive operations + +### Supported Database Types + +- **Relational**: PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite +- **Cloud Warehouses**: Snowflake, BigQuery, Redshift, Databricks +- **Analytics**: ClickHouse, DuckDB, MotherDuck, StarRocks, Trino, Proton +- **NoSQL**: MongoDB, ElasticSearch, Prometheus +- **Specialized**: Azure Table, BigTable, Exasol, D1 + +--- + +## 2. MCP Database Tool Integration + +The `database` tool provides database-specific operations through various actions. All operations require an existing database connection to be configured. + +### Available Actions + +- `docs` - Get comprehensive database documentation +- `get_schemata` - Get database structure (schema/table/column level) +- `get_schemas` - Get list of schema names +- `get_columns` - Get column metadata for a table +- `query` - Execute read-only SQL queries + +### Basic Tool Usage + +```json +{ + "action": "action_name", + "input": { + "connection": "DATABASE_CONNECTION_NAME", + "parameter1": "value1", + "parameter2": "value2" + } +} +``` + +### Getting Documentation + +```json +{ + "action": "docs", + "input": {} +} +``` + +**Important**: Database operations require a Pro token and are rate-limited. + +--- + +## 3. Database Operations + +### Prerequisites + +Before using database operations: + +1. **Connection Setup**: Database connection must be configured using the `connection` tool +2. **Connection Testing**: Verify connection works with `connection` tool `test` action +3. **Permissions**: Database user must have SELECT permissions on target objects +4. **Token**: Operations require a valid Sling CLI Pro token + +### Operation Categories + +#### Schema and Structure Operations +- `get_schemata` - Multi-level schema exploration +- `get_schemas` - Simple schema list +- `get_columns` - Detailed column information + +#### Data Access Operations +- `query` - Read-only SQL execution + +--- + +## 4. Schema and Metadata Operations + +### Getting Database Schemata + +Retrieve database structure at different levels of detail: + +```json +{ + "action": "get_schemata", + "input": { + "connection": "MY_POSTGRES", + "level": "table", + "schema_name": "public", + "table_names": ["users", "orders"] + } +} +``` + +**Parameters:** +- `connection` (required) - Database connection name +- `level` (required) - Level of detail: `schema`, `table`, or `column` +- `schema_name` (optional) - Specific schema (required for table/column levels if not fetching all schemas) +- `table_names` (optional) - Specific tables to focus on (relevant for table/column levels) + +**Levels Explained:** + +#### Schema Level +Lists all schemas/databases in the connection: +```json +{ + "action": "get_schemata", + "input": { + "connection": "MY_POSTGRES", + "level": "schema" + } +} +``` + +#### Table Level +Lists tables in specified schema(s): +```json +{ + "action": "get_schemata", + "input": { + "connection": "MY_POSTGRES", + "level": "table", + "schema_name": "public" + } +} +``` + +#### Column Level +Lists columns in specified tables: +```json +{ + "action": "get_schemata", + "input": { + "connection": "MY_POSTGRES", + "level": "column", + "schema_name": "public", + "table_names": ["users", "orders"] + } +} +``` + +### Getting Schema Names + +Get a simple list of schema names: + +```json +{ + "action": "get_schemas", + "input": { + "connection": "MY_POSTGRES" + } +} +``` + +**Parameters:** +- `connection` (required) - Database connection name + +**Returns:** Array of schema names available in the database. + +### Getting Column Metadata + +Get detailed column information for a specific table: + +```json +{ + "action": "get_columns", + "input": { + "connection": "MY_POSTGRES", + "table_name": "public.users" + } +} +``` + +**Parameters:** +- `connection` (required) - Database connection name +- `table_name` (required) - Fully qualified table name (e.g., `schema.table` or just `table`) + +**Returns:** Detailed column metadata including: +- Column names +- Data types +- Nullability +- Default values +- Position/order +- Constraints + +--- + +## 5. Query Operations + +### Executing SQL Queries + +Run read-only SQL queries on database connections: + +```json +{ + "action": "query", + "input": { + "connection": "MY_POSTGRES", + "query": "SELECT * FROM public.users LIMIT 10", + "limit": 100, + "transient": false + } +} +``` + +**Parameters:** +- `connection` (required) - Database connection name +- `query` (required) - SQL query to execute +- `limit` (optional) - Maximum rows to return (default: 100) +- `transient` (optional) - Use transient connection (default: false) + +### Query Safety and Restrictions + +**WARNING: Only use this tool for SELECT queries and other read-only operations. Never execute destructive queries such as DELETE, DROP, TRUNCATE, ALTER, UPDATE, INSERT, or any other data modification operations.** + +#### Allowed Operations +- `SELECT` statements +- `SHOW` statements +- `DESCRIBE` or `DESC` statements +- `EXPLAIN` statements +- Views and CTEs (Common Table Expressions) +- Read-only functions and procedures + +#### Prohibited Operations +- `DELETE` - Data deletion +- `DROP` - Object deletion +- `TRUNCATE` - Table truncation +- `ALTER` - Schema modifications +- `UPDATE` - Data modification +- `INSERT` - Data insertion +- `CREATE` - Object creation +- `GRANT`/`REVOKE` - Permission changes + +#### If Destructive Operations Are Needed +If a destructive operation is deemed necessary: +1. **DO NOT execute it directly** +2. Formulate the required SQL statement +3. **Return it to the USER for manual review and execution** + +### Query Best Practices + +1. **Always use LIMIT clause** to minimize data retrieval +2. **Select only necessary columns** to reduce computation and data transfer +3. **Use appropriate WHERE clauses** to filter data efficiently +4. **Test queries with small limits** before increasing data volume +5. **Consider query performance** and database load + +### Example Queries + +#### Basic Data Exploration +```json +{ + "action": "query", + "input": { + "connection": "MY_POSTGRES", + "query": "SELECT * FROM public.users LIMIT 5", + "limit": 5 + } +} +``` + +#### Analytical Query +```json +{ + "action": "query", + "input": { + "connection": "MY_POSTGRES", + "query": "SELECT status, COUNT(*) as count FROM orders WHERE created_date >= '2024-01-01' GROUP BY status ORDER BY count DESC LIMIT 20", + "limit": 20 + } +} +``` + +#### Metadata Query +```json +{ + "action": "query", + "input": { + "connection": "MY_POSTGRES", + "query": "SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' LIMIT 50", + "limit": 50 + } +} +``` + +#### Using Transient Connection +```json +{ + "action": "query", + "input": { + "connection": "MY_POSTGRES", + "query": "SELECT version()", + "transient": true + } +} +``` + +--- + +## 6. Common Usage Patterns + +### Database Exploration Workflow + +1. **List Schemas** +```json +{ + "action": "get_schemas", + "input": { + "connection": "MY_DB" + } +} +``` + +2. **Explore Tables in Schema** +```json +{ + "action": "get_schemata", + "input": { + "connection": "MY_DB", + "level": "table", + "schema_name": "production" + } +} +``` + +3. **Examine Table Structure** +```json +{ + "action": "get_columns", + "input": { + "connection": "MY_DB", + "table_name": "production.users" + } +} +``` + +4. **Sample Data** +```json +{ + "action": "query", + "input": { + "connection": "MY_DB", + "query": "SELECT * FROM production.users LIMIT 3" + } +} +``` + +### Data Analysis Workflow + +1. **Understand Data Volume** +```json +{ + "action": "query", + "input": { + "connection": "MY_DB", + "query": "SELECT COUNT(*) as total_records FROM sales.orders" + } +} +``` + +2. **Explore Data Distribution** +```json +{ + "action": "query", + "input": { + "connection": "MY_DB", + "query": "SELECT status, COUNT(*) as count FROM sales.orders GROUP BY status ORDER BY count DESC LIMIT 10" + } +} +``` + +3. **Check Data Quality** +```json +{ + "action": "query", + "input": { + "connection": "MY_DB", + "query": "SELECT COUNT(*) as nulls FROM sales.orders WHERE customer_id IS NULL" + } +} +``` + +### Cross-Database Pattern + +When working with multiple databases: + +```json +// Explore source database +{ + "action": "get_schemata", + "input": { + "connection": "SOURCE_DB", + "level": "table", + "schema_name": "public" + } +} + +// Check target database structure +{ + "action": "get_schemata", + "input": { + "connection": "TARGET_DW", + "level": "schema" + } +} + +// Compare data samples +{ + "action": "query", + "input": { + "connection": "SOURCE_DB", + "query": "SELECT COUNT(*), MIN(created_at), MAX(created_at) FROM public.users" + } +} +``` + +--- + +## 7. Database-Specific Examples + +### PostgreSQL + +```json +// Get table sizes +{ + "action": "query", + "input": { + "connection": "MY_POSTGRES", + "query": "SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size FROM pg_tables WHERE schemaname = 'public' ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC LIMIT 10" + } +} + +// Check indexes +{ + "action": "query", + "input": { + "connection": "MY_POSTGRES", + "query": "SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'users' LIMIT 20" + } +} +``` + +### MySQL + +```json +// Get database size +{ + "action": "query", + "input": { + "connection": "MY_MYSQL", + "query": "SELECT table_schema, ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS 'DB Size in MB' FROM information_schema.tables GROUP BY table_schema" + } +} +``` + +### Snowflake + +```json +// Get warehouse info +{ + "action": "query", + "input": { + "connection": "MY_SNOWFLAKE", + "query": "SHOW WAREHOUSES" + } +} + +// Check table clustering +{ + "action": "query", + "input": { + "connection": "MY_SNOWFLAKE", + "query": "SELECT * FROM information_schema.tables WHERE table_schema = 'PUBLIC' LIMIT 10" + } +} +``` + +### BigQuery + +```json +// Get dataset info +{ + "action": "query", + "input": { + "connection": "MY_BIGQUERY", + "query": "SELECT schema_name, catalog_name FROM INFORMATION_SCHEMA.SCHEMATA LIMIT 20" + } +} + +// Check table partitioning +{ + "action": "query", + "input": { + "connection": "MY_BIGQUERY", + "query": "SELECT table_name, partition_id, total_rows FROM `project.dataset.INFORMATION_SCHEMA.PARTITIONS` WHERE table_name = 'my_table' LIMIT 10" + } +} +``` + +### ClickHouse + +```json +// Get system information +{ + "action": "query", + "input": { + "connection": "MY_CLICKHOUSE", + "query": "SELECT * FROM system.databases LIMIT 20" + } +} + +// Check table engines +{ + "action": "query", + "input": { + "connection": "MY_CLICKHOUSE", + "query": "SELECT database, name, engine, total_rows FROM system.tables WHERE database = 'default' LIMIT 20" + } +} +``` + +--- + +## 8. Best Practices + +### Query Performance + +1. **Use LIMIT Always**: Include LIMIT clause to prevent large data transfers +2. **Select Specific Columns**: Avoid SELECT * for performance +3. **Filter Early**: Use WHERE clauses to reduce data processing +4. **Index Awareness**: Consider query execution plans for complex queries +5. **Connection Reuse**: Avoid transient connections for multiple queries + +### Security and Safety + +1. **Read-Only Principle**: Never execute modifying operations +2. **Permission Verification**: Ensure database user has minimal necessary permissions +3. **Query Review**: Review complex queries before execution +4. **Sensitive Data**: Be careful with personally identifiable information (PII) +5. **Connection Limits**: Be mindful of database connection limits + +### Error Handling + +1. **Start Simple**: Begin with basic queries and add complexity +2. **Syntax Validation**: Understand database-specific SQL dialects +3. **Connection Testing**: Verify connection works before complex operations +4. **Graceful Degradation**: Have fallback approaches for failed operations + +### Resource Management + +1. **Query Timeouts**: Be aware of database query timeout settings +2. **Result Size Limits**: Consider memory constraints for large result sets +3. **Concurrent Operations**: Avoid overwhelming database with simultaneous queries +4. **Connection Pooling**: Understand connection pool implications + +--- + +## 9. Troubleshooting Database Operations + +### Common Query Errors + +**Syntax Errors** +- Verify SQL syntax for specific database dialect +- Check table and column name spelling +- Ensure proper quoting of identifiers + +**Permission Errors** +- Verify database user has SELECT permissions +- Check schema access permissions +- Confirm connection user credentials + +**Connection Errors** +- Test basic connection with `connection` tool +- Verify database is accessible and running +- Check network connectivity and firewall settings + +### Schema Discovery Issues + +**No Schemas Returned** +- Check database user permissions +- Verify connection is to correct database instance +- Some databases may not support INFORMATION_SCHEMA + +**Missing Tables or Columns** +- Confirm user has access to target objects +- Check if objects exist in specified schema +- Verify case sensitivity requirements + +**Slow Schema Operations** +- Large databases may take time for full discovery +- Consider filtering with table_names parameter +- Use more specific schema_name values + +### Performance Issues + +**Slow Queries** +- Add or reduce LIMIT clause +- Simplify WHERE conditions +- Check database performance and load +- Consider query optimization + +**Timeout Errors** +- Reduce query complexity or data volume +- Check database timeout settings +- Consider breaking large operations into smaller parts + +**Memory Issues** +- Reduce LIMIT parameter +- Select fewer columns +- Filter data more aggressively + +### Connection-Specific Issues + +**PostgreSQL** +- Check `pg_hba.conf` for access control +- Verify SSL settings if required +- Consider connection pooling settings + +**MySQL** +- Verify user has proper host permissions +- Check MySQL timeout settings +- Consider max_connections limit + +**Snowflake** +- Ensure warehouse is running +- Check role permissions and privileges +- Verify account identifier is correct + +**BigQuery** +- Confirm project ID and dataset access +- Check service account permissions +- Verify billing is enabled for project + +### Getting Help + +When troubleshooting database operations: + +1. **Use Connection Test**: Verify basic connectivity with `connection` tool +2. **Enable Debug Logging**: Use debug flags in connection testing +3. **Start Simple**: Begin with basic queries and build complexity +4. **Check Documentation**: Reference database-specific documentation +5. **Verify Permissions**: Ensure appropriate database access rights + +### Error Message Patterns + +**"could not init database connection"** +- Basic connection failure +- Check connection configuration and credentials +- Verify database server is accessible + +**"permission denied"** +- Insufficient database permissions +- Grant appropriate SELECT permissions to database user +- Check schema and table access rights + +**"table or view does not exist"** +- Object not found or access denied +- Verify object names and schema +- Check if user has visibility to objects + +**"connection refused"** +- Network or server connectivity issue +- Verify host, port, and network access +- Check if database service is running + +--- + +This comprehensive guide covers all aspects of database operations using the Sling MCP `database` tool. All operations are designed to be safe, read-only, and provide structured access to database resources across multiple database platforms. \ No newline at end of file diff --git a/cmd/sling/resource/llm_CONNECTION_FILE.md b/cmd/sling/resource/llm_CONNECTION_FILE.md new file mode 100644 index 000000000..6dc7e47d1 --- /dev/null +++ b/cmd/sling/resource/llm_CONNECTION_FILE.md @@ -0,0 +1,848 @@ +# Sling File System Connection MCP Guide for LLMs + +## Table of Contents + +1. [Introduction and Overview](#1-introduction-and-overview) +2. [MCP File System Tool Integration](#2-mcp-file-system-tool-integration) +3. [File System Operations](#3-file-system-operations) +4. [File Listing and Discovery](#4-file-listing-and-discovery) +5. [File Copy Operations](#5-file-copy-operations) +6. [File Inspection and Metadata](#6-file-inspection-and-metadata) +7. [Common Usage Patterns](#7-common-usage-patterns) +8. [Storage-Specific Examples](#8-storage-specific-examples) +9. [Best Practices](#9-best-practices) +10. [Troubleshooting File Operations](#10-troubleshooting-file-operations) + +--- + +## 1. Introduction and Overview + +### What are File System Operations? + +File system operations in Sling provide comprehensive functionality for interacting with file-based connections, including cloud storage, local file systems, and file transfer protocols. These operations enable file listing, copying, inspection, and metadata retrieval across multiple storage platforms. + +### Key Capabilities + +- **File Listing**: Browse directories and discover files with pattern matching +- **File Copying**: Transfer files between different storage systems +- **File Inspection**: Get detailed metadata about files and directories +- **Cross-Platform Support**: Works with 13+ file system types with consistent interface +- **Pattern Matching**: Advanced filtering with glob patterns for file discovery + +### Supported File System Types + +- **Cloud Storage**: AWS S3, Google Cloud Storage, Azure Blob Storage +- **S3-Compatible**: MinIO, DigitalOcean Spaces, Cloudflare R2, Wasabi, Backblaze B2 +- **File Transfer**: FTP, SFTP +- **Cloud Drives**: Google Drive +- **Local**: Local file system + +--- + +## 2. MCP File System Tool Integration + +The `file_system` tool provides file system-specific operations through various actions. All operations require an existing file system connection to be configured. + +### Available Actions + +- `docs` - Get comprehensive file system documentation +- `list` - List files in a file system connection +- `copy` - Copy files between connections +- `inspect` - Inspect file/directory metadata + +### Basic Tool Usage + +```json +{ + "action": "action_name", + "input": { + "connection": "FILE_CONNECTION_NAME", + "parameter1": "value1", + "parameter2": "value2" + } +} +``` + +### Getting Documentation + +```json +{ + "action": "docs", + "input": {} +} +``` + +**Important**: File system operations require a Pro token and are rate-limited. + +--- + +## 3. File System Operations + +### Prerequisites + +Before using file system operations: + +1. **Connection Setup**: File system connection must be configured using the `connection` tool +2. **Connection Testing**: Verify connection works with `connection` tool `test` action +3. **Permissions**: Connection must have appropriate read/write permissions +4. **Token**: Operations require a valid Sling CLI Pro token + +### Operation Categories + +#### File Discovery Operations +- `list` - Browse directories and list files + +#### File Management Operations +- `copy` - Transfer files between connections +- `inspect` - Get file and directory metadata + +--- + +## 4. File Listing and Discovery + +### Listing Files + +List files and directories in a file system connection: + +```json +{ + "action": "list", + "input": { + "connection": "MY_S3", + "path": "data/", + "recursive": false, + "only": "files" + } +} +``` + +**Parameters:** +- `connection` (required) - File system connection name +- `path` (required) - Path to list files from +- `recursive` (optional) - Whether to list files recursively (default: false) +- `only` (optional) - Filter results to only 'files' or 'folders' + +**Filter Options:** +- `"files"` - Show only files +- `"folders"` - Show only directories/folders +- *omit parameter* - Show both files and folders + +### Basic File Listing + +List files in a specific directory: +```json +{ + "action": "list", + "input": { + "connection": "MY_S3", + "path": "data/csv_files/", + "recursive": false, + "only": "files" + } +} +``` + +### Recursive Listing + +List all files recursively in a folder structure: +```json +{ + "action": "list", + "input": { + "connection": "LOCAL_FS", + "path": "/home/user/documents", + "recursive": true + } +} +``` + +### Directory-Only Listing + +List only directories/folders: +```json +{ + "action": "list", + "input": { + "connection": "MY_AZURE", + "path": "containers/", + "only": "folders" + } +} +``` + +### Path Patterns + +File system paths vary by connection type: + +**Cloud Storage (S3, GCS, Azure):** +- `data/` - List contents of data folder +- `logs/2024/` - List contents of logs/2024 folder +- `/` or `""` - List root-level contents + +**Local File System:** +- `/absolute/path/` - Absolute path (Unix/Linux) +- `C:\Windows\` - Absolute path (Windows) +- `relative/path/` - Relative path + +**FTP/SFTP:** +- `/home/user/` - Absolute path +- `uploads/` - Relative path from connection root + +--- + +## 5. File Copy Operations + +### Copying Files + +Copy files between different connections: + +```json +{ + "action": "copy", + "input": { + "source_location": "MY_S3/data/input.csv", + "target_location": "LOCAL/tmp/output.csv", + "recursive": false + } +} +``` + +**Parameters:** +- `source_location` (required) - Source in format `connection_name/path` +- `target_location` (required) - Target in format `connection_name/path` +- `recursive` (optional) - Copy recursively for directories (default: false) + +### Location String Format + +The location string format combines connection name and path: + +**Format:** `CONNECTION_NAME/path/to/resource` + +**Examples:** +- Cloud storage: `MY_S3/folder/file.txt` +- Local relative: `LOCAL/relative/path/file.txt` +- Local absolute: `LOCAL//absolute/path/file.txt` *(note double slash for absolute)* + +### Single File Copy + +Copy a single file between connections: +```json +{ + "action": "copy", + "input": { + "source_location": "local/path/to/source.csv", + "target_location": "s3/bucket/folder/destination.csv", + "recursive": false + } +} +``` + +### Directory Copy + +Copy entire directory recursively: +```json +{ + "action": "copy", + "input": { + "source_location": "s3/bucket/source_folder/", + "target_location": "local/backup/target_folder/", + "recursive": true + } +} +``` + +### Multi-Cloud Copy + +Copy files between different cloud providers: +```json +{ + "action": "copy", + "input": { + "source_location": "AWS_S3/data/export.csv", + "target_location": "GCS_BUCKET/imports/data.csv" + } +} +``` + +### Local to Cloud Backup + +Backup local files to cloud storage: +```json +{ + "action": "copy", + "input": { + "source_location": "LOCAL//home/user/documents/", + "target_location": "BACKUP_S3/daily-backup/documents/", + "recursive": true + } +} +``` + +--- + +## 6. File Inspection and Metadata + +### Inspecting Files + +Get metadata about files or directories: + +```json +{ + "action": "inspect", + "input": { + "connection": "MY_S3", + "path": "data/large_dataset/", + "recursive": true + } +} +``` + +**Parameters:** +- `connection` (required) - File system connection name +- `path` (required) - Path to inspect +- `recursive` (optional) - Get recursive statistics for directories (default: false) + +### File Metadata + +Inspect a single file: +```json +{ + "action": "inspect", + "input": { + "connection": "MY_S3", + "path": "data/large_file.parquet" + } +} +``` + +**Returns:** +- File size +- Creation time +- Modification time +- Content type/MIME type +- Storage class (cloud storage) +- Permissions (where applicable) + +### Directory Statistics + +Get recursive directory statistics: +```json +{ + "action": "inspect", + "input": { + "connection": "LOCAL_FS", + "path": "/var/log/", + "recursive": true + } +} +``` + +**Returns:** +- Total size +- File count +- Directory count +- Size distribution +- Last modified times + +### Batch File Inspection + +Inspect multiple files by combining with list operation: +```json +// First list files +{ + "action": "list", + "input": { + "connection": "MY_S3", + "path": "data/", + "only": "files" + } +} + +// Then inspect specific files based on list results +{ + "action": "inspect", + "input": { + "connection": "MY_S3", + "path": "data/important_file.csv" + } +} +``` + +--- + +## 7. Common Usage Patterns + +### File Discovery Workflow + +1. **Browse Root Directory** +```json +{ + "action": "list", + "input": { + "connection": "MY_STORAGE", + "path": "" + } +} +``` + +2. **Explore Specific Folders** +```json +{ + "action": "list", + "input": { + "connection": "MY_STORAGE", + "path": "data/2024/", + "recursive": true, + "only": "files" + } +} +``` + +3. **Inspect Interesting Files** +```json +{ + "action": "inspect", + "input": { + "connection": "MY_STORAGE", + "path": "data/2024/large_dataset.parquet" + } +} +``` + +### Data Migration Workflow + +1. **List Source Files** +```json +{ + "action": "list", + "input": { + "connection": "OLD_STORAGE", + "path": "legacy_data/", + "recursive": true + } +} +``` + +2. **Copy Files to New Location** +```json +{ + "action": "copy", + "input": { + "source_location": "OLD_STORAGE/legacy_data/", + "target_location": "NEW_STORAGE/migrated_data/", + "recursive": true + } +} +``` + +3. **Verify Copy Completed** +```json +{ + "action": "inspect", + "input": { + "connection": "NEW_STORAGE", + "path": "migrated_data/", + "recursive": true + } +} +``` + +### Backup and Sync Pattern + +1. **Inspect Source Directory** +```json +{ + "action": "inspect", + "input": { + "connection": "LOCAL", + "path": "/important/data/", + "recursive": true + } +} +``` + +2. **Create Timestamped Backup** +```json +{ + "action": "copy", + "input": { + "source_location": "LOCAL//important/data/", + "target_location": "BACKUP_S3/backups/2024-08-24/data/", + "recursive": true + } +} +``` + +3. **Verify Backup Size** +```json +{ + "action": "inspect", + "input": { + "connection": "BACKUP_S3", + "path": "backups/2024-08-24/", + "recursive": true + } +} +``` + +### File Organization Pattern + +1. **List Unorganized Files** +```json +{ + "action": "list", + "input": { + "connection": "MESSY_STORAGE", + "path": "uploads/", + "only": "files" + } +} +``` + +2. **Copy Files to Organized Structure** +```json +{ + "action": "copy", + "input": { + "source_location": "MESSY_STORAGE/uploads/document1.pdf", + "target_location": "ORGANIZED_STORAGE/documents/2024/document1.pdf" + } +} +``` + +3. **Verify Organization** +```json +{ + "action": "list", + "input": { + "connection": "ORGANIZED_STORAGE", + "path": "documents/", + "recursive": true + } +} +``` + +--- + +## 8. Storage-Specific Examples + +### AWS S3 + +```json +// List S3 bucket contents +{ + "action": "list", + "input": { + "connection": "MY_S3", + "path": "data/logs/", + "recursive": false + } +} + +// Copy to S3 with folder structure +{ + "action": "copy", + "input": { + "source_location": "LOCAL/exports/", + "target_location": "MY_S3/processed/daily/", + "recursive": true + } +} + +// Inspect S3 object metadata +{ + "action": "inspect", + "input": { + "connection": "MY_S3", + "path": "processed/daily/report.parquet" + } +} +``` + +### Google Cloud Storage + +```json +// List GCS bucket +{ + "action": "list", + "input": { + "connection": "MY_GCS", + "path": "analytics/", + "only": "files" + } +} + +// Copy between GCS and local +{ + "action": "copy", + "input": { + "source_location": "MY_GCS/analytics/results.csv", + "target_location": "LOCAL/downloads/gcs_results.csv" + } +} +``` + +### Azure Blob Storage + +```json +// List Azure container +{ + "action": "list", + "input": { + "connection": "MY_AZURE", + "path": "container/data/", + "recursive": true + } +} + +// Copy to Azure Blob +{ + "action": "copy", + "input": { + "source_location": "LOCAL/uploads/file.json", + "target_location": "MY_AZURE/container/processed/file.json" + } +} +``` + +### Local File System + +```json +// List local directory +{ + "action": "list", + "input": { + "connection": "LOCAL", + "path": "/tmp/processing/", + "only": "files" + } +} + +// Copy local files +{ + "action": "copy", + "input": { + "source_location": "LOCAL//home/user/data/", + "target_location": "LOCAL//tmp/backup/", + "recursive": true + } +} + +// Inspect local file +{ + "action": "inspect", + "input": { + "connection": "LOCAL", + "path": "/tmp/large_file.db" + } +} +``` + +### SFTP + +```json +// List SFTP directory +{ + "action": "list", + "input": { + "connection": "MY_SFTP", + "path": "/uploads/", + "recursive": false + } +} + +// Copy from SFTP to cloud +{ + "action": "copy", + "input": { + "source_location": "MY_SFTP/uploads/data.csv", + "target_location": "MY_S3/ingestion/sftp_data.csv" + } +} +``` + +### FTP + +```json +// List FTP directory +{ + "action": "list", + "input": { + "connection": "MY_FTP", + "path": "/pub/data/", + "only": "files" + } +} + +// Download from FTP +{ + "action": "copy", + "input": { + "source_location": "MY_FTP/pub/data/report.txt", + "target_location": "LOCAL/downloads/ftp_report.txt" + } +} +``` + +--- + +## 9. Best Practices + +### File Operations + +1. **Test Connections First**: Always verify connection works before file operations +2. **Use Specific Paths**: Avoid overly broad recursive operations on large directories +3. **Check Permissions**: Ensure connections have appropriate read/write access +4. **Monitor Transfer Progress**: For large files, consider breaking into smaller operations +5. **Verify Copy Results**: Use inspect to confirm successful file transfers + +### Performance Optimization + +1. **Batch Operations**: Group related file operations when possible +2. **Avoid Deep Recursion**: Limit recursive operations on deep directory structures +3. **Use Filters**: Apply "only" parameter to reduce unnecessary results +4. **Regional Proximity**: Use storage regions close to your processing location +5. **Compression**: Enable compression for large file transfers when supported + +### Security and Access + +1. **Least Privilege**: Grant minimal necessary permissions to connections +2. **Credential Management**: Use secure credential storage and rotation +3. **Path Validation**: Validate paths to prevent unauthorized access +4. **Encryption**: Use encrypted connections (HTTPS, SFTP) when available +5. **Access Logging**: Monitor file access patterns for security + +### Error Prevention + +1. **Path Existence**: Verify paths exist before operations +2. **Space Availability**: Check available storage space for large copies +3. **Overwrite Protection**: Be cautious with copy operations that may overwrite +4. **Connection Timeouts**: Set appropriate timeouts for large operations +5. **Retry Logic**: Implement retry patterns for network-dependent operations + +--- + +## 10. Troubleshooting File Operations + +### Common File Operation Errors + +**Path Not Found** +- Verify path exists and is accessible +- Check connection permissions +- Ensure proper path format for connection type + +**Permission Denied** +- Verify connection credentials have necessary permissions +- Check file/directory access rights +- Ensure write permissions for copy target locations + +**Connection Timeouts** +- Check network connectivity +- Verify storage service availability +- Consider breaking large operations into smaller chunks + +### List Operation Issues + +**Empty Results** +- Verify path exists and contains files +- Check connection permissions for directory access +- Ensure proper path format (trailing slashes, etc.) + +**Slow Listing** +- Avoid overly broad recursive operations +- Use specific paths instead of root-level listings +- Consider pagination for very large directories + +**Missing Files** +- Check if files have been moved or deleted +- Verify connection time zone settings +- Confirm path case sensitivity requirements + +### Copy Operation Issues + +**Copy Failures** +- Verify source file exists and is accessible +- Check target location has write permissions +- Ensure sufficient storage space at target +- Confirm no file locks or exclusive access issues + +**Incomplete Copies** +- Check for network interruptions during transfer +- Verify file sizes match between source and target +- Consider retry mechanisms for large files + +**Overwrite Issues** +- Check if target file already exists +- Verify overwrite permissions +- Consider backup strategies before overwriting + +### Inspect Operation Issues + +**Missing Metadata** +- Some storage types provide limited metadata +- Check if connection supports detailed inspection +- Verify file exists and is accessible + +**Slow Inspection** +- Large directories with recursive inspection can be slow +- Consider non-recursive inspection first +- Use specific file paths when possible + +### Storage-Specific Issues + +**AWS S3** +- Verify S3 bucket permissions and IAM roles +- Check S3 bucket region matches connection configuration +- Ensure S3 bucket exists and is accessible + +**Google Cloud Storage** +- Confirm GCS bucket permissions +- Verify service account has appropriate roles +- Check if GCS APIs are enabled for the project + +**Azure Blob Storage** +- Verify Azure storage account access keys +- Check container permissions and access levels +- Ensure storage account is accessible from your location + +**Local File System** +- Check file system permissions (chmod/chown on Unix) +- Verify paths exist and are accessible +- Consider disk space limitations + +**FTP/SFTP** +- Verify server accessibility and credentials +- Check firewall settings for FTP passive/active modes +- Confirm SSH key authentication for SFTP + +### Getting Help + +When troubleshooting file system operations: + +1. **Test Basic Connectivity**: Use `connection` tool to verify connection works +2. **Start Simple**: Begin with basic list operations before complex copies +3. **Check Paths**: Verify all paths exist and are properly formatted +4. **Monitor Permissions**: Ensure appropriate access rights for all operations +5. **Use Debug Logging**: Enable debug mode in connection testing + +### Error Message Patterns + +**"could not init file connection - connection must be a file/storage type"** +- Using file operations on a database connection +- Verify connection type is appropriate for file operations + +**"access denied" or "permission denied"** +- Insufficient permissions for file/directory access +- Check connection credentials and access rights + +**"path not found" or "no such file or directory"** +- Specified path does not exist +- Verify path spelling and existence + +**"connection timeout"** +- Network or server connectivity issue +- Check network connection and server availability + +--- + +This comprehensive guide covers all aspects of file system operations using the Sling MCP `file_system` tool. All operations provide structured access to file resources across multiple storage platforms with consistent interfaces and error handling. \ No newline at end of file diff --git a/cmd/sling/resource/llm_REPLICATION.md b/cmd/sling/resource/llm_REPLICATION.md new file mode 100644 index 000000000..d5114df73 --- /dev/null +++ b/cmd/sling/resource/llm_REPLICATION.md @@ -0,0 +1,1909 @@ +# Sling Replication Guide for LLMs + +## Table of Contents + +1. [Introduction and Overview](#1-introduction-and-overview) +2. [Quick Start Guide](#2-quick-start-guide) +3. [Replication Structure](#3-replication-structure) +4. [Modes and Strategies](#4-modes-and-strategies) +5. [Source Configuration](#5-source-configuration) +6. [Target Configuration](#6-target-configuration) +7. [Column Management](#7-column-management) +8. [Data Transformations](#8-data-transformations) +9. [Variables and Dynamic Configuration](#9-variables-and-dynamic-configuration) +10. [Wildcards and Patterns](#10-wildcards-and-patterns) +11. [Hooks and Automation](#11-hooks-and-automation) +12. [Examples - Database to Database](#12-examples---database-to-database) +13. [Examples - File to Database](#13-examples---file-to-database) +14. [Examples - Database to File](#14-examples---database-to-file) +15. [Advanced Topics](#15-advanced-topics) +16. [MCP Tool Integration](#16-mcp-tool-integration) +17. [Best Practices](#17-best-practices) +18. [Troubleshooting](#18-troubleshooting) + +--- + +## 1. Introduction and Overview + +### What are Sling Replications? + +Sling replications are YAML or JSON configuration files that define data movement operations from source to target systems. They are the best way to use Sling in a reusable, scalable manner, allowing you to define multiple data streams with shared configuration patterns. + +### Key Benefits + +- **Reusability**: Define defaults that apply to multiple streams +- **Scalability**: Handle multiple tables/files in a single configuration +- **Flexibility**: Override defaults on a per-stream basis +- **Pattern Matching**: Use wildcards to match multiple sources +- **Automation**: Built-in parallel processing and error handling + +### When to Use Replications + +- Moving multiple tables between databases +- Processing multiple files from a directory +- Applying consistent transformations across datasets +- Implementing incremental data pipelines +- Standardizing data movement workflows + +### Core Concepts + +- **Source**: The origin system (database, file system, cloud storage) +- **Target**: The destination system (database, file system, cloud storage) +- **Streams**: Individual data sources (tables, files) to replicate +- **Defaults**: Shared configuration applied to all streams +- **Modes**: How data is loaded (full-refresh, incremental, etc.) + +### MCP Tool Integration + +The Sling MCP tool provides these replication commands: +- `replication/docs` - Get documentation +- `replication/parse` - Parse and validate configuration +- `replication/compile` - Compile configuration with validation +- `replication/run` - Execute the replication + +--- + +## 2. Quick Start Guide + +### Essential MCP Commands + +```json +// Get replication documentation +{ + "action": "docs", + "input": {} +} + +// Parse a replication file +{ + "action": "parse", + "input": { + "file_path": "/path/to/replication.yaml" + } +} + +// Run a replication +{ + "action": "run", + "input": { + "file_path": "/path/to/replication.yaml", + "mode": "incremental" + } +} +``` + +### Basic Replication Structure + +```yaml +source: SOURCE_CONNECTION +target: TARGET_CONNECTION + +defaults: + mode: full-refresh + object: '{target_schema}.{stream_schema}_{stream_table}' + +streams: + schema.table1: + schema.table2: + mode: incremental + primary_key: [id] + +env: + SLING_THREADS: 3 +``` + +### Simple Database-to-Database Example + +```yaml +source: MY_POSTGRES +target: MY_SNOWFLAKE + +defaults: + mode: full-refresh + object: 'warehouse.{stream_schema}_{stream_table}' + +streams: + # Single table + public.customers: + + # All tables in schema + public.*: + + # Exclude specific table + public.sensitive_data: + disabled: true + +env: + SLING_THREADS: 5 +``` + +### Simple File-to-Database Example + +```yaml +source: LOCAL +target: POSTGRES + +defaults: + mode: full-refresh + object: 'staging.{stream_file_name}' + source_options: + format: csv + header: true + +streams: + 'file://data/customers.csv': + 'file://data/products.csv': + 'data/*.csv': # All CSV files in directory + +env: + SLING_THREADS: 3 +``` + +--- + +## 3. Replication Structure + +### Root Level Keys + +```yaml +# Required keys +source: +target: +streams: + +# Optional keys +defaults: +hooks: +env: +``` + +### Stream Configuration Map + +Each stream accepts the following configuration: + +```yaml +streams: + stream_name: + # Target specification + object: + + # Load behavior + mode: full-refresh | incremental | truncate | snapshot | backfill + disabled: true | false + + # Key columns + primary_key: [column1, column2] + update_key: column_name + + # Column management + columns: {column_name: data_type} + select: [column1, column2] # or [-column3, -column4] to exclude columns + + # Data source + sql: + files: [file1.csv, file2.csv] + where: + single: true | false + + # Transformations + transforms: + + # Options + source_options: + target_options: + + # Hooks + hooks: + + # Description + description: +``` + +### Using Defaults + +The `defaults` section allows you to define common configuration that applies to all streams: + +```yaml +defaults: + mode: incremental + object: '{target_schema}.{stream_schema}_{stream_table}' + primary_key: [id] + source_options: + empty_as_null: false + target_options: + column_casing: snake + +streams: + # Inherits all defaults + schema.table1: + + # Overrides specific defaults + schema.table2: + mode: full-refresh + target_options: + column_casing: upper +``` + +### Configuration Inheritance + +- Stream-level configuration overrides defaults +- Arrays and objects are replaced entirely (not merged) +- Use defaults for common patterns, stream-level for exceptions + +--- + +## 4. Modes and Strategies + +### Available Modes + +| Mode | Description | Use Case | +|------|-------------|----------| +| `full-refresh` | Drop and recreate target table | Complete data replacement | +| `incremental` | Merge/upsert new data | Ongoing data updates | +| `truncate` | Clear table, preserve structure | Refresh while keeping DDL | +| `snapshot` | Append with timestamp | Historical data tracking | +| `backfill` | Load specific date/ID ranges | Historical data recovery | + +### Incremental Mode Strategies + +| Primary Key | Update Key | Strategy | Description | +|-------------|------------|----------|-------------| +| ✅ | ✅ | New Data Upsert | Only load records after max(update_key) | +| ✅ | ❌ | Full Data Upsert | Load all data, upsert by primary key | +| ❌ | ✅ | Append Only | Only load records after max(update_key) | + +### Incremental with Custom SQL + +Use placeholders for incremental loading: + +```yaml +streams: + custom_query: + sql: | + SELECT * FROM transactions + WHERE created_at > coalesce({incremental_value}, '1900-01-01') + AND status = 'completed' + update_key: created_at + mode: incremental +``` + +**Placeholder Options:** +- `{incremental_where_cond}` - Complete WHERE condition +- `{incremental_value}` - Just the incremental value + +### Backfill Mode + +Load historical data in chunks: + +```yaml +streams: + large_table: + mode: backfill + update_key: created_date + source_options: + range: '2023-01-01,2023-12-31' + chunk_size: 7d # Process 7 days at a time +``` + +--- + +## 5. Source Configuration + +### Database Sources + +```yaml +defaults: + source_options: + limit: 100000 # Row limit + empty_as_null: false +``` + +### File Sources + +#### CSV Files +```yaml +defaults: + source_options: + format: csv + header: true + delimiter: ',' + encoding: utf8 + skip_blank_lines: true +``` + +#### JSON Files +```yaml +defaults: + source_options: + format: json + jmespath: 'data[*]' # Extract nested data + flatten: true +``` + +#### Excel Files +```yaml +defaults: + source_options: + format: xlsx + sheet: 'Sheet1' # or 'Sheet1!A1:H100' + header: true +``` + +#### Parquet Files +```yaml +defaults: + source_options: + format: parquet + compression: auto +``` + +### Encoding Options + +Handle international characters: + +```yaml +source_options: + encoding: utf8 | latin1 | windows1252 | windows1250 | utf16 +``` + +### Compression Support + +```yaml +source_options: + compression: auto | gzip | zip | snappy | zstd +``` + +### Custom SQL Sources + +```yaml +streams: + analytics_view: + sql: | + SELECT + customer_id, + SUM(order_total) as total_spent, + COUNT(*) as order_count + FROM orders + WHERE order_date >= '2024-01-01' + GROUP BY customer_id + object: analytics.customer_summary +``` + +### JMESPath for JSON + +Extract specific data from complex JSON: + +```yaml +streams: + 'api_response.json': + source_options: + jmespath: 'results[*].{id: id, name: name, email: contact.email}' + flatten: true +``` + +--- + +## 6. Target Configuration + +### Database Targets + +#### Column Casing +```yaml +defaults: + target_options: + column_casing: snake # source | target | snake | upper | lower | normalize +``` + +#### Column Typing +```yaml +defaults: + target_options: + column_typing: + string: + length_factor: 2 # Double string lengths + max_length: 8000 + decimal: + min_precision: 10 + max_precision: 38 +``` + +#### Table Management +```yaml +defaults: + target_options: + add_new_columns: true + adjust_column_type: false + use_bulk: true + direct_insert: true # Use when user does not want a temp table. +``` + +#### Custom Table DDL +```yaml +defaults: + target_options: + table_ddl: 'CREATE TABLE {stream_table} ({col_types}) ENGINE=MergeTree' +``` + +#### Table Keys +```yaml +defaults: + target_options: + table_keys: + primary: [id] + index: [customer_id, created_at] + cluster: [region] # BigQuery/Snowflake + partition: [date_column] # PostgreSQL/ClickHouse +``` + +### File Targets + +#### CSV Export +```yaml +defaults: + target_options: + format: csv + header: true + delimiter: ',' + encoding: utf8 +``` + +#### Parquet Export +```yaml +defaults: + target_options: + format: parquet + compression: snappy + file_max_rows: 1000000 +``` + +#### JSON Export +```yaml +defaults: + target_options: + format: jsonlines # or json + compression: gzip +``` + +#### File Splitting +```yaml +defaults: + target_options: + file_max_rows: 500000 + file_max_bytes: 50000000 +``` + +--- + +## 7. Column Management + +### Selecting Columns + +```yaml +streams: + customer_data: + # Include only specific columns + select: [id, name, email, created_at] + + sensitive_data: + # Exclude specific columns + select: [-ssn, -credit_card, -password] +``` + +### Column Type Casting + +```yaml +defaults: + columns: + id: bigint + amount: decimal(10,2) + status: string(50) + created_at: datetime + +streams: + special_table: + columns: + '*': string # Cast all columns to string + id: bigint # Override for specific column +``` + +### Data Type Options + +- `bigint` - Large integers +- `integer` - Standard integers +- `decimal(precision, scale)` - Fixed-point decimals +- `string(length)` - Variable character strings +- `text(length)` - Large text fields +- `bool` - Boolean values +- `datetime` - Date and time values +- `json` - JSON data + +### Column Constraints + +```yaml +columns: + age: 'integer | value >= 0 AND value <= 120' + email: 'string(255) | value IS NOT NULL' + amount: 'decimal(10,2) | value > 0' +``` + +### Schema Evolution + +Sling automatically handles: +- Adding new columns from source +- Preserving existing data when columns are removed from source +- Type adjustments (when `adjust_column_type: true`) + +--- + +## 8. Data Transformations + +### Modern Transform Syntax (v1.4.17+) + +#### Array Transforms (Global) +Apply to all columns: + +```yaml +transforms: ['trim_space', 'remove_diacritics'] +``` + +#### Map Transforms (Column-Specific) +Apply to specific columns: + +```yaml +transforms: + name: ['trim_space', 'upper'] + email: ['lower'] + phone: ['replace_non_printable'] +``` + +#### Staged Transforms (Most Powerful) +Multi-stage with expressions: + +```yaml +transforms: + # Stage 1: Clean text fields + - name: 'trim_space(value)' + email: 'lower(value)' + + # Stage 2: Create computed columns + - full_name: 'record.first_name + " " + record.last_name' + email_hash: 'hash(record.email, "md5")' + + # Stage 3: Conditional logic + - customer_tier: 'record.total_spent >= 1000 ? "premium" : "standard"' + discount: 'record.customer_tier == "premium" ? 0.15 : 0.05' +``` + +### Available Functions (50+) + +See https://docs.slingdata.io/concepts/functions for complete list. + +#### String Functions +- `upper(value)` - Convert to uppercase +- `lower(value)` - Convert to lowercase +- `trim_space(value)` - Remove whitespace +- `replace(value, old, new)` - Replace text +- `substring(value, start, length)` - Extract substring +- `hash(value, algorithm)` - Hash value (md5, sha256) + +#### Numeric Functions +- `int_parse(value)` - Convert to integer +- `float_parse(value)` - Convert to float +- `greatest(val1, val2, ...)` - Maximum value +- `least(val1, val2, ...)` - Minimum value + +#### Date Functions +- `now()` - Current timestamp +- `date_parse(value, format)` - Parse date string +- `date_format(value, format)` - Format date +- `date_add(date, interval, unit)` - Add time interval + +#### Utility Functions +- `coalesce(val1, val2, ...)` - First non-null value +- `cast(value, type)` - Type conversion +- `uuid()` - Generate UUID +- `if(condition, true_val, false_val)` - Conditional + +### Complex Transform Examples + +#### Customer Data Processing +```yaml +streams: + customers: + transforms: + # Stage 1: Clean and normalize + - first_name: 'trim_space(upper(value))' + email: 'lower(trim_space(value))' + phone: 'replace(value, "[^0-9]", "")' + + # Stage 2: Create computed fields + - full_name: 'record.first_name + " " + record.last_name' + email_domain: 'substring(record.email, strpos(record.email, "@") + 1)' + + # Stage 3: Business logic + - customer_type: | + record.total_orders >= 50 ? "vip" : ( + record.total_orders >= 10 ? "regular" : "new" + ) +``` + +### Legacy Transforms (Pre v1.4.17) + +Still supported for backwards compatibility: + +```yaml +transforms: + - trim_space + - replace_accents + - replace_non_printable + - empty_as_null +``` + +--- + +## 9. Variables and Dynamic Configuration + +### Runtime Variables + +Automatically replaced at execution: + +#### Table/Stream Variables +- `{stream_name}` - Full stream name +- `{stream_schema}` - Source schema name +- `{stream_table}` - Source table name +- `{stream_full_name}` - Fully qualified source name + +#### Connection Variables +- `{source_name}` - Source connection name +- `{target_name}` - Target connection name +- `{target_schema}` - Target default schema + +#### Target Variables +- `{object_name}` - Target object name +- `{object_schema}` - Target object schema +- `{object_table}` - Target object table + +#### File Variables +- `{stream_file_name}` - File name without extension +- `{stream_file_folder}` - Parent directory +- `{stream_file_ext}` - File extension +- `{stream_file_path}` - Full file path + +### Timestamp Variables + +- `{run_timestamp}` - Execution timestamp (2006_01_02_150405) +- `{YYYY}` - 4-digit year +- `{MM}` - 2-digit month +- `{DD}` - 2-digit day +- `{HH}` - 2-digit hour (24-hour) +- `{ISO8601}` - ISO 8601 format + +### Partition Variables (Parquet Files) + +Requires `update_key` specification: + +- `{part_year}` - Year partition +- `{part_month}` - Month partition +- `{part_year_month}` - Combined year-month +- `{part_day}` - Day partition +- `{part_week}` - Week partition +- `{part_hour}` - Hour partition + +```yaml +streams: + transactions: + object: 'data/{part_year}/{part_month}/{stream_table}' + update_key: transaction_date + target_options: + format: parquet +``` + +### Environment Variables + +#### In Configuration Files +```yaml +env: + TARGET_SCHEMA: '${TARGET_SCHEMA}' # From environment + BATCH_SIZE: 10000 + SLING_THREADS: 5 +``` + +#### In Object Names +```yaml +defaults: + object: '{target_schema}.{stream_schema}_{stream_table}_{YYYY}_{MM}_{DD}' + +env: + target_schema: '${PROD_SCHEMA}' # From environment +``` + +### Global Sling Variables + +Set via `env` section or environment: + +```yaml +env: + SLING_THREADS: 10 # Parallel stream processing + SLING_RETRIES: 2 # Retry failed streams + SLING_LOADED_AT_COLUMN: true # Add _sling_loaded_at timestamp + SLING_STREAM_URL_COLUMN: true # Add _sling_stream_url for files +``` + +--- + +## 10. Wildcards and Patterns + +### Schema Wildcards + +Match all tables in a schema: + +```yaml +streams: + # All tables in public schema + public.*: + + # All tables in multiple schemas with prefix/suffix + sales.marketing_*: # use a prefix + inventory.*_latest: # use a suffix + + # Exclude specific tables + public.*: + public.temp_table: + disabled: true +``` + +### File Wildcards + +Match multiple files: + +```yaml +streams: + # All CSV files in directory + 'data/*.csv': + + # All JSON files recursively + 'data/**/*.json': + + # Specific pattern + 'logs/access_log_*.txt': + + # Multiple patterns + '*.csv': + '*.xlsx': +``` + +### Pattern Matching Examples + +```yaml +source: POSTGRES +target: SNOWFLAKE + +defaults: + mode: incremental + object: 'warehouse.{stream_schema}_{stream_table}' + +streams: + # All user-related tables + public.user_*: + primary_key: [user_id] + + # All tables except sensitive ones + public.*: + public.passwords: + disabled: true + public.credit_cards: + disabled: true + + # Specific tables with custom config + public.large_table: + source_options: + chunk_size: 12h +``` + +### Single Stream Mode + +Treat wildcards as single stream: + +```yaml +streams: + # Process all files as one stream + 'logs/*.json': + single: true + object: combined_logs + source_options: + format: jsonlines +``` + +--- + +## 11. Hooks and Automation + +### Hook Types + +#### Replication-Level Hooks +Execute at replication start/end: + +```yaml +hooks: + start: + - type: command + command: 'echo "Starting replication at $(date)"' + + - type: query + connection: MY_DB + sql: 'UPDATE job_status SET status = "running" WHERE job_id = "repl_001"' + + end: + - type: command + command: 'echo "Replication completed at $(date)"' +``` + +#### Stream-Level Hooks +Execute before/after each stream: + +```yaml +defaults: + hooks: + pre: + - type: query + connection: TARGET_DB + sql: 'CREATE SCHEMA IF NOT EXISTS {target_schema}' + + post: + - type: query + connection: TARGET_DB + sql: 'ANALYZE TABLE {object_name}' + +streams: + important_table: + hooks: + post: + - type: command + command: 'python validate_data.py {object_name}' +``` + +### Available Hook Types + +See https://docs.slingdata.io/concepts/hooks for complete list. + +- `command` - Execute shell commands +- `query` - Run SQL queries +- `http` - Make HTTP requests +- `copy` - Copy files +- `delete` - Delete files +- `write` - Write content to files + +### Hook Examples + +```yaml +hooks: + start: + # Notification + - type: http + url: 'https://slack.com/api/chat.postMessage' + method: POST + headers: + Authorization: 'Bearer ${SLACK_TOKEN}' + body: | + { + "channel": "#data-pipeline", + "text": "Starting data replication" + } + + end: + # Cleanup + - type: command + command: 'rm -f /tmp/temp_files_*' + + # Update status + - type: query + connection: METADATA_DB + sql: | + INSERT INTO replication_log (job_id, status, completed_at) + VALUES ('repl_001', 'completed', NOW()) +``` + +--- + +## 12. Examples - Database to Database + +### Full Refresh Replication + +```yaml +source: POSTGRES_PROD +target: SNOWFLAKE_DW + +defaults: + mode: full-refresh + object: 'analytics.{stream_schema}_{stream_table}' + target_options: + column_casing: snake + +streams: + # Single table + public.customers: + + # All sales tables + sales.*: + object: 'sales_data.{stream_table}' + + # Large table with chunking + public.transactions: + source_options: + chunk_count: 10 + +env: + SLING_THREADS: 5 + SLING_LOADED_AT_COLUMN: true +``` + +### Incremental Replication + +```yaml +source: MYSQL_APP +target: POSTGRES_DW + +defaults: + mode: incremental + object: 'warehouse.{stream_table}' + primary_key: [id] + update_key: updated_at + + source_options: + empty_as_null: false + + target_options: + column_casing: snake + add_new_columns: true + +streams: + # Standard incremental + app.users: + app.orders: + primary_key: [order_id] + + # Append-only (no primary key) + app.events: + primary_key: [] + update_key: created_at + + # Custom SQL with incremental + app.user_summary: + sql: | + SELECT + user_id, + COUNT(*) as order_count, + MAX(created_at) as last_order_date + FROM orders + WHERE updated_at > coalesce({incremental_value}, '1900-01-01') + GROUP BY user_id + update_key: last_order_date + +env: + SLING_THREADS: 3 + SLING_RETRIES: 2 +``` + +### Backfill Historical Data + +```yaml +source: POSTGRES_OLD +target: BIGQUERY_DW + +defaults: + mode: backfill + object: 'historical.{stream_table}' + update_key: created_date + + source_options: + range: '2020-01-01,2023-12-31' + chunk_size: 30d # 30-day chunks + +streams: + transactions: + primary_key: [transaction_id] + + user_activity: + source_options: + chunk_size: 7d # Smaller chunks for large table + +env: + SLING_THREADS: 2 # Conservative for historical loads +``` + +### Multi-Schema Replication + +```yaml +source: ORACLE_ERP +target: REDSHIFT_DW + +defaults: + mode: incremental + primary_key: [id] + update_key: last_modified + object: '{stream_schema}_data.{stream_table}' + + target_options: + column_casing: lower + table_keys: + sort: [id, last_modified] + +streams: + # Finance schema + finance.*: + target_options: + table_keys: + sort: [account_id, transaction_date] + + # HR schema + hr.*: + object: 'human_resources.{stream_table}' + + # Sensitive table exclusions + hr.salaries: + disabled: true + finance.audit_log: + disabled: true + +env: + SLING_THREADS: 8 +``` + +--- + +## 13. Examples - File to Database + +### CSV Import + +```yaml +source: LOCAL +target: POSTGRES + +defaults: + mode: full-refresh + object: 'staging.{stream_file_name}' + + source_options: + format: csv + header: true + encoding: utf8 + empty_as_null: true + skip_blank_lines: true + +streams: + 'data/customers.csv': + columns: + customer_id: bigint + email: string(255) + created_at: datetime + + 'data/products.csv': + target_options: + column_casing: snake + + # All CSV files in directory + 'imports/*.csv': + +env: + SLING_THREADS: 3 +``` + +### JSON File Processing + +```yaml +source: S3_BUCKET +target: SNOWFLAKE + +defaults: + mode: incremental + object: 'json_data.{stream_file_name}' + primary_key: [id] + + source_options: + format: json + flatten: true + jmespath: 'data[*]' + +streams: + # API response files + 's3://bucket/api_responses/*.json': + transforms: + - id: 'cast(value, "string")' + timestamp: 'date_parse(value, "2006-01-02T15:04:05Z")' + + # Nested JSON structure + 's3://bucket/complex_data.json': + source_options: + jmespath: 'results[*].{id: id, name: name, details: metadata}' + +env: + SLING_THREADS: 5 +``` + +### Excel File Import + +```yaml +source: LOCAL +target: MYSQL + +defaults: + mode: truncate + object: 'excel_imports.{stream_file_name}' + + source_options: + format: xlsx + header: true + +streams: + 'reports/monthly_sales.xlsx': + source_options: + sheet: 'Sales Data' + + 'reports/inventory.xlsx': + source_options: + sheet: 'Current Stock!A1:H1000' + columns: + quantity: integer + price: decimal(10,2) + +env: + SLING_THREADS: 2 +``` + +### Incremental File Loading + +```yaml +source: SFTP_SERVER +target: POSTGRES + +defaults: + mode: incremental + object: 'file_data.{stream_file_name}' + update_key: file_date + primary_key: [record_id] + + source_options: + format: csv + header: true + +streams: + # Daily files with date in filename + 'daily_reports/sales_*.csv': + transforms: + - file_date: 'date_parse(record.filename_date, "2006-01-02")' + record_id: 'record.id + "_" + record.filename_date' + +env: + SLING_STREAM_URL_COLUMN: true # Track source file + SLING_THREADS: 3 +``` + +### Parquet File Processing + +```yaml +source: AZURE_BLOB +target: BIGQUERY + +defaults: + mode: full-refresh + object: 'parquet_data.{stream_file_name}' + + source_options: + format: parquet + compression: auto + +streams: + # Partitioned parquet files + 'data/year=*/month=*/*.parquet': + transforms: + - year: 'int_parse(substring(record._file_path, strpos(record._file_path, "year=") + 5, 4))' + month: 'int_parse(substring(record._file_path, strpos(record._file_path, "month=") + 6, 2))' + +env: + SLING_THREADS: 10 +``` + +--- + +## 14. Examples - Database to File + +### CSV Export + +```yaml +source: POSTGRES +target: LOCAL + +defaults: + mode: full-refresh + object: 'exports/{stream_table}_{YYYY}_{MM}_{DD}.csv' + + target_options: + format: csv + header: true + encoding: utf8 + +streams: + public.customers: + select: [id, name, email, created_at] # Exclude sensitive data + + public.orders: + where: "created_at >= '2024-01-01'" # Filter data + + # Large table with file splitting + public.transactions: + target_options: + file_max_rows: 100000 # Split into multiple files + +env: + SLING_THREADS: 4 +``` + +### Parquet Export with Partitioning + +```yaml +source: SNOWFLAKE +target: S3_BUCKET + +defaults: + mode: full-refresh + update_key: created_date + object: 's3://bucket/data/{stream_table}/{part_year}/{part_month}/' + + target_options: + format: parquet + compression: snappy + file_max_rows: 1000000 + +streams: + warehouse.sales: + where: "created_date >= '2023-01-01'" + + warehouse.customers: + # No partitioning + object: 's3://bucket/reference/customers.parquet' + +env: + SLING_THREADS: 6 +``` + +### JSON Export + +```yaml +source: MONGODB +target: GCS + +defaults: + mode: full-refresh + object: 'gs://bucket/exports/{stream_table}_{ISO8601}.jsonlines' + + target_options: + format: jsonlines + compression: gzip + +streams: + # Collections + users: + orders: + products: + +env: + SLING_THREADS: 3 +``` + +### Incremental Export + +```yaml +source: MYSQL +target: AZURE_BLOB + +defaults: + mode: incremental + update_key: updated_at + object: 'exports/{stream_table}/year={part_year}/month={part_month}/' + + target_options: + format: parquet + file_max_rows: 500000 + +streams: + app.user_activities: + where: "updated_at >= '2024-01-01'" + + app.transactions: + primary_key: [transaction_id] + +env: + SLING_THREADS: 4 +``` + +--- + +## 15. Advanced Topics + +### Parallel Processing + +Control concurrency with `SLING_THREADS`: + +```yaml +env: + SLING_THREADS: 10 # Process up to 10 streams simultaneously +``` + +**Guidelines:** +- Database sources: 5-15 threads depending on connection capacity +- File sources: 10-50 threads depending on I/O capacity. Great for Cloud Storage. +- Consider target database connection limits + +### Error Handling and Retries + +```yaml +env: + SLING_RETRIES: 3 # Retry failed streams 3 times + SLING_RETRY_DELAY: 30s # Wait 30 seconds between retries +``` + +### Performance Optimization + +#### Direct Insert Mode +Skip temporary tables for append-only scenarios: + +```yaml +defaults: + target_options: + direct_insert: true # Only works with certain modes +``` + +#### Bulk Loading +Use external tools when available: + +```yaml +defaults: + target_options: + use_bulk: true # Uses bcp, sqlldr, mysql client +``` + +#### Chunking Large Tables +Split large tables for parallel processing: + +```yaml +streams: + large_table: + source_options: + chunk_count: 20 # Split into 20 chunks + # OR + chunk_size: 6h # Split by time intervals +``` + +### Advanced Transformations + +Complex business logic with staged transforms: + +```yaml +streams: + customer_analytics: + transforms: + # Stage 1: Data cleansing + - email: 'lower(trim_space(value))' + phone: 'replace(value, "[^0-9+]", "")' + name: 'trim_space(value)' + + # Stage 2: Computed metrics + - days_since_signup: 'date_diff(now(), record.created_at, "day")' + lifetime_value: 'coalesce(record.total_spent, 0) * 1.2' + + # Stage 3: Segmentation + - customer_segment: | + record.lifetime_value >= 10000 ? "enterprise" : ( + record.lifetime_value >= 1000 ? "professional" : ( + record.days_since_signup <= 30 ? "new" : "standard" + ) + ) + + # Stage 4: Risk scoring + - risk_score: | + (record.failed_payments * 0.3) + + (record.days_since_last_login * 0.1) + + (record.support_tickets * 0.2) +``` + +--- + +## 16. MCP Tool Integration + +### Using Replication Commands + +#### Get Documentation +```json +{ + "action": "docs", + "input": {} +} +``` + +#### Parse Configuration +```json +{ + "action": "parse", + "input": { + "file_path": "/path/to/replication.yaml", + "working_dir": "/optional/work/dir" + } +} +``` + +#### Validate Configuration +```json +{ + "action": "compile", + "input": { + "file_path": "/path/to/replication.yaml", + "select_streams": ["table1", "table2"], // Optional + "working_dir": "/optional/work/dir" + } +} +``` + +#### Execute Replication +```json +{ + "action": "run", + "input": { + "file_path": "/path/to/replication.yaml", + "select_streams": ["specific_table"], // Optional: run specific streams + "working_dir": "/project/directory", // Optional: change directory + "range": "2024-01-01,2024-01-31", // Optional: backfill range + "mode": "incremental", // Optional: override mode + "env": { // Optional: environment variables + "CUSTOM_VAR": "value", + "SLING_THREADS": "5" + } + } +} +``` + +### Combining with Connection Tools + +First check connections: +```json +{ + "action": "list", + "input": {} +} +``` + +Then test connections: +```json +{ + "action": "test", + "input": { + "connection": "MY_SOURCE_DB", + "debug": true + } +} +``` + +Finally run replication: +```json +{ + "action": "run", + "input": { + "file_path": "/path/to/replication.yaml" + } +} +``` + +### Development Workflow + +1. **Parse** configuration for syntax validation +2. **Compile** to check connections and streams +3. **Test** on subset of streams first +4. **Run** full replication +5. **Monitor** with debug/trace options + +--- + +## 17. Best Practices + +### Configuration Organization + +#### Use Meaningful Names +```yaml +# Good +source: prod_postgres_crm +target: dw_snowflake_analytics + +# Avoid +source: db1 +target: db2 +``` + +#### Structure with Defaults +```yaml +# Define common patterns in defaults +defaults: + mode: incremental + object: 'warehouse.{stream_schema}_{stream_table}' + primary_key: [id] + update_key: updated_at + + target_options: + column_casing: snake + add_new_columns: true + +# Override only when necessary +streams: + public.users: # Uses all defaults + + public.logs: # Override for specific needs + mode: full-refresh + primary_key: [] +``` + +#### Organize by Purpose +```yaml +# Group related streams +streams: + # Core business data + public.customers: + public.orders: + public.products: + + # Analytics data + analytics.user_metrics: + analytics.sales_summary: + + # Logging data + logs.application_logs: + mode: full-refresh +``` + +### Performance Best Practices + +#### Optimize Thread Usage +- Start with 5 threads and adjust based on performance +- Database sources: Monitor connection pool limits +- File sources: Consider I/O capacity + +#### Use Appropriate Modes +- `full-refresh`: Small, frequently changing tables +- `incremental`: Large tables with clear update patterns +- `truncate`: When you need to preserve table structure + +#### Leverage Chunking +```yaml +streams: + large_table: + source_options: + chunk_size: 12h # For time-based data + # OR + chunk_count: 10 # For evenly distributed data +``` + +### Security Considerations + +#### Sensitive Data Handling +```yaml +streams: + users: + select: [-password, -ssn, -credit_card] # Exclude sensitive columns + transforms: + - email: 'hash(value, "sha256")' # Hash PII +``` + +#### Connection Security +- Store connection strings in environment variables +- Use secure authentication methods +- Rotate credentials regularly + +#### Data Encryption +```yaml +target_options: + encryption: true # Enable if supported by target + compression: gzip # Compress sensitive data +``` + +### Monitoring and Observability + +#### Add Metadata Columns +```yaml +env: + SLING_LOADED_AT_COLUMN: true # Track load timestamps + SLING_STREAM_URL_COLUMN: true # Track source file paths (for file sources) +``` + +#### Use Hooks for Monitoring +```yaml +hooks: + start: + - type: http + url: 'https://monitoring.company.com/api/jobs/start' + body: '{"job": "data_replication", "status": "started"}' + + end: + - type: http + url: 'https://monitoring.company.com/api/jobs/complete' + body: '{"job": "data_replication", "status": "completed"}' +``` + +### Testing Strategies + +#### Test with Subsets First +```json +{ + "action": "run", + "input": { + "file_path": "/path/to/replication.yaml", + "select_streams": ["small_test_table"] + } +} +``` + +#### Use Development Targets +```yaml +# Development configuration +target: DEV_DATABASE + +defaults: + object: 'dev_{stream_table}' +``` + +#### Validate Data Quality +```yaml +streams: + critical_table: + hooks: + post: + - type: query + connection: TARGET_DB + sql: | + SELECT + COUNT(*) as row_count, + COUNT(DISTINCT id) as unique_ids + FROM {object_name} + HAVING COUNT(*) < 1000 -- Fail if too few rows +``` + +--- + +## 18. Troubleshooting + +### Common Issues and Solutions + +#### Connection Errors +``` +Error: failed to connect to source database +``` + +**Solutions:** +1. Test connection first: +```json +{ + "action": "test", + "input": { + "connection": "MY_SOURCE_DB", + "debug": true + } +} +``` + +2. Check connection string format +3. Verify network access and firewall rules +4. Confirm authentication credentials + +#### Schema/Table Not Found +``` +Error: table "schema.table" does not exist +``` + +**Solutions:** +1. List available streams: +```json +{ + "action": "discover", + "input": { + "connection": "MY_SOURCE_DB", + "pattern": "schema.*" + } +} +``` + +2. Check case sensitivity +3. Verify permissions + +#### Type Conversion Errors +``` +Error: cannot convert value to target type +``` + +**Solutions:** +1. Add explicit type casting: +```yaml +columns: + problematic_column: string # Force to string first + +transforms: + - problematic_column: 'cast(value, "integer")' # Then convert +``` + +2. Handle null values: +```yaml +transforms: + - amount: 'coalesce(value, 0)' +``` + +#### Memory Issues +``` +Error: out of memory processing large dataset +``` + +**Solutions:** +1. Reduce batch size: +```yaml +env: + SLING_BATCH_SIZE: 10000 +``` + +2. Enable chunking: +```yaml +streams: + large_table: + source_options: + chunk_size: 6h +``` + +3. Use file splitting: +```yaml +target_options: + file_max_rows: 100000 +``` + +### Debug Options + +#### Enable Debug Logging +```json +{ + "action": "run", + "input": { + "file_path": "/path/to/replication.yaml", + "env": { + "DEBUG": "true" + } + } +} +``` + +#### Parse Configuration +Always validate before running: +```json +{ + "action": "parse", + "input": { + "file_path": "/path/to/replication.yaml" + } +} +``` + +#### Compile Configuration +Check connections and streams: +```json +{ + "action": "compile", + "input": { + "file_path": "/path/to/replication.yaml" + } +} +``` + +### Performance Issues + +#### Slow Performance +1. **Check thread usage:** +```yaml +env: + SLING_THREADS: 10 # Increase if system can handle +``` + +2. **Enable bulk loading:** +```yaml +target_options: + use_bulk: true +``` + +3. **Use direct insert for append-only:** +```yaml +target_options: + direct_insert: true # Skip temporary tables (if not incremental or backfill) +``` + +#### High Memory Usage +1. **Reduce batch sizes:** +```yaml +env: + SLING_BATCH_SIZE: 5000 +``` + +2. **Use file-based temporary storage:** +```yaml +target_options: + file_max_rows: 50000 +``` + +### Data Quality Issues + +#### Missing Data +1. **Check incremental logic:** +```yaml +streams: + my_table: + sql: | + SELECT * FROM my_table + WHERE updated_at > coalesce({incremental_value}, '1900-01-01') +``` + +2. **Verify primary/update keys:** +```yaml +streams: + my_table: + primary_key: [id] # Ensure exists and is unique + update_key: updated_at # Ensure has values +``` + +#### Duplicate Data +1. **Check primary key configuration:** +```yaml +streams: + my_table: + primary_key: [id, version] # Use composite if needed +``` + +2. **Use appropriate mode:** +```yaml +streams: + my_table: + mode: incremental # Not full-refresh for updates +``` + +### Error Recovery + +#### Resume Failed Replications +Sling automatically tracks progress, but you can: + +1. **Run specific streams:** +```json +{ + "action": "run", + "input": { + "file_path": "/path/to/replication.yaml", + "select_streams": ["failed_stream"] + } +} +``` + +2. **Use retry configuration:** +```yaml +env: + SLING_RETRIES: 3 + SLING_RETRY_DELAY: 60s +``` + +This comprehensive guide provides everything needed to effectively use Sling replications with the MCP tool, from basic concepts to advanced troubleshooting techniques. \ No newline at end of file From 60ea9bcfa3032a26396d6c7ba1e0a745d9b2b96a Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 16:57:35 -0300 Subject: [PATCH 29/37] fix: update flarco/g dependency version to v0.1.157 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 478c91971..40393010a 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/exasol/exasol-driver-go v1.0.14 github.com/fatih/color v1.18.0 github.com/flarco/bigquery v0.0.9 - github.com/flarco/g v0.1.156 + github.com/flarco/g v0.1.157 github.com/getsentry/sentry-go v0.27.0 github.com/go-sql-driver/mysql v1.9.1 github.com/gobwas/glob v0.2.3 From 69adfe4b3624d3a683f990f490481d91321abdc0 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 18:03:14 -0300 Subject: [PATCH 30/37] fix: enhance GetLocalConns to handle URL parsing from payload --- core/dbio/connection/connection_local.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/dbio/connection/connection_local.go b/core/dbio/connection/connection_local.go index 7b02ef90c..011e03114 100644 --- a/core/dbio/connection/connection_local.go +++ b/core/dbio/connection/connection_local.go @@ -202,6 +202,14 @@ func GetLocalConns(options ...any) ConnEntries { g.Warn(e) continue } + } else if url, ok := payload["url"]; ok && err == nil { + cType := SchemeType(cast.ToString(url)) + conn, err = NewConnectionFromMap(g.M("name", key, "type", cType, "data", payload)) + if err != nil { + e := g.F("could not parse env payload %s: %s", key, g.ErrMsgSimple(err)) + g.Warn(e) + continue + } } else { // Parse URL if !strings.Contains(val, "://") || strings.Contains(val, "{") || strings.Contains(val, "[") { From cf879cf05f19f878e5a66441e7eeaa0115201715 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 18:42:11 -0300 Subject: [PATCH 31/37] refactor: replace interface{} with any --- core/dbio/iop/dataset.go | 36 +++++++------- core/dbio/iop/datatype_test.go | 26 +++++----- core/dbio/iop/duckdb.go | 12 ++--- core/dbio/iop/json.go | 82 +++++++++++++++---------------- core/dbio/iop/parquet.go | 2 +- core/dbio/iop/queue_test.go | 12 ++--- core/dbio/iop/stream_processor.go | 50 +++++++++---------- 7 files changed, 110 insertions(+), 110 deletions(-) diff --git a/core/dbio/iop/dataset.go b/core/dbio/iop/dataset.go index ab87f1088..4f0b318fb 100644 --- a/core/dbio/iop/dataset.go +++ b/core/dbio/iop/dataset.go @@ -37,7 +37,7 @@ func NewDataset(columns Columns) (data Dataset) { data = Dataset{ Result: nil, Columns: NewColumns(columns...), - Rows: [][]interface{}{}, + Rows: [][]any{}, Sp: NewStreamProcessor(), } @@ -45,24 +45,24 @@ func NewDataset(columns Columns) (data Dataset) { } // NewDatasetFromMap return a new dataset -func NewDatasetFromMap(m map[string]interface{}) (data Dataset) { +func NewDatasetFromMap(m map[string]any) (data Dataset) { data = Dataset{ Result: nil, Columns: Columns{}, - Rows: [][]interface{}{}, + Rows: [][]any{}, Sp: NewStreamProcessor(), } if fieldsI, ok := m["headers"]; ok { fields := []string{} - for _, f := range fieldsI.([]interface{}) { + for _, f := range fieldsI.([]any) { fields = append(fields, cast.ToString(f)) } data.Columns = NewColumnsFromFields(fields...) } if rowsI, ok := m["rows"]; ok { - for _, rowI := range rowsI.([]interface{}) { - data.Rows = append(data.Rows, rowI.([]interface{})) + for _, rowI := range rowsI.([]any) { + data.Rows = append(data.Rows, rowI.([]any)) } } @@ -303,7 +303,7 @@ func (data *Dataset) Stream(Props ...map[string]string) *Datastream { } // FirstVal returns the first value from the first row -func (data *Dataset) FirstVal() interface{} { +func (data *Dataset) FirstVal() any { if len(data.Rows) > 0 && len(data.Rows[0]) > 0 { return data.Rows[0][0] } @@ -311,7 +311,7 @@ func (data *Dataset) FirstVal() interface{} { } // FirstRow returns the first row -func (data *Dataset) FirstRow() []interface{} { +func (data *Dataset) FirstRow() []any { if len(data.Rows) > 0 { return data.Rows[0] } @@ -319,8 +319,8 @@ func (data *Dataset) FirstRow() []interface{} { } // ColValues returns the values of a one column as array -func (data *Dataset) ColValues(col int) []interface{} { - vals := make([]interface{}, len(data.Rows)) +func (data *Dataset) ColValues(col int) []any { + vals := make([]any, len(data.Rows)) for i, row := range data.Rows { vals[i] = row[col] } @@ -354,14 +354,14 @@ func (data *Dataset) ColValuesStr(col int) []string { } // Records return rows of maps -func (data *Dataset) Records(lower ...bool) []map[string]interface{} { +func (data *Dataset) Records(lower ...bool) []map[string]any { Lower := true if len(lower) > 0 { Lower = lower[0] } - records := make([]map[string]interface{}, len(data.Rows)) + records := make([]map[string]any, len(data.Rows)) for i, row := range data.Rows { - rec := map[string]interface{}{} + rec := map[string]any{} for j, field := range data.GetFields(Lower) { rec[field] = row[j] } @@ -392,14 +392,14 @@ func (data *Dataset) RecordsString(lower ...bool) []map[string]string { } // RecordsCasted return rows of maps or casted values -func (data *Dataset) RecordsCasted(lower ...bool) []map[string]interface{} { +func (data *Dataset) RecordsCasted(lower ...bool) []map[string]any { Lower := true if len(lower) > 0 { Lower = lower[0] } - records := make([]map[string]interface{}, len(data.Rows)) + records := make([]map[string]any, len(data.Rows)) for i, row := range data.Rows { - rec := map[string]interface{}{} + rec := map[string]any{} for j, field := range data.GetFields(Lower) { if row[j] == nil { rec[field] = nil @@ -413,7 +413,7 @@ func (data *Dataset) RecordsCasted(lower ...bool) []map[string]interface{} { } // ToJSONMap converst to a JSON object -func (data *Dataset) ToJSONMap() map[string]interface{} { +func (data *Dataset) ToJSONMap() map[string]any { return g.M("headers", data.GetFields(), "rows", data.Rows) } @@ -535,7 +535,7 @@ func (data *Dataset) InferColumnTypes() { } if looksLikeJson(valStr) { - var v interface{} + var v any if err := g.Unmarshal(valStr, &v); err == nil { columns[j].Stats.JsonCnt++ } else { diff --git a/core/dbio/iop/datatype_test.go b/core/dbio/iop/datatype_test.go index 057630b3c..439bfd8b5 100755 --- a/core/dbio/iop/datatype_test.go +++ b/core/dbio/iop/datatype_test.go @@ -53,28 +53,28 @@ func BenchmarkParseString8Bool(b *testing.B) { bParseString(sp, "true", b) } -func initProcessRow(name string) (columns []Column, row []interface{}) { +func initProcessRow(name string) (columns []Column, row []any) { if name == "float" { - row = []interface{}{5535.21414} + row = []any{5535.21414} } else if name == "decimal" { - row = []interface{}{[]byte{0x31, 0x30, 0x33, 0x32, 0x2e, 0x34, 0x34, 0x32, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30}} - row = []interface{}{"5535.214140000"} + row = []any{[]byte{0x31, 0x30, 0x33, 0x32, 0x2e, 0x34, 0x34, 0x32, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30}} + row = []any{"5535.214140000"} } else if name == "int" { - row = []interface{}{int(48714719874194)} + row = []any{int(48714719874194)} } else if name == "int64" { - row = []interface{}{int64(48714719874194)} + row = []any{int64(48714719874194)} } else if name == "string" { - row = []interface{}{g.RandString(g.AlphaNumericRunes, 1000)} + row = []any{g.RandString(g.AlphaNumericRunes, 1000)} } else if name == "timestamp" || name == "timestampz" { - row = []interface{}{time.Now()} - // row = []interface{}{"17-OCT-20 07.01.59.000000 PM"} + row = []any{time.Now()} + // row = []any{"17-OCT-20 07.01.59.000000 PM"} } else if name == "bool" { - row = []interface{}{false} + row = []any{false} } else if name == "blank" { - row = []interface{}{""} + row = []any{""} } else { - row = []interface{}{ + row = []any{ "fritz", "larco", 55, 563525, 5535.21414, true, time.Now(), } } @@ -285,7 +285,7 @@ func BenchmarkIsUTC(b *testing.B) { } func TestInterfVal(t *testing.T) { - row := make([]interface{}, 3) + row := make([]any, 3) g.P(row[0]) row[0] = float64(0) g.P(row[0]) diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index b12f0dc51..d9d700a9e 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -472,7 +472,7 @@ func (duck *DuckDb) Close() error { } // Exec executes a SQL query and returns the result -func (duck *DuckDb) Exec(sql string, args ...interface{}) (result sql.Result, err error) { +func (duck *DuckDb) Exec(sql string, args ...any) (result sql.Result, err error) { result, err = duck.ExecContext(duck.Context.Ctx, sql, args...) if err != nil { @@ -571,7 +571,7 @@ func (duck *DuckDb) closeStdinAndWait() (err error) { } // ExecContext executes a SQL query with context and returns the result -func (duck *DuckDb) ExecContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error) { +func (duck *DuckDb) ExecContext(ctx context.Context, sql string, args ...any) (result sql.Result, err error) { if !cast.ToBool(duck.GetProp("connected")) { if err = duck.Open(); err != nil { return nil, g.Error(err, "Could not open DuckDB connection") @@ -697,12 +697,12 @@ func (duck *DuckDb) waitForResult(dq *duckDbQuery) (result sql.Result, err error } // Query runs a sql query, returns `Dataset` -func (duck *DuckDb) Query(sql string, options ...map[string]interface{}) (data Dataset, err error) { +func (duck *DuckDb) Query(sql string, options ...map[string]any) (data Dataset, err error) { return duck.QueryContext(context.Background(), sql, options...) } // QueryContext runs a sql query with context, returns `Dataset` -func (duck *DuckDb) QueryContext(ctx context.Context, sql string, options ...map[string]interface{}) (data Dataset, err error) { +func (duck *DuckDb) QueryContext(ctx context.Context, sql string, options ...map[string]any) (data Dataset, err error) { ds, err := duck.StreamContext(ctx, sql, options...) if err != nil { @@ -720,12 +720,12 @@ func (duck *DuckDb) QueryContext(ctx context.Context, sql string, options ...map } // Stream runs a sql query, returns `Datastream` -func (duck *DuckDb) Stream(sql string, options ...map[string]interface{}) (ds *Datastream, err error) { +func (duck *DuckDb) Stream(sql string, options ...map[string]any) (ds *Datastream, err error) { return duck.StreamContext(duck.Context.Ctx, sql, options...) } // StreamContext runs a sql query with context, returns `Datastream` -func (duck *DuckDb) StreamContext(ctx context.Context, sql string, options ...map[string]interface{}) (ds *Datastream, err error) { +func (duck *DuckDb) StreamContext(ctx context.Context, sql string, options ...map[string]any) (ds *Datastream, err error) { if !cast.ToBool(duck.GetProp("connected")) { if err = duck.Open(); err != nil { return nil, g.Error(err, "Could not open DuckDB connection") diff --git a/core/dbio/iop/json.go b/core/dbio/iop/json.go index a78249436..bba59f0b6 100644 --- a/core/dbio/iop/json.go +++ b/core/dbio/iop/json.go @@ -26,7 +26,7 @@ type jsonStream struct { decoder decoderLike jmespath string flatten int - buffer chan []interface{} + buffer chan []any } func NewJSONStream(ds *Datastream, decoder decoderLike, flatten int, jmespath string) *jsonStream { @@ -36,7 +36,7 @@ func NewJSONStream(ds *Datastream, decoder decoderLike, flatten int, jmespath st decoder: decoder, flatten: flatten, jmespath: jmespath, - buffer: make(chan []interface{}, 100000), + buffer: make(chan []any, 100000), sp: NewStreamProcessor(), } if flatten < 0 { @@ -55,7 +55,7 @@ func NewJSONStream(ds *Datastream, decoder decoderLike, flatten int, jmespath st } func (js *jsonStream) NextFunc(it *Iterator) bool { - var recordsInterf []map[string]interface{} + var recordsInterf []map[string]any var err error if it.Closed { return false @@ -68,7 +68,7 @@ func (js *jsonStream) NextFunc(it *Iterator) bool { default: } - var payload interface{} + var payload any if js.HasMapPayload { m := g.M() err = js.decoder.Decode(&m) @@ -93,58 +93,58 @@ func (js *jsonStream) NextFunc(it *Iterator) bool { } switch payloadV := payload.(type) { - case map[string]interface{}: + case map[string]any: // is one record recordsInterf = js.extractNestedArray(payloadV) if len(recordsInterf) == 0 { - recordsInterf = []map[string]interface{}{payloadV} + recordsInterf = []map[string]any{payloadV} } - case map[interface{}]interface{}: + case map[any]any: // is one record - interf := map[string]interface{}{} + interf := map[string]any{} for k, v := range payloadV { interf[cast.ToString(k)] = v } recordsInterf = js.extractNestedArray(interf) if len(recordsInterf) == 0 { - recordsInterf = []map[string]interface{}{interf} + recordsInterf = []map[string]any{interf} } - case []interface{}: - recordsInterf = []map[string]interface{}{} + case []any: + recordsInterf = []map[string]any{} recList := payloadV if len(recList) == 0 { return js.NextFunc(it) } switch recList[0].(type) { - case map[interface{}]interface{}: + case map[any]any: for _, rec := range recList { - newRec := map[string]interface{}{} - for k, v := range rec.(map[interface{}]interface{}) { + newRec := map[string]any{} + for k, v := range rec.(map[any]any) { newRec[cast.ToString(k)] = v } recordsInterf = append(recordsInterf, newRec) } - case map[string]interface{}: + case map[string]any: for _, val := range recList { - recordsInterf = append(recordsInterf, val.(map[string]interface{})) + recordsInterf = append(recordsInterf, val.(map[string]any)) } default: // is array of single values for _, val := range recList { - recordsInterf = append(recordsInterf, map[string]interface{}{"data": val}) + recordsInterf = append(recordsInterf, map[string]any{"data": val}) } } - case []map[interface{}]interface{}: - recordsInterf = []map[string]interface{}{} + case []map[any]any: + recordsInterf = []map[string]any{} for _, rec := range payloadV { - newRec := map[string]interface{}{} + newRec := map[string]any{} for k, v := range rec { newRec[cast.ToString(k)] = v } recordsInterf = append(recordsInterf, newRec) } - case []map[string]interface{}: + case []map[string]any: recordsInterf = payloadV default: err = g.Error("unhandled JSON interface type: %#v", payloadV) @@ -180,11 +180,11 @@ func (js *jsonStream) addColumn(cols ...Column) { mux.Unlock() } -func (js *jsonStream) parseRecords(records []map[string]interface{}) { +func (js *jsonStream) parseRecords(records []map[string]any) { for _, rec := range records { if js.flatten < 0 { - js.buffer <- []interface{}{g.Marshal(rec)} + js.buffer <- []any{g.Marshal(rec)} continue } @@ -192,11 +192,11 @@ func (js *jsonStream) parseRecords(records []map[string]interface{}) { keys := lo.Keys(newRec) sort.Strings(keys) - row := make([]interface{}, len(js.ds.Columns)) + row := make([]any, len(js.ds.Columns)) colsToAdd := Columns{} for _, colName := range keys { // cast arrays as string - if arr, ok := newRec[colName].([]interface{}); ok { + if arr, ok := newRec[colName].([]any); ok { newRec[colName] = g.Marshal(arr) } @@ -224,12 +224,12 @@ func (js *jsonStream) parseRecords(records []map[string]interface{}) { // g.Debug("JSON Stream -> Parsed %d records", len(records)) } -func (js *jsonStream) extractNestedArray(rec map[string]interface{}) (recordsInterf []map[string]interface{}) { +func (js *jsonStream) extractNestedArray(rec map[string]any) (recordsInterf []map[string]any) { if js.flatten < 0 { - return []map[string]interface{}{rec} + return []map[string]any{rec} } - recordsInterf = []map[string]interface{}{} + recordsInterf = []map[string]any{} sliceKeyValLen := map[string]int{} maxLen := 0 @@ -247,7 +247,7 @@ func (js *jsonStream) extractNestedArray(rec map[string]interface{}) (recordsInt return sliceKeyValLen[k] == maxLen }) - var payload interface{} + var payload any for _, key := range keys { // have predefined list for now switch strings.ToLower(key) { @@ -257,42 +257,42 @@ func (js *jsonStream) extractNestedArray(rec map[string]interface{}) (recordsInt } switch payloadV := payload.(type) { - case []interface{}: - recordsInterf = []map[string]interface{}{} + case []any: + recordsInterf = []map[string]any{} recList := payloadV if len(recList) == 0 { return } switch recList[0].(type) { - case map[interface{}]interface{}: + case map[any]any: for _, rec := range recList { - newRec := map[string]interface{}{} - for k, v := range rec.(map[interface{}]interface{}) { + newRec := map[string]any{} + for k, v := range rec.(map[any]any) { newRec[cast.ToString(k)] = v } recordsInterf = append(recordsInterf, newRec) } - case map[string]interface{}: + case map[string]any: for _, val := range recList { - recordsInterf = append(recordsInterf, val.(map[string]interface{})) + recordsInterf = append(recordsInterf, val.(map[string]any)) } default: // is array of single values for _, val := range recList { - recordsInterf = append(recordsInterf, map[string]interface{}{"data": val}) + recordsInterf = append(recordsInterf, map[string]any{"data": val}) } } - case []map[interface{}]interface{}: - recordsInterf = []map[string]interface{}{} + case []map[any]any: + recordsInterf = []map[string]any{} for _, rec := range payloadV { - newRec := map[string]interface{}{} + newRec := map[string]any{} for k, v := range rec { newRec[cast.ToString(k)] = v } recordsInterf = append(recordsInterf, newRec) } - case []map[string]interface{}: + case []map[string]any: recordsInterf = payloadV } diff --git a/core/dbio/iop/parquet.go b/core/dbio/iop/parquet.go index 4e98fda98..b48d964f1 100644 --- a/core/dbio/iop/parquet.go +++ b/core/dbio/iop/parquet.go @@ -94,7 +94,7 @@ func (p *Parquet) nextFunc(it *Iterator) bool { return false } - it.Row = make([]interface{}, len(it.ds.Columns)) + it.Row = make([]any, len(it.ds.Columns)) for k, v := range row { col := it.ds.Columns[p.colMap[strings.ToLower(k)]] i := col.Position - 1 diff --git a/core/dbio/iop/queue_test.go b/core/dbio/iop/queue_test.go index bacefe2f5..f647c67b7 100644 --- a/core/dbio/iop/queue_test.go +++ b/core/dbio/iop/queue_test.go @@ -34,10 +34,10 @@ func TestQueue_WriteAndRead(t *testing.T) { assert.NoError(t, err) // Test writing json-serializable data - testData := map[string]interface{}{ + testData := map[string]any{ "name": "test", "value": 123, - "nested": map[string]interface{}{ + "nested": map[string]any{ "key": "value", }, } @@ -64,13 +64,13 @@ func TestQueue_WriteAndRead(t *testing.T) { assert.True(t, hasMore) // Should be our map data - mapItem, ok := item.(map[string]interface{}) + mapItem, ok := item.(map[string]any) assert.True(t, ok) assert.Equal(t, "test", mapItem["name"]) assert.Equal(t, float64(123), mapItem["value"]) // JSON numbers are floats // Check nested map - nestedMap, ok := mapItem["nested"].(map[string]interface{}) + nestedMap, ok := mapItem["nested"].(map[string]any) assert.True(t, ok) assert.Equal(t, "value", nestedMap["key"]) @@ -249,7 +249,7 @@ func TestQueue_LargeDataVolume(t *testing.T) { // Generate a larger dataset const itemCount = 1000 for i := 0; i < itemCount; i++ { - data := map[string]interface{}{ + data := map[string]any{ "id": i, "value": "This is test value " + string(rune(i%26+65)), // A-Z repeating } @@ -270,7 +270,7 @@ func TestQueue_LargeDataVolume(t *testing.T) { break } - mapItem, ok := item.(map[string]interface{}) + mapItem, ok := item.(map[string]any) assert.True(t, ok) id := int(mapItem["id"].(float64)) diff --git a/core/dbio/iop/stream_processor.go b/core/dbio/iop/stream_processor.go index 720731105..8cf96d908 100644 --- a/core/dbio/iop/stream_processor.go +++ b/core/dbio/iop/stream_processor.go @@ -36,7 +36,7 @@ type StreamProcessor struct { unrecognizedDate string warn bool skipCurrent bool // whether to skip current row (for constraints) - parseFuncs map[string]func(s string) (interface{}, error) + parseFuncs map[string]func(s string) (any, error) decReplRegex *regexp.Regexp ds *Datastream dateLayouts []string @@ -211,24 +211,24 @@ func NewStreamProcessor() *StreamProcessor { // if val is '0400', '0401'. Such as codes. hasZeroPrefix := func(s string) bool { return len(s) >= 2 && s[0] == '0' && s[1] != '.' } - sp.parseFuncs = map[string]func(s string) (interface{}, error){ - "int": func(s string) (interface{}, error) { + sp.parseFuncs = map[string]func(s string) (any, error){ + "int": func(s string) (any, error) { if hasZeroPrefix(s) { return s, errors.New("number has zero prefix, treat as string") } // return fastfloat.ParseInt64(s) return strconv.ParseInt(s, 10, 64) }, - "float": func(s string) (interface{}, error) { + "float": func(s string) (any, error) { if hasZeroPrefix(s) { return s, errors.New("number has zero prefix, treat as string") } return strconv.ParseFloat(strings.Replace(s, ",", ".", 1), 64) }, - "time": func(s string) (interface{}, error) { + "time": func(s string) (any, error) { return sp.ParseTime(s) }, - "bool": func(s string) (interface{}, error) { + "bool": func(s string) (any, error) { return cast.ToBoolE(s) }, } @@ -462,7 +462,7 @@ func (sp *StreamProcessor) applyTransforms(transformsPayload string) { // Copyright 2011 The Go Authors. All rights reserved. // indirect returns the value, after dereferencing as many times // as necessary to reach the base type (or nil). -func (sp *StreamProcessor) indirect(a interface{}) interface{} { +func (sp *StreamProcessor) indirect(a any) any { if a == nil { return nil } @@ -477,7 +477,7 @@ func (sp *StreamProcessor) indirect(a interface{}) interface{} { return v.Interface() } -func (sp *StreamProcessor) toFloat64E(i interface{}) (float64, error) { +func (sp *StreamProcessor) toFloat64E(i any) (float64, error) { i = sp.indirect(i) switch s := i.(type) { @@ -529,8 +529,8 @@ func (sp *StreamProcessor) toFloat64E(i interface{}) (float64, error) { // CastType casts the type of an interface // CastType is used to cast the interface place holders? -func (sp *StreamProcessor) CastType(val interface{}, typ ColumnType) interface{} { - var nVal interface{} +func (sp *StreamProcessor) CastType(val any, typ ColumnType) any { + var nVal any switch { case typ.IsBinary(): @@ -558,7 +558,7 @@ func (sp *StreamProcessor) CastType(val interface{}, typ ColumnType) interface{} } // GetType returns the type of an interface -func (sp *StreamProcessor) GetType(val interface{}) (typ ColumnType) { +func (sp *StreamProcessor) GetType(val any) (typ ColumnType) { data := NewDataset(NewColumnsFromFields("col")) data.Append([]any{val}) if ds := sp.ds; ds != nil { @@ -999,7 +999,7 @@ func (sp *StreamProcessor) CastToStringE(val any) (valString string, err error) // CastToStringCSV to string. used for csv writing // slows processing down 5% with upstream CastRow or 35% without upstream CastRow -func (sp *StreamProcessor) CastToStringCSV(i int, val interface{}, valType ...ColumnType) string { +func (sp *StreamProcessor) CastToStringCSV(i int, val any, valType ...ColumnType) string { typ := ColumnType("") switch v := val.(type) { case time.Time: @@ -1059,7 +1059,7 @@ func (sp *StreamProcessor) CastToStringCSV(i int, val interface{}, valType ...Co } // CastToStringSafe to masks to count bytes (even safer) -func (sp *StreamProcessor) CastToStringSafeMask(i int, val interface{}, valType ...ColumnType) string { +func (sp *StreamProcessor) CastToStringSafeMask(i int, val any, valType ...ColumnType) string { typ := ColumnType("") switch v := val.(type) { case time.Time: @@ -1098,8 +1098,8 @@ func (sp *StreamProcessor) CastToStringSafeMask(i int, val interface{}, valType } // CastValWithoutStats casts the value without counting stats -func (sp *StreamProcessor) CastValWithoutStats(i int, val interface{}, typ ColumnType) interface{} { - var nVal interface{} +func (sp *StreamProcessor) CastValWithoutStats(i int, val any, typ ColumnType) any { + var nVal any if nil == val { return nil } @@ -1145,7 +1145,7 @@ func (sp *StreamProcessor) CastValWithoutStats(i int, val interface{}, typ Colum } // CastToBool converts interface to bool -func (sp *StreamProcessor) CastToBool(i interface{}) (b bool, err error) { +func (sp *StreamProcessor) CastToBool(i any) (b bool, err error) { i = sp.indirect(i) switch b := i.(type) { @@ -1169,7 +1169,7 @@ func (sp *StreamProcessor) CastToBool(i interface{}) (b bool, err error) { } // CastToTime converts interface to time -func (sp *StreamProcessor) CastToTime(i interface{}) (t time.Time, err error) { +func (sp *StreamProcessor) CastToTime(i any) (t time.Time, err error) { i = sp.indirect(i) switch v := i.(type) { case nil: @@ -1184,7 +1184,7 @@ func (sp *StreamProcessor) CastToTime(i interface{}) (t time.Time, err error) { } // ParseTime parses a date string and returns time.Time -func (sp *StreamProcessor) ParseTime(i interface{}) (t time.Time, err error) { +func (sp *StreamProcessor) ParseTime(i any) (t time.Time, err error) { if tv, ok := i.(time.Time); ok { return tv, nil } else if tv, ok := i.(*time.Time); ok && tv != nil { @@ -1228,7 +1228,7 @@ func (sp *StreamProcessor) ParseTime(i interface{}) (t time.Time, err error) { // datetime: "timestamp" // timestamp: "timestamp" // text: "text" -func (sp *StreamProcessor) ParseString(s string, jj ...int) interface{} { +func (sp *StreamProcessor) ParseString(s string, jj ...int) any { if s == "" { return nil } @@ -1285,8 +1285,8 @@ func (sp *StreamProcessor) ParseString(s string, jj ...int) interface{} { } // ProcessVal processes a value -func (sp *StreamProcessor) ProcessVal(val interface{}) interface{} { - var nVal interface{} +func (sp *StreamProcessor) ProcessVal(val any) any { + var nVal any switch v := val.(type) { case []uint8: nVal = cast.ToString(val) @@ -1299,8 +1299,8 @@ func (sp *StreamProcessor) ProcessVal(val interface{}) interface{} { } // ParseVal parses the value into its appropriate type -func (sp *StreamProcessor) ParseVal(val interface{}) interface{} { - var nVal interface{} +func (sp *StreamProcessor) ParseVal(val any) any { + var nVal any switch v := val.(type) { case time.Time: nVal = cast.ToTime(val) @@ -1334,7 +1334,7 @@ func (sp *StreamProcessor) ParseVal(val interface{}) interface{} { // CastRow casts each value of a row // slows down processing about 40%? -func (sp *StreamProcessor) CastRow(row []interface{}, columns Columns) []interface{} { +func (sp *StreamProcessor) CastRow(row []any, columns Columns) []any { sp.N++ // Ensure usable types sp.rowBlankValCnt = 0 @@ -1373,7 +1373,7 @@ func (sp *StreamProcessor) CastRow(row []interface{}, columns Columns) []interfa } // ProcessRow processes a row -func (sp *StreamProcessor) ProcessRow(row []interface{}) []interface{} { +func (sp *StreamProcessor) ProcessRow(row []any) []any { // Ensure usable types for i, val := range row { row[i] = sp.ProcessVal(val) From 3a59b9f8a96a613f7edcff0c905b664dc728f891 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 19:01:46 -0300 Subject: [PATCH 32/37] feat: add CheckType method to determine data types and enhance row processing --- core/dbio/iop/datastream.go | 21 ++++++++++ core/dbio/iop/stream_processor.go | 68 +++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/core/dbio/iop/datastream.go b/core/dbio/iop/datastream.go index 5f9c7b731..28336cc03 100644 --- a/core/dbio/iop/datastream.go +++ b/core/dbio/iop/datastream.go @@ -719,6 +719,27 @@ loop: } row := ds.Sp.ProcessRow(ds.it.Row) + + // evaluate transforms, only to determine if type change + // will be re-evaluated again below for storing result + if transforms := ds.Sp.Config.transforms; transforms != nil { + newRow, err := ds.Sp.Config.transforms.Evaluate(row) + if ds.Context.CaptureErr(err) { + break loop + } + + // determine if column type changed + for i, oldVal := range row { + newVal := newRow[i] + colType := ds.Columns[i].Type + oldType := ds.Sp.CheckType(oldVal) + newType := ds.Sp.CheckType(newVal) + if oldType != newType && colType != newType { + ds.Sp.ds.ChangeColumn(i, newType) + } + } + } + ds.Buffer = append(ds.Buffer, row) if ds.it.Counter >= cast.ToUint64(SampleSize) { break loop diff --git a/core/dbio/iop/stream_processor.go b/core/dbio/iop/stream_processor.go index 8cf96d908..b37d2b2e9 100644 --- a/core/dbio/iop/stream_processor.go +++ b/core/dbio/iop/stream_processor.go @@ -1,6 +1,7 @@ package iop import ( + "database/sql" "encoding/json" "errors" "fmt" @@ -569,6 +570,73 @@ func (sp *StreamProcessor) GetType(val any) (typ ColumnType) { return data.Columns[0].Type } +// CheckType returns the type of an interface +func (sp *StreamProcessor) CheckType(v any) (typ ColumnType) { + switch v.(type) { + // Integer types + case int, int8, int16, int32: + return IntegerType + case int64: + return BigIntType + case uint, uint8, uint16, uint32, uint64: + return BigIntType + + // Float types + case float32, float64: + return FloatType + + // Decimal types + case decimal.Decimal: + return DecimalType + + // Boolean + case bool: + return BoolType + + // Time types + case time.Time, *time.Time: + return TimestampType + + // String/Text types + case string: + return StringType + case *string: + return StringType + case []uint8: + return StringType + + // BigInt types + case *big.Int, big.Int: + return BigIntType + + // JSON types + case json.Number: + return DecimalType + case json.RawMessage: + return JsonType + case map[string]string, map[string]any, map[any]any, []any, []string: + return JsonType + case chJSON: // Clickhouse JSON interface + return JsonType + + // SQL Null types + case sql.NullString: + return StringType + case sql.NullInt64: + return BigIntType + case sql.NullFloat64: + return FloatType + case sql.NullBool: + return BoolType + case sql.NullTime: + return TimestampType + + // Default case + default: + return StringType + } +} + // commitChecksum increments the checksum. This is needed due to reprocessing rows func (sp *StreamProcessor) commitChecksum() { for i, val := range sp.rowChecksum { From 3a84134b3b6fa12b6c13dd8e984aebf221feee42 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 19:51:59 -0300 Subject: [PATCH 33/37] fix: add mutex for thread-safe access to typeTemplate cache --- core/dbio/dbio_types.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index 43794f279..efb632f23 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -6,6 +6,7 @@ import ( "encoding/csv" "io" "strings" + "sync" "unicode" "github.com/flarco/g" @@ -372,11 +373,16 @@ func (template Template) Value(path string) (value string) { // a cache for templates (so we only read once) var typeTemplate = map[Type]Template{} +var typeTemplateMutex sync.RWMutex func (t Type) Template() (template Template, err error) { + // Check if template exists in cache (with read lock) + typeTemplateMutex.RLock() if val, ok := typeTemplate[t]; ok { + typeTemplateMutex.RUnlock() return val, nil } + typeTemplateMutex.RUnlock() template = Template{ Core: map[string]string{}, @@ -510,8 +516,10 @@ func (t Type) Template() (template Template, err error) { template.GeneralTypeMap[gt] = rec[t.String()] } - // cache + // cache with write lock + typeTemplateMutex.Lock() typeTemplate[t] = template + typeTemplateMutex.Unlock() return template, nil } From 6068fd303fc24455a2006dc67604a4e0797d5e3e Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 19:58:10 -0300 Subject: [PATCH 34/37] fix: create a copy of the row for safe evaluation of transforms --- core/dbio/iop/datastream.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/dbio/iop/datastream.go b/core/dbio/iop/datastream.go index 28336cc03..6a2c56f78 100644 --- a/core/dbio/iop/datastream.go +++ b/core/dbio/iop/datastream.go @@ -723,7 +723,10 @@ loop: // evaluate transforms, only to determine if type change // will be re-evaluated again below for storing result if transforms := ds.Sp.Config.transforms; transforms != nil { - newRow, err := ds.Sp.Config.transforms.Evaluate(row) + // make a copy of row so we don't reference the pointed values + rowCopy := make([]any, len(row)) + copy(rowCopy, row) + newRow, err := ds.Sp.Config.transforms.Evaluate(rowCopy) if ds.Context.CaptureErr(err) { break loop } From 0f4719f819669f50ff7432f57b053ac39eab0d33 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 20:41:52 -0300 Subject: [PATCH 35/37] feat: add support for Arrow format and define column types in Dataflow test --- core/dbio/iop/duckdb.go | 9 +++++++-- core/dbio/iop/duckdb_test.go | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index d9d700a9e..7bb1d6433 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -1208,11 +1208,16 @@ func (duck *DuckDb) DataflowToHttpStream(df *Dataflow, sc StreamConfig) (streamP contentType := "text/csv" format := dbio.FileTypeCsv // default to CSV if sc.Format == dbio.FileTypeArrow { - if installed, _ := duck.CheckExtension("arrow"); installed { + useArrow := true + if val := os.Getenv("SLING_DUCKDB_ARROW"); val != "" { + useArrow = cast.ToBool(val) + } + duck.AddExtension("arrow") + if installed, _ := duck.CheckExtension("arrow"); installed && useArrow { contentType = "application/vnd.apache.arrow.stream" format = dbio.FileTypeArrow } else { - g.Warn("duckdb extension arrow is not installed") + g.Warn("duckdb extension arrow is not installed or is disabled") } } diff --git a/core/dbio/iop/duckdb_test.go b/core/dbio/iop/duckdb_test.go index 269a41142..148f43d75 100644 --- a/core/dbio/iop/duckdb_test.go +++ b/core/dbio/iop/duckdb_test.go @@ -142,6 +142,9 @@ func TestDuckDbDataflowToHttpStream(t *testing.T) { df := NewDataflow() columns := NewColumnsFromFields("id", "name", "value") + columns[0].Type = IntegerType + columns[1].Type = StringType + columns[2].Type = DecimalType df.Columns = columns df.Ready = true From 9f63326c41fc1b970661bbf76911843b1fe6241d Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 22:04:07 -0300 Subject: [PATCH 36/37] fix: change default import method to CSV for DuckDB connections --- core/dbio/database/database_duckdb_unix.go | 2 +- core/dbio/database/database_duckdb_windows.go | 2 +- core/dbio/filesys/fs.go | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/dbio/database/database_duckdb_unix.go b/core/dbio/database/database_duckdb_unix.go index 8aab5e2dd..56f5c60a4 100644 --- a/core/dbio/database/database_duckdb_unix.go +++ b/core/dbio/database/database_duckdb_unix.go @@ -28,7 +28,7 @@ func (conn *DuckDbConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (cou case "arrow_http": return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) default: - return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) + return conn.importViaHTTP(tableFName, df, dbio.FileTypeCsv) } } diff --git a/core/dbio/database/database_duckdb_windows.go b/core/dbio/database/database_duckdb_windows.go index e8f50dba8..293d3558b 100644 --- a/core/dbio/database/database_duckdb_windows.go +++ b/core/dbio/database/database_duckdb_windows.go @@ -16,6 +16,6 @@ func (conn *DuckDbConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (cou case "arrow_http": return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) default: - return conn.importViaHTTP(tableFName, df, dbio.FileTypeArrow) + return conn.importViaHTTP(tableFName, df, dbio.FileTypeCsv) } } diff --git a/core/dbio/filesys/fs.go b/core/dbio/filesys/fs.go index 6778b2590..3d899eb96 100755 --- a/core/dbio/filesys/fs.go +++ b/core/dbio/filesys/fs.go @@ -1268,7 +1268,6 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw props := g.MapToKVArr(fs.Props()) duck := iop.NewDuckDb(context.Background(), props...) - duck.AddExtension("arrow") duckURI := duck.PrepareFsSecretAndURI(uri) sp := iop.NewStreamProcessor() @@ -1290,7 +1289,7 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw // merge into single stream to push into duckdb duckSc := duck.DefaultCsvConfig() duckSc.FileMaxRows = sc.FileMaxRows - duckSc.Format = dbio.FileTypeArrow + // duckSc.Format = dbio.FileTypeArrow streamPartChn, err := duck.DataflowToHttpStream(df, duckSc) if err != nil { return bw, g.Error(err) From 42a67b48b75d307ba4a3fa28b7482eb43201709b Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 24 Aug 2025 22:38:38 -0300 Subject: [PATCH 37/37] feat: enable Arrow extension and set column types in Dataflow tests --- core/dbio/iop/duckdb.go | 5 ++--- core/dbio/iop/duckdb_test.go | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/dbio/iop/duckdb.go b/core/dbio/iop/duckdb.go index 7bb1d6433..fd7bb6bf2 100644 --- a/core/dbio/iop/duckdb.go +++ b/core/dbio/iop/duckdb.go @@ -1212,12 +1212,11 @@ func (duck *DuckDb) DataflowToHttpStream(df *Dataflow, sc StreamConfig) (streamP if val := os.Getenv("SLING_DUCKDB_ARROW"); val != "" { useArrow = cast.ToBool(val) } - duck.AddExtension("arrow") - if installed, _ := duck.CheckExtension("arrow"); installed && useArrow { + if useArrow { contentType = "application/vnd.apache.arrow.stream" format = dbio.FileTypeArrow } else { - g.Warn("duckdb extension arrow is not installed or is disabled") + g.Debug("duckdb extension arrow is disabled, using csv") } } diff --git a/core/dbio/iop/duckdb_test.go b/core/dbio/iop/duckdb_test.go index 148f43d75..8146ba99a 100644 --- a/core/dbio/iop/duckdb_test.go +++ b/core/dbio/iop/duckdb_test.go @@ -240,6 +240,8 @@ func TestDuckDbDataflowToHttpStream(t *testing.T) { df := NewDataflow() columns := NewColumnsFromFields("id", "value") + columns[0].Type = IntegerType + columns[1].Type = DecimalType df.Columns = columns df.Ready = true @@ -272,6 +274,7 @@ func TestDuckDbDataflowToHttpStream(t *testing.T) { // Create DuckDB instance duck := NewDuckDb(ctx) + duck.AddExtension("arrow") defer duck.Close() // Test DataflowToHttpStream with Arrow format