diff --git a/api/specs/sling_platform.yaml b/api/specs/sling_platform.yaml index c2d6411ac..25f42efb6 100644 --- a/api/specs/sling_platform.yaml +++ b/api/specs/sling_platform.yaml @@ -2,7 +2,7 @@ name: sling-platform defaults: state: - base_url: https://api.slingdata.io + base_url: '{ coalesce(secrets.base_url, "https://api.slingdata.io") }' request: headers: @@ -22,9 +22,9 @@ endpoints: primary_key: [name] processors: - - expression: date_format(now(), "%Y") + - expression: int_parse(date_format(now(), "%Y")) output: record.loaded_year - - expression: log(record) + # - expression: log(record) executions: @@ -42,7 +42,7 @@ endpoints: executions_many: state: - start_date: '{ coalesce(env.START_DATE, state.max_start_time, "2025-01-01") }' + start_date: '{ coalesce(env.START_DATE, sync.max_start_time, "2025-01-01T00:00:00.000000Z") }' end_date: '{ coalesce(env.END_DATE, date_format(now(), "%Y-%m-%dT%H:%M:%S.%fZ")) }' # external sync on start/end of replication. null on first run. @@ -52,7 +52,8 @@ endpoints: url: '{state.base_url}/execution/list' parameters: - limit: 500 + limit: 100 + order: asc filters: > {"period":["{state.start_date}","{state.end_date}"]} @@ -67,7 +68,6 @@ endpoints: start_time: start_time } primary_key: [exec_id] - # limit: 3 processors: - aggregation: maximum @@ -78,14 +78,11 @@ endpoints: ) output: state.max_start_time - # pagination: - # # stop if condition is met - # stop_condition: > - # length(response.records) == 0 || - # state.max_start_time == state.start_date + pagination: + # stop if condition is met + stop_condition: length(response.records) == 0 - # # set state for next page - # next_state: - # start_date: state.max_start_time - # end_date: date_format(now(), "%Y-%m-%dT%H:%M:%S.%fZ") + # set state for next page + next_state: + start_date: '{state.max_start_time}' diff --git a/api/specs/stripe.yaml b/api/specs/stripe.yaml index f816272e1..5e9195fad 100644 --- a/api/specs/stripe.yaml +++ b/api/specs/stripe.yaml @@ -509,7 +509,7 @@ endpoints: response: records: jmespath: "data[]" - primary_key: ["id"] + primary_key: [ subscription_id, id ] processors: - expression: "state.subscription_id" @@ -521,7 +521,7 @@ endpoints: docs: https://docs.stripe.com/api/events state: - starting_after: '{ sync.last_id }' # will be null on first run + ending_before: '{ sync.last_id }' # will be null on first run created_gte: '{coalesce(env.CREATED_GTE, date_format(date_add(now(), -30, "day"), "%s"))}' sync: [ last_id ] @@ -530,13 +530,20 @@ endpoints: url: '{state.base_url}/events' parameters: limit: '{state.limit}' + ending_before: '{state.ending_before}' starting_after: '{state.starting_after}' # created[gte]: '{state.created_gte}' # types: customer.subscription.created,customer.subscription.updated,customer.subscription.deleted + # Strange pagination logic: + # on first run (no `sync.last_id` value), the most recent event is first, + # so we have to use `starting_after` to paginate through all the records. + # on incremental run (with a `sync.last_id` value), in order to get the + # records created after the last known event ID, we have to use `ending_before` pagination: next_state: - starting_after: '{response.records[-1].id}' + starting_after: '{ is_null(sync.last_id) ? response.records[length(response.records)-1].id : nil }' + ending_before: '{ !is_null(sync.last_id) ? response.records[0].id : nil }' stop_condition: "response.json.has_more == false || length(response.records) < 1" response: @@ -547,4 +554,4 @@ endpoints: processors: - expression: "record.id" output: "state.last_id" - aggregation: "last" \ No newline at end of file + aggregation: "first" \ No newline at end of file diff --git a/cmd/sling/resource/llm_PIPELINE.md b/cmd/sling/resource/llm_PIPELINE.md new file mode 100644 index 000000000..8e78f4110 --- /dev/null +++ b/cmd/sling/resource/llm_PIPELINE.md @@ -0,0 +1,655 @@ +# Sling Pipeline Guide for LLMs + +## Table of Contents + +1. [Introduction and Overview](#1-introduction-and-overview) +2. [Quick Start Guide](#2-quick-start-guide) +3. [Pipeline Structure](#3-pipeline-structure) +4. [Task Types (Hooks)](#4-task-types-hooks) +5. [Variables and Dynamic Configuration](#5-variables-and-dynamic-configuration) +6. [Control Flow](#6-control-flow) +7. [Error Handling](#7-error-handling) +8. [Examples](#8-examples) +9. [MCP Tool Integration](#9-mcp-tool-integration) +10. [Best Practices](#10-best-practices) + +--- + +## 1. Introduction and Overview + +### What are Sling Pipelines? + +Sling pipelines are powerful YAML-based workflow definitions that allow you to orchestrate a sequence of tasks. Unlike replications, which are focused on moving data between a single source and target, pipelines are designed to execute a series of arbitrary steps, enabling complex workflows, automation, and integration between various systems. + +### Key Benefits + +- **Orchestration**: Chain multiple operations in a specific order. +- **Modularity**: Create reusable tasks and group them logically. +- **Flexibility**: Combine data replication, shell commands, HTTP requests, SQL queries, and more. +- **Automation**: Automate complex processes like data validation, cleanup, and notifications. +- **Control Flow**: Implement conditional logic and looping. + +### When to Use Pipelines + +- When you need to run tasks before or after a data replication. +- To automate multi-step data processing jobs. +- For data quality checks and validation. +- To integrate Sling with other tools and APIs. +- For cleanup, archiving, or file management tasks. + +### Core Concepts + +- **Pipeline**: A YAML file containing a list of steps to be executed. +- **Step**: A single task to be performed within the pipeline. +- **Task Type (Hook)**: The type of operation for a step (e.g., `command`, `query`, `http`). +- **Variables**: Dynamic placeholders for values that are resolved at runtime. + +### MCP Tool Integration + +The Sling MCP tool provides these pipeline commands: +- `pipeline/docs` - Get documentation. +- `pipeline/parse` - Parse and validate the pipeline configuration. +- `pipeline/run` - Execute the pipeline. + +--- + +## 2. Quick Start Guide + +### Essential MCP Commands + +```json +// Get pipeline documentation +{ + "action": "docs", + "input": {} +} + +// Parse a pipeline file +{ + "action": "parse", + "input": { + "file_path": "/path/to/pipeline.yaml" + } +} + +// Run a pipeline +{ + "action": "run", + "input": { + "file_path": "/path/to/pipeline.yaml" + } +} +``` + +### Basic Pipeline Structure + +```yaml +env: + MY_VARIABLE: "some_value" + +steps: + - type: log + message: "Starting the pipeline..." + + - type: replication + path: /path/to/my_replication.yaml + id: my_replication + + - type: query + connection: MY_DB + query: "SELECT COUNT(*) FROM my_table;" + id: count_query +``` + +### Simple Example: Load data and then run a query + +```yaml +steps: + - type: run + source: "file://data/users.csv" + target: "MY_POSTGRES.public.users" + mode: "full-refresh" + id: load_data + + - type: query + connection: "MY_POSTGRES" + query: "SELECT * FROM public.users LIMIT 5;" + if: state.load_data.status == "success" +``` + +--- + +## 3. Pipeline Structure + +### Root Level Keys + +```yaml +# Optional keys +env: +steps: +``` + +### Step Configuration + +Each step in the `steps` list is an object with the following keys: + +```yaml +steps: + - name: "A descriptive name for the step" # Optional + type: # Required (e.g., command, query, http) + id: # Optional: identifier for referencing output + if: # Optional: condition to run the step + on_failure: abort # Optional: abort/warn/quiet/skip + ... # Other parameters depend on the type +``` + +### Example with multiple steps + +```yaml +env: + TARGET_SCHEMA: "analytics" + +steps: + - type: log + message: "Pipeline started at {timestamp.datetime}" + + - type: replication + path: replications/main_replication.yaml + env: + SLING_THREADS: 8 + id: main_replication + + - type: command + command: "grep -q 'ERROR' /var/log/app.log" + if: "false" # This step will be skipped + + - type: http + url: "https://hooks.slack.com/services/..." + method: "POST" + payload: | + { + "text": "Pipeline completed successfully for schema: {env.TARGET_SCHEMA}" + } + if: state.main_replication.status == "success" +``` + +--- + +## 4. Task Types (Hooks) + +Pipelines are composed of steps, and each step has a `type`, which is also known as a "hook". + +**Important Note:** The hook configurations shown here are for use in **pipelines**. These are different from replication hooks which use `hooks:` at the root level with `pre:`, `post:`, `end:` sections. Pipeline hooks use `steps:` at the root level with `type:` directly in each step. + +Here are the available task types: + +### `log` + +Prints a message to the console. + +```yaml +- type: log + level: info # debug, info, warn, error (default: info) + message: "This is a log message. Current user: {env.USER}" +``` + +[Log Hook Documentation](https://docs.slingdata.io/concepts/hooks/log) + +### `run` + +Executes a simple data transfer, like a one-off `sling` CLI command. + +```yaml +- type: run + source: "MY_POSTGRES.public.users" + target: "file://users.csv" + mode: "full-refresh" + source_options: + limit: 100 +``` + +Note: The `run` type is used for simple data transfers. For complex replications with multiple streams and configurations, use the `replication` type instead. + +### `replication` + +Runs a full replication job from a YAML file. + +```yaml +- type: replication + path: /path/to/my_replication.yaml + select_streams: ["users", "orders"] # Optional + mode: "incremental" # Optional + env: # Optional + SLING_THREADS: 4 +``` + +[Replication Hook Documentation](https://docs.slingdata.io/concepts/hooks/replication) + +### `query` + +Executes a SQL query against a connection. + +```yaml +- type: query + connection: MY_SNOWFLAKE + query: "GRANT SELECT ON ALL TABLES IN SCHEMA analytics TO ROLE reporting;" + into: query_result +``` + +[Query Hook Documentation](https://docs.slingdata.io/concepts/hooks/query) + +### `http` + +Makes an HTTP request. + +```yaml +- type: http + url: "https://api.example.com/v1/status" + method: "GET" # GET, POST, PUT, DELETE, etc. + headers: + Authorization: "Bearer {env.API_TOKEN}" + payload: '{"key": "value"}' # For POST/PUT requests + into: api_response +``` + +[HTTP Hook Documentation](https://docs.slingdata.io/concepts/hooks/http) + +### `command` + +Executes a shell command. + +```yaml +- type: command + command: "ls -la /data" + working_dir: "/app" # Optional + print: true # Optional: Print command output (default: true) + into: command_output +``` + +[Command Hook Documentation](https://docs.slingdata.io/concepts/hooks/command) + +### `check` + +Evaluates a condition and fails the pipeline if the condition is not met. + +```yaml +- name: "Check if row count is greater than 0" + type: check + check: "run.total_rows > threshold" # Required: The condition to evaluate + failure_message: '{run.total_rows} is below threshold' # Optional: the message to use as an error + vars: # Optional: Local variables for the check + threshold: 1000 + on_failure: abort # Optional: abort/warn/quiet/skip +``` + +[Check Hook Documentation](https://docs.slingdata.io/concepts/hooks/check) + +### `copy` + +Copies files or directories. + +```yaml +- type: copy + from: "file://data/today/" + to: "s3://my-bucket/archive/{YYYY}/{MM}/{DD}/" + recursive: true +``` + +[Copy Hook Documentation](https://docs.slingdata.io/concepts/hooks/copy) + +### `delete` + +Deletes files or directories. + +```yaml +- type: delete + location: "file:///tmp/temp_files/" + recursive: true +``` + +[Delete Hook Documentation](https://docs.slingdata.io/concepts/hooks/delete) + +### `write` + +Writes content to a file. + +```yaml +- type: write + to: "file://report.txt" + content: "Pipeline run on {YYYY}-{MM}-{DD}. Status: OK" +``` + +[Write Hook Documentation](https://docs.slingdata.io/concepts/hooks/write) + +### `inspect` + +Inspects files, directories, or database objects to retrieve metadata. + +```yaml +- type: inspect + location: "connection_name/path/to/file_or_table" + recursive: true # Optional: For files, get nested file stats + id: file_info +``` + +Example for database table: +```yaml +- type: inspect + location: "postgres/public.users" + id: table_info +``` + +[Inspect Hook Documentation](https://docs.slingdata.io/concepts/hooks/inspect) + +### `list` + +Lists files or directories from a storage connection. + +```yaml +- type: list + location: "aws_s3/path/to/directory" + recursive: false # Optional: List files recursively + only: files # Optional: List only files or only folders + id: file_list +``` + +[List Hook Documentation](https://docs.slingdata.io/concepts/hooks/list) + +### `store` + +Stores values in memory for later access within the pipeline. + +```yaml +- type: store + key: my_variable + value: "some value to store" +``` + +Later access the stored value: +```yaml +- type: log + message: "Stored value: {store.my_variable}" +``` + +[Store Hook Documentation](https://docs.slingdata.io/concepts/hooks/store) + +### `read` + +Reads the contents of files from storage connections. + +```yaml +- type: read + from: "connection/path/to/file.txt" + into: file_content # Optional: Store content in variable +``` + +[Read Hook Documentation](https://docs.slingdata.io/concepts/hooks/read) + +### `group` + +Groups a set of steps to be executed together. Useful for looping. + +```yaml +- type: group + loop: ["users", "products", "orders"] + steps: + - type: log + message: "Processing table: {loop.value}" + - type: run + source: "POSTGRES.public.{loop.value}" + target: "SNOWFLAKE.analytics.{loop.value}" +``` + +[Group Hook Documentation](https://docs.slingdata.io/concepts/hooks/group) + +--- + +## 5. Variables and Dynamic Configuration + +### Environment Variables + +Define variables at the root `env` block or pass them at runtime. Access them using `{env.VAR_NAME}`. + +```yaml +env: + SCHEMA: "production" + +steps: + - type: log + message: "Using schema: {env.SCHEMA}" +``` + +### Runtime Variables + +Sling provides built-in variables that are resolved at execution time. + +- `{timestamp.YYYY}`, `{timestamp.MM}`, `{timestamp.DD}`, `{timestamp.HH}`: Date and time parts. +- `{timestamp.date}`: Date in YYYY-MM-DD format. +- `{timestamp.datetime}`: Full datetime. +- `{timestamp.unix}`: Unix timestamp. + +### Accessing Step Results + +Save the output of a step using the `id` field and access it in subsequent steps using `{state.step_id}`. + +```yaml +steps: + - type: query + connection: "MY_DB" + query: "SELECT COUNT(*) as user_count FROM users;" + id: "user_count_query" + + - type: log + message: "Number of users: {state.user_count_query.result[0].user_count}" +``` + +For `command` steps, access output using `{state.step_id.output.stdout}`. + +--- + +## 6. Control Flow + +### Conditional Execution (`if`) + +Use the `if` key to conditionally execute a step based on an expression. + +```yaml +steps: + - type: command + command: "date +%u" # Returns 1-7 (Mon-Sun) + id: "day_of_week" + + - type: replication + path: "weekend_job.yaml" + if: "cast(state.day_of_week.output.stdout, 'int') > 5" # Only run on Sat or Sun +``` + +### Looping (`loop` in `group`) + +Use a `group` step with `loop` to iterate over a list of items. The current item is available as `{loop.value}` and the index as `{loop.index}`. + +```yaml +- type: group + loop: + - "customers.csv" + - "products.csv" + - "orders.csv" + steps: + - type: log + message: "Processing file {loop.index + 1}: {loop.value}" + - type: run + source: "file://data/{loop.value}" + target: "STAGING.{loop.value.split('.')[0]}" + mode: "truncate" +``` + +You can also loop over the results of a previous step: + +```yaml +steps: + - type: list + location: "s3://my-bucket/data/" + id: "files_list" + + - type: group + loop: state.files_list.result + steps: + - type: log + message: "Processing: {loop.value.name}" +``` + +--- + +## 7. Error Handling + +### `on_failure` + +Control what happens when a step fails using the `on_failure` parameter. + +```yaml +steps: + - type: delete + location: "file:///tmp/old_logs" + on_failure: warn # Options: abort (default), warn, quiet, skip +``` + +--- + +## 8. Examples + +### Example 1: Daily Data Warehouse Load + +This pipeline runs a replication, then executes a dbt model, and sends a Slack notification. + +```yaml +env: + SLACK_WEBHOOK_URL: "..." + +steps: + - type: log + message: "Starting daily data warehouse load" + + - type: replication + path: "replications/pg_to_sflake.yaml" + mode: "incremental" + id: dw_load + + - type: command + command: "dbt run --select my_models" + working_dir: "/path/to/dbt/project" + if: state.dw_load.status == "success" + + - type: http + url: "{env.SLACK_WEBHOOK_URL}" + method: "POST" + payload: '{"text": "Daily DW load completed successfully!"}' + if: state.dw_load.status == "success" + + - type: http + url: "{env.SLACK_WEBHOOK_URL}" + method: "POST" + payload: '{"text": "ERROR: Daily DW load failed!"}' + if: state.dw_load.status == "error" +``` + +### Example 2: File Processing Loop + +This pipeline downloads a file, processes each line, and then archives the file. + +```yaml +steps: + - type: copy + from: "sftp://remote/data/transactions.csv" + to: "file://temp/transactions.csv" + id: download_file + + - type: read + from: "file://temp/transactions.csv" + id: file_content + + - type: group + loop: state.file_content.content.split('\n') + steps: + - type: http + url: "https://api.my-app.com/process" + method: "POST" + payload: "{loop.value}" + if: "loop.value != ''" # Avoid processing empty lines + + - type: copy + from: "file://temp/transactions.csv" + to: "s3://my-archive/transactions/{timestamp.YYYY}_{timestamp.MM}_{timestamp.DD}.csv" + + - type: delete + location: "file://temp/transactions.csv" +``` + +--- + +## 9. MCP Tool Integration + +### Using Pipeline Commands + +#### Get Documentation +```json +{ + "action": "docs", + "input": {} +} +``` + +#### Parse Configuration +```json +{ + "action": "parse", + "input": { + "file_path": "/path/to/pipeline.yaml" + } +} +``` + +#### Execute Pipeline +```json +{ + "action": "run", + "input": { + "file_path": "/path/to/pipeline.yaml", + "env": { + "CUSTOM_VAR": "value" + } + } +} +``` + +### Development Workflow + +1. **Write** your pipeline YAML file. +2. **Parse** the configuration to validate syntax: `{"action": "parse", "input": {"file_path": "my_pipeline.yaml"}}`. +3. **Test** individual steps if possible (e.g., run `query` or `command` steps manually). +4. **Run** the full pipeline: `{"action": "run", "input": {"file_path": "my_pipeline.yaml"}}`. +5. **Debug** by checking the logs and the output of each step. + +--- + +## 10. Best Practices + +### Configuration Organization + +- Use descriptive names for your steps. +- Group related steps logically. +- Use the `env` block for global variables to avoid hardcoding values. + +### Performance + +- Be mindful of long-running `command` or `http` steps that might block the pipeline. +- Use `replication` for large data movements as it's optimized for performance. + +### Security + +- Store sensitive values like API keys and passwords in environment variables, not directly in the pipeline file. +- Be cautious with the `command` hook, as it can execute any shell command. + +### Readability + +- Add comments to your YAML file to explain complex steps. +- Break down large pipelines into smaller, more manageable sub-pipelines if possible (though Sling doesn't directly support nested pipelines, you can call one from another using a `command` step). diff --git a/cmd/sling/resource/llm_PYTHON.md b/cmd/sling/resource/llm_PYTHON.md new file mode 100644 index 000000000..90a994711 --- /dev/null +++ b/cmd/sling/resource/llm_PYTHON.md @@ -0,0 +1,310 @@ +# Sling + Python Guide for LLMs + +## Overview +Sling is a powerful data movement and transformation tool that provides Python bindings for seamless integration. The Python package (`sling`) wraps the Sling CLI to enable efficient data transfers between databases, files, and Python data structures. + +## Installation + +```bash +# Basic installation +pip install sling + +# With Apache Arrow support for high-performance streaming +pip install sling[arrow] +``` + +## Core Classes + +### 1. Sling Class - Primary Interface +The `Sling` class mirrors all CLI functionalities and provides the most flexible interface. + +```python +from sling import Sling, Mode, Format, Compression +from sling.options import SourceOptions, TargetOptions +``` + +#### Key Parameters: +- **Source**: `src_conn`, `src_stream`, `src_options` +- **Target**: `tgt_conn`, `tgt_object`, `tgt_options` +- **Transforms**: `select`, `where`, `transforms`, `columns` +- **Mode**: `mode` (full-refresh, incremental, truncate, snapshot, backfill) +- **Limits**: `limit`, `offset`, `range` +- **Keys**: `primary_key`, `update_key` +- **Config**: `env`, `debug`, `trace` +- **Python-specific**: `input` (data input from Python) + +### 2. Replication Class +For managing multi-stream data replications. + +```python +from sling import Replication, ReplicationStream +``` + +### 3. Pipeline Class +For executing sequential data processing steps. + +```python +from sling import Pipeline, Hook +``` + +## Common Usage Patterns + +### 1. Basic Data Transfer +```python +# Database to database +sling = Sling( + src_conn="postgres", + src_stream="public.users", + tgt_conn="snowflake", + tgt_object="analytics.users", + mode="full-refresh" +) +sling.run() + +# File to database +sling = Sling( + src_stream="file:///path/to/data.csv", + tgt_conn="postgres", + tgt_object="public.imported_data" +) +sling.run() + +# SQL query as source +sling = Sling( + src_conn="mysql", + src_stream="select * from orders where created_at > '2024-01-01'", + tgt_conn="snowflake", + tgt_object="staging.recent_orders" +) +sling.run() +``` + +### 2. Python Data Input +```python +# From list of dictionaries +data = [ + {"id": 1, "name": "Alice", "age": 30}, + {"id": 2, "name": "Bob", "age": 25} +] +sling = Sling( + input=data, + tgt_conn="postgres", + tgt_object="public.users" +) +sling.run() + +# From pandas DataFrame +import pandas as pd +df = pd.DataFrame(data) +sling = Sling( + input=df, + tgt_conn="snowflake", + tgt_object="public.users" +) +sling.run() + +# From polars DataFrame +import polars as pl +df = pl.DataFrame(data) +sling = Sling( + input=df, + tgt_conn="duckdb", + tgt_object="users" +) +sling.run() +``` + +### 3. Streaming Output +```python +# Stream data as Python dictionaries +sling = Sling( + src_conn="postgres", + src_stream="select * from large_table" +) +for record in sling.stream(): + # Process each record + print(record) + +# High-performance Arrow streaming +reader = sling.stream_arrow() +for batch in reader: + # Process Arrow RecordBatch + df = batch.to_pandas() + process_batch(df) +``` + +### 4. Data Transformation +```python +# Column selection +sling = Sling( + src_conn="postgres", + src_stream="users", + select=["id", "name", "email"], + tgt_conn="snowflake", + tgt_object="users_subset" +) + +# Filtering with WHERE clause +sling = Sling( + src_conn="mysql", + src_stream="orders", + where="status = 'completed' and amount > 100", + tgt_conn="postgres", + tgt_object="completed_orders" +) + +# Column transformations +transforms = [{ + "email": "lower(value)", + "full_name": "concat(record.first_name, ' ', record.last_name)", + "created_date": "date(record.created_at)" +}] +sling = Sling( + src_conn="postgres", + src_stream="users", + transforms=transforms, + tgt_conn="snowflake", + tgt_object="transformed_users" +) + +# Column type casting +columns = { + "id": "bigint", + "price": "decimal(10,2)", + "created_at": "timestamp" +} +sling = Sling( + src_stream="file://data.csv", + columns=columns, + tgt_conn="postgres", + tgt_object="products" +) +``` + +### 5. Advanced Options +```python +# Source options +src_opts = SourceOptions( + format=Format.CSV, + delimiter="|", + header=True, + null_if="NULL", +) + +# Target options +tgt_opts = TargetOptions( + format=Format.PARQUET, + compression=Compression.ZSTD, + file_max_rows=100000, + column_casing="snake" +) + +sling = Sling( + src_stream="file://input.csv", + src_options=src_opts, + tgt_object="file://output.parquet", + tgt_options=tgt_opts +) +sling.run() +``` + +### 6. Incremental Loading +```python +# Using primary key for incremental updates +sling = Sling( + src_conn="postgres", + src_stream="events", + tgt_conn="snowflake", + tgt_object="events", + mode="incremental", + primary_key=["event_id"], + update_key="updated_at" +) +sling.run() +``` + +### 7. Replication Configuration +```python +from sling import Replication, ReplicationStream + +# Create replication with multiple streams +replication = Replication( + source="postgres", + target="snowflake", + streams={ + "public.users": ReplicationStream( + mode="full-refresh", + object="analytics.users" + ), + "public.orders": ReplicationStream( + mode="incremental", + object="analytics.orders", + primary_key=["order_id"], + update_key="modified_at" + ) + }, + debug=True +) +replication.run() + +# Load from YAML file +replication = Replication(file_path="replication.yaml") +replication.run() +``` + +## Environment Variables +```python +# Set connection environment variables +import os + +os.environ["POSTGRES_URL"] = "postgresql://user:pass@localhost:5432/db" +os.environ["SNOWFLAKE_URL"] = "snowflake://user:pass@account/database/schema" + +# Use connection names in Sling +sling = Sling( + src_conn="POSTGRES", + src_stream="users", + tgt_conn="SNOWFLAKE", + tgt_object="users" +) + +# Pass environment variables to Sling +env = { + "SLING_LOADED_AT_COLUMN": "timestamp", + "SLING_STREAM_URL_COLUMN": "true" +} +sling = Sling( + src_conn="postgres", + src_stream="events", + tgt_conn="snowflake", + tgt_object="events", + env=env +) +``` + +## Debugging + +```python +# Enable debug output +sling = Sling( + src_conn="postgres", + src_stream="users", + tgt_conn="snowflake", + tgt_object="users", + debug=True # Shows detailed execution info +) + +# Enable trace-level debugging +sling = Sling( + src_conn="postgres", + src_stream="users", + tgt_conn="snowflake", + tgt_object="users", + trace=True # Shows even more detailed info +) +``` + +## Additional Resources +- Documentation: https://docs.slingdata.io +- GitHub: https://github.com/slingdata-io/sling-python +- Examples: https://docs.slingdata.io/examples/sling-python \ No newline at end of file diff --git a/cmd/sling/resource/mcp.yaml b/cmd/sling/resource/mcp.yaml index 40b1572cb..35f17129b 100644 --- a/cmd/sling/resource/mcp.yaml +++ b/cmd/sling/resource/mcp.yaml @@ -443,6 +443,13 @@ tools: **For `action: "docs"`**: - No parameters required (empty object) + + **For `action: "parse"`**: + - `file_path` (string, required): Path to the pipeline configuration file to read. + + **For `action: "run"`**: + - `file_path` (string, required): Path to the pipeline configuration file to execute. + - `env` (object, optional): Environment variables to set for the pipeline run. **Output**: The output format depends on the action and audience: - **For User**: Human-readable messages, CSV tables, or status updates @@ -460,7 +467,7 @@ tools: { "action": "parse", "input": { - "file_path": "path/to/my_pipeline.yaml", + "file_path": "path/to/my_pipeline.yaml" } } @@ -471,6 +478,17 @@ tools: "file_path": "path/to/my_pipeline.yaml" } } + + # Run a pipeline with environment variables + { + "action": "run", + "input": { + "file_path": "path/to/my_pipeline.yaml", + "env": { + "MY_VAR": "my_value" + } + } + } ``` diff --git a/cmd/sling/sling_run.go b/cmd/sling/sling_run.go index 9de499bb3..b94fed50c 100755 --- a/cmd/sling/sling_run.go +++ b/cmd/sling/sling_run.go @@ -211,7 +211,7 @@ runReplication: if env.NoColor { g.Info(text) } else { - g.Info(g.Colorize(g.ColorCyan, text)) + g.Info(env.CyanString(text)) } } diff --git a/cmd/sling/tests/replications/apis/r.62.shopify.yaml b/cmd/sling/tests/replications/apis/r.shopify.yaml similarity index 100% rename from cmd/sling/tests/replications/apis/r.62.shopify.yaml rename to cmd/sling/tests/replications/apis/r.shopify.yaml diff --git a/cmd/sling/tests/replications/apis/r.sling_platform.yaml b/cmd/sling/tests/replications/apis/r.sling_platform.yaml new file mode 100644 index 000000000..c65cdb816 --- /dev/null +++ b/cmd/sling/tests/replications/apis/r.sling_platform.yaml @@ -0,0 +1,26 @@ +source: sling_platform_api_test +target: postgres + +defaults: + object: '{source_name}.{stream_name}' + mode: full-refresh + +streams: + '*': + connections: + hooks: + pre: + - type: query + connection: postgres + query: | + with cte as ( + select 1 as id union select 2 as id union select 3 as id union select 4 as id + ) + select id from cte order by id + into: records + + executions_many: + mode: incremental + +env: + SLING_STATE: postgres/sling_state.sling_platform_api_staging \ No newline at end of file diff --git a/cmd/sling/tests/replications/apis/r.61.stripe.yaml b/cmd/sling/tests/replications/apis/r.stripe.yaml similarity index 69% rename from cmd/sling/tests/replications/apis/r.61.stripe.yaml rename to cmd/sling/tests/replications/apis/r.stripe.yaml index 6c5acdaa9..cc201c958 100644 --- a/cmd/sling/tests/replications/apis/r.61.stripe.yaml +++ b/cmd/sling/tests/replications/apis/r.stripe.yaml @@ -1,4 +1,4 @@ -# ./sling run -r cmd/sling/tests/replications/apis/r.61.stripe.yaml --streams customer,customer_balance_transaction -d --mode full-refresh +# ./sling run -r cmd/sling/tests/replications/apis/r.stripe.yaml --streams customer,customer_balance_transaction -d --mode full-refresh source: stripe_sling target: postgres diff --git a/cmd/sling/tests/replications/apis/test_spec/sling_platform.yaml b/cmd/sling/tests/replications/apis/test_spec/sling_platform.yaml new file mode 100644 index 000000000..0e30be367 --- /dev/null +++ b/cmd/sling/tests/replications/apis/test_spec/sling_platform.yaml @@ -0,0 +1,92 @@ +name: sling-platform + +defaults: + state: + base_url: '{ secrets.base_url }' + + request: + headers: + Authorization: 'Sling-Project-Token {secrets.api_token}' + Content-Type: application/json + + +endpoints: + connections: + + iterate: + over: "state.replication_store.records" + into: "state.test_record" + + request: + url: '{state.base_url}/connection/list?test-value={state.test_record.id}' + + response: + records: + jmespath: connections[*] + primary_key: [name] + + processors: + - expression: int_parse(date_format(now(), "%Y")) + output: record.loaded_year + # - expression: log(record) + + executions: + + request: + url: '{state.base_url}/execution/list' + + response: + records: + jmespath: executions[*] + primary_key: [exec_id] + + # processors: + # - expression: log(record) + + executions_many: + + state: + start_date: '{ coalesce(env.START_DATE, sync.max_start_time, "2025-01-01T00:00:00.000000Z") }' + end_date: '{ coalesce(env.END_DATE, date_format(now(), "%Y-%m-%dT%H:%M:%S.%fZ")) }' + + # external sync on start/end of replication. null on first run. + sync: [ max_start_time ] + + request: + url: '{state.base_url}/execution/list' + + parameters: + limit: 100 + order: asc + filters: > + {"period":["{state.start_date}","{state.end_date}"]} + + response: + records: + jmespath: > + executions[*].{ + exec_id: exec_id, + rows: rows, + replication_md5: replication_md5, + task_md5: task_md5, + start_time: start_time + } + primary_key: [exec_id] + + processors: + - aggregation: maximum + expression: | + date_format( + date_parse(record.start_time), + "%Y-%m-%dT%H:%M:%S.%fZ" + ) + output: state.max_start_time + + pagination: + # stop if condition is met + stop_condition: length(response.records) == 0 + + # set state for next page + next_state: + start_date: '{state.max_start_time}' + diff --git a/cmd/sling/tests/replications/r.28.oracle_number_scale.yaml b/cmd/sling/tests/replications/r.28.oracle_number_scale.yaml index 4ac2dcecf..de13ec7b9 100644 --- a/cmd/sling/tests/replications/r.28.oracle_number_scale.yaml +++ b/cmd/sling/tests/replications/r.28.oracle_number_scale.yaml @@ -34,14 +34,15 @@ hooks: - type: query connection: '{source.name}' query: | + INSERT INTO {env.schema}.TEST_NUMBER_PRECISION (id, value_decimal, value_explicit, value_integer) - VALUES (1, 0.01233, 0.01, 10) + VALUES (3, 123456789123456789.123456789, 1234567.12, 123456789); - - type: query - connection: '{source.name}' - query: | INSERT INTO {env.schema}.TEST_NUMBER_PRECISION (id, value_decimal, value_explicit, value_integer) - VALUES (2, 123.456789, 123.45, 123) + VALUES (1, 0.01233, 0.01, 10); + + INSERT INTO {env.schema}.TEST_NUMBER_PRECISION (id, value_decimal, value_explicit, value_integer) + VALUES (2, 123.456789, 123.45, 123); - type: query connection: '{source.name}' @@ -80,6 +81,8 @@ hooks: store.result[0].value_decimal => {store.result[0].value_decimal} store.result[1] => {store.result[1]} store.result[1].value_decimal => {store.result[1].value_decimal} + store.result[2] => {store.result[2]} + store.result[2].value_decimal => {store.result[2].value_decimal} # ensure the decimal values are preserved - type: check @@ -91,6 +94,9 @@ hooks: - type: check check: float_parse(store.result[1].value_decimal) == 123.456789 + - type: check + check: store.result[2].value_decimal == "123456789123456789.123456789" + - type: query connection: '{source.name}' query: | diff --git a/cmd/sling/tests/replications/r.51.xml_postgres_import.yaml b/cmd/sling/tests/replications/r.51.xml_postgres_import.yaml new file mode 100644 index 000000000..ae8b07d07 --- /dev/null +++ b/cmd/sling/tests/replications/r.51.xml_postgres_import.yaml @@ -0,0 +1,140 @@ +source: LOCAL +target: postgres + +defaults: + mode: full-refresh + +source_options: + flatten: 1 + +hooks: + start: + - type: command + command: | + cat > /tmp/test_xml_data.xml << 'EOF' + + + + 1 + John Doe + john.doe@example.com + 30 + true + 2024-01-15 10:30:00 + 75000.50 + + + 2 + Jane Smith + jane.smith@example.com + 28 + false + 2024-02-20 14:45:30 + 82000.75 + + + 3 + Bob Johnson + bob.johnson@example.com + 35 + true + 2024-03-10 09:15:45 + 95000.00 + + + 4 + Alice Brown + alice.brown@example.com + 42 + true + 2024-04-05 16:20:10 + 105000.25 + + + 5 + Charlie Wilson + charlie.wilson@example.com + 26 + false + 2024-05-12 11:30:00 + 65000.00 + + + EOF + + - type: query + connection: '{target.name}' + query: DROP TABLE IF EXISTS public.xml_file_test; + + end: + # Verify data was imported + - type: query + connection: '{target.name}' + query: SELECT COUNT(*) as count FROM public.xml_file_test + into: result + + - type: check + check: store.result[0].count == 5 + success_message: "✓ All 5 XML records imported successfully" + + # Verify data integrity + - type: query + connection: '{target.name}' + query: | + SELECT * + FROM public.xml_file_test + ORDER BY id + LIMIT 3 + into: sample_data + + - type: log + message: | + XML Import Test Results: + - Total records imported: {store.result[0].count} + - Sample data: + { pretty_table(store.sample_data) } + + # Verify specific record values + - type: query + connection: '{target.name}' + query: | + SELECT * + FROM public.xml_file_test + WHERE id = 1 + into: first_record + + - type: check + check: store.first_record[0].name == "John Doe" + success_message: "✓ Name field imported correctly" + + - type: check + check: store.first_record[0].email == "john.doe@example.com" + success_message: "✓ Email field imported correctly" + + - type: check + check: int_parse(store.first_record[0].age) == 30 + success_message: "✓ Age field imported correctly" + + - type: check + check: float_parse(store.first_record[0].salary) == 75000.50 + success_message: "✓ Salary field imported correctly" + + # Cleanup + - type: command + command: rm -f /tmp/test_xml_data.xml + + - type: query + connection: '{target.name}' + query: DROP TABLE IF EXISTS public.xml_file_test; + + - type: log + message: | + ✓ XML to PostgreSQL import test completed successfully! + ✓ All fields imported with correct data types + ✓ Test cleanup completed + +streams: + file:///tmp/test_xml_data.xml: + object: public.xml_file_test + source_options: + flatten: true \ No newline at end of file diff --git a/cmd/sling/tests/replications/r.52.parquet_file_splitting.yaml b/cmd/sling/tests/replications/r.52.parquet_file_splitting.yaml new file mode 100644 index 000000000..676c91c40 --- /dev/null +++ b/cmd/sling/tests/replications/r.52.parquet_file_splitting.yaml @@ -0,0 +1,40 @@ +source: postgres +target: '{my_target}' + +defaults: + mode: full-refresh + object: 'splitting-test/{stream_schema}/{stream_table}' + +hooks: + end: + # List files in the target folder to verify correct chunking + - type: list + id: list_files + location: '{env.my_target}/splitting-test/public/test1k_athena_pg/' + only: files + + - type: log + message: | + Files created: {length(state.list_files.result)} + File list: {state.list_files.result} + + # Check that exactly 11 files were created (1000 rows / 100 rows per file = 10 files + 1 manifest) + - type: check + check: length(state.list_files.result) == 11 + failure_message: "Expected 11 files but got {length(state.list_files.result)}" + + # Clean up files after test + - type: delete + location: '{env.my_target}/splitting-test/' + recursive: true + +streams: + public.test1k_athena_pg: + sql: select * from {stream_schema}.{stream_table} + mode: full-refresh + target_options: + file_max_rows: 100 + format: parquet + +env: + my_target: ${MY_TARGET} \ No newline at end of file diff --git a/cmd/sling/tests/replications/r.53.prometheus_incremental.yaml b/cmd/sling/tests/replications/r.53.prometheus_incremental.yaml new file mode 100644 index 000000000..ab9220fe3 --- /dev/null +++ b/cmd/sling/tests/replications/r.53.prometheus_incremental.yaml @@ -0,0 +1,21 @@ +source: prometheus +target: postgres + +hooks: + start: + # Clean up test table + - type: query + connection: '{target.name}' + query: DROP TABLE IF EXISTS public.prometheus_no_data_test; + + end: + # Clean up + - type: query + connection: '{target.name}' + query: DROP TABLE IF EXISTS public.prometheus_no_data_test; + +streams: + # Query for data that doesn't exist - should not error in full-refresh mode + 'go_memstats_alloc_bytes_total{job="nonexistent_job_that_will_not_match"} # {"start": "now-1h", "end": "now", "step": "5m"}': + object: public.prometheus_no_data_test + mode: full-refresh \ No newline at end of file diff --git a/cmd/sling/tests/suite.cli.yaml b/cmd/sling/tests/suite.cli.yaml index e4e0f346c..3da79858c 100644 --- a/cmd/sling/tests/suite.cli.yaml +++ b/cmd/sling/tests/suite.cli.yaml @@ -858,3 +858,90 @@ - "Very high scale BIGNUMERIC value preserved accurately" - "CSV to BigQuery BIGNUMERIC conversion test completed successfully!" - "execution succeeded" + +- id: 108 + name: Run XML to PostgreSQL import test + run: 'sling run -r cmd/sling/tests/replications/r.51.xml_postgres_import.yaml' + streams: 1 + rows: 5 + output_contains: + - 'execution succeeded' + - '✓ All 5 XML records imported successfully' + - '✓ Name field imported correctly' + - '✓ Email field imported correctly' + - '✓ Age field imported correctly' + - '✓ Salary field imported correctly' + - '✓ XML to PostgreSQL import test completed successfully!' + +- id: 109 + name: Test API -- Sling Platform + run: sling run -r cmd/sling/tests/replications/apis/r.sling_platform.yaml -d + streams: 3 + output_contains: + - 'execution succeeded' + + # test iterations + - '/connection/list?test-value=1' + - '/connection/list?test-value=2' + - '/connection/list?test-value=3' + - '/connection/list?test-value=4' + - '/execution/list?filters=%7B%22period' + +- id: 110 + name: Test API -- Sling Stripe + run: sling run -r cmd/sling/tests/replications/apis/r.stripe.yaml + output_contains: + - 'execution succeeded' + +- id: 111 + name: Run AWS S3 file splitting test + run: 'sling run -r cmd/sling/tests/replications/r.52.parquet_file_splitting.yaml' + env: + MY_TARGET: aws_s3 + streams: 1 + rows: 1000 + output_contains: + - 'execution succeeded' + +- id: 112 + name: Run AZURE_STORAGE file splitting test + run: 'sling run -r cmd/sling/tests/replications/r.52.parquet_file_splitting.yaml' + env: + MY_TARGET: AZURE_STORAGE + streams: 1 + rows: 1000 + output_contains: + - 'execution succeeded' + +- id: 113 + name: Run GOOGLE_STORAGE file splitting test + run: 'sling run -r cmd/sling/tests/replications/r.52.parquet_file_splitting.yaml' + env: + MY_TARGET: GOOGLE_STORAGE + streams: 1 + rows: 1000 + output_contains: + - 'execution succeeded' + +- id: 114 + name: Run local file splitting test + run: 'sling run -r cmd/sling/tests/replications/r.52.parquet_file_splitting.yaml' + env: + MY_TARGET: local + streams: 1 + rows: 1000 + output_contains: + - 'execution succeeded' + +# Test Prometheus with no data (should not error) +- id: 115 + run: 'sling run -r cmd/sling/tests/replications/r.53.prometheus_incremental.yaml' + env: + SLING_ALLOW_EMPTY: 'TRUE' + conns: + - prometheus + - postgres + streams: 1 + rows: 0 + output_contains: + - 'execution succeeded' diff --git a/core/dbio/api/api.go b/core/dbio/api/api.go index 86d114601..2755faed8 100644 --- a/core/dbio/api/api.go +++ b/core/dbio/api/api.go @@ -68,6 +68,25 @@ func NewAPIConnection(ctx context.Context, spec Spec, data map[string]any) (ac * return ac, nil } +func (ac *APIConnection) GetReplicationStore() (store map[string]any) { + err := g.JSONConvert(ac.State.State["replication_store"], &store) + if err != nil { + g.Warn("could not unmarshal from API state replication_store: " + err.Error()) + } + + if store == nil { + store = g.M() + } + return store +} + +func (ac *APIConnection) SetReplicationStore(store map[string]any) { + if store == nil { + store = g.M() + } + ac.State.State["replication_store"] = store +} + // Close performs cleanup of all resources func (ac *APIConnection) Close() error { ac.CloseAllQueues() @@ -296,9 +315,9 @@ func (ac *APIConnection) renderAny(input any, extraMaps ...map[string]any) (outp // GetSyncedState cycles through each endpoint, and collects the values // for each of the Endpoint.Sync values. Output is a -// map[Endpoint.Name]map[Sync.value] = Endpoint.syncMap[Sync.value] -func (ac *APIConnection) GetSyncedState(endpointName string) (data map[string]map[string]any, err error) { - data = make(map[string]map[string]any) +// map[Sync.value] = Endpoint.syncMap[Sync.value] +func (ac *APIConnection) GetSyncedState(endpointName string) (data map[string]any, err error) { + data = make(map[string]any) // Iterate through all endpoints for _, endpoint := range ac.Spec.EndpointMap { @@ -307,14 +326,11 @@ func (ac *APIConnection) GetSyncedState(endpointName string) (data map[string]ma continue } - // Initialize map for this endpoint - data[endpoint.Name] = make(map[string]any) - // Collect each sync value from endpoint's state with proper locking endpoint.context.Lock() for _, syncKey := range endpoint.Sync { if val, ok := endpoint.State[syncKey]; ok { - data[endpoint.Name][syncKey] = val + data[syncKey] = val } } endpoint.context.Unlock() @@ -325,8 +341,8 @@ func (ac *APIConnection) GetSyncedState(endpointName string) (data map[string]ma // PutSyncedState restores the state from previous run in each endpoint // using the Endpoint.Sync values. -// Inputs is map[Endpoint.Name]map[Sync.value] = Endpoint.syncMap[Sync.value] -func (ac *APIConnection) PutSyncedState(endpointName string, data map[string]map[string]any) (err error) { +// Inputs is map[Sync.value] = Endpoint.syncMap[Sync.value] +func (ac *APIConnection) PutSyncedState(endpointName string, data map[string]any) (err error) { // Iterate through all endpoints for key, endpoint := range ac.Spec.EndpointMap { @@ -335,11 +351,6 @@ func (ac *APIConnection) PutSyncedState(endpointName string, data map[string]map continue } - endpointData, exists := data[endpoint.Name] - if !exists { - continue - } - // Initialize state map if nil and sync state with proper locking endpoint.context.Lock() if endpoint.syncMap == nil { @@ -347,7 +358,7 @@ func (ac *APIConnection) PutSyncedState(endpointName string, data map[string]map } // Restore each sync value to endpoint's state - for key, val := range endpointData { + for key, val := range data { // Only restore if it's in the sync list for _, syncKey := range endpoint.Sync { if syncKey == key { diff --git a/core/dbio/api/spec.go b/core/dbio/api/spec.go index 940444818..73ab9a4be 100644 --- a/core/dbio/api/spec.go +++ b/core/dbio/api/spec.go @@ -189,6 +189,7 @@ type Endpoint struct { context *g.Context uniqueKeys map[string]struct{} // for PrimaryKey deduplication bloomFilter *bloom.BloomFilter + aggregate AggregateState } func (ep *Endpoint) SetStateVal(key string, val any) { @@ -366,6 +367,14 @@ type Iteration struct { endpoint *Endpoint } +func (iter *Iteration) Debug(text string, args ...any) { + if iter.field != "" { + id := g.F("i%02d", iter.id) + text = env.DarkGrayString(id) + " " + text + } + g.Debug(text, args...) +} + // StateMap stores the current state of an endpoint's execution type StateMap map[string]any @@ -478,7 +487,6 @@ type Rule struct { type SingleRequest struct { Request *RequestState `yaml:"request" json:"request"` Response *ResponseState `yaml:"response" json:"response"` - Aggregate AggregateState `yaml:"-" json:"-"` httpReq *http.Request `yaml:"-" json:"-"` httpRespWg sync.WaitGroup `yaml:"-" json:"-"` id string `yaml:"-" json:"-"` diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index 630613fec..95d725e53 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -852,10 +852,23 @@ func (c *Connection) setURL() (err error) { if _, ok := c.Data["schema"]; ok && c.Data["schema"] != "" { template = template + "&schema={schema}" } + case dbio.TypeDbArrowFlight: + setIfMissing("username", c.Data["user"]) + setIfMissing("password", "") + setIfMissing("port", c.Type.DefPort()) + setIfMissing("schema", "") + + template = "flightsql://{username}:{password}@{host}:{port}/?" + if _, ok := c.Data["schema"]; ok && c.Data["schema"] != "" { + template = template + "&schema={schema}" + } case dbio.TypeFileSftp, dbio.TypeFileFtp: setIfMissing("password", "") setIfMissing("port", c.Type.DefPort()) template = c.Type.String() + "://{user}:{password}@{host}:{port}/" + if path := cast.ToString(c.Data["path"]); path != "" { + template = template + path + } case dbio.TypeFileS3, dbio.TypeFileGoogle, dbio.TypeFileGoogleDrive, dbio.TypeFileAzure, dbio.TypeFileLocal: return nil diff --git a/core/dbio/connection/connection_discover.go b/core/dbio/connection/connection_discover.go index 9dc0d1590..00ef44ee6 100644 --- a/core/dbio/connection/connection_discover.go +++ b/core/dbio/connection/connection_discover.go @@ -13,6 +13,7 @@ import ( "github.com/slingdata-io/sling-cli/core/dbio/database" "github.com/slingdata-io/sling-cli/core/dbio/filesys" "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/slingdata-io/sling-cli/core/env" "github.com/spf13/cast" ) @@ -77,7 +78,10 @@ func (c *Connection) Test() (ok bool, err error) { } limit := cast.ToInt(g.Getenv("SLING_TEST_ENDPOINT_LIMIT", "10")) - g.Debug("testing endpoints with a record limit: %d. Set env var SLING_TEST_ENDPOINT_LIMIT to modify.", limit) + if limit > 1000 { + limit = 1000 // let's set the max limit to 1000 for testing + } + g.Debug(env.MagentaString(g.F("testing endpoints with a record limit: %d. Set env var SLING_TEST_ENDPOINT_LIMIT to modify.", limit))) for _, endpoint := range endpoints { // check for match to test (if provided) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 9872be0d8..042079f6f 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -24,6 +24,7 @@ import ( "github.com/flarco/g" "github.com/slingdata-io/sling-cli/core/env" + _ "github.com/apache/arrow-go/v18/arrow/flight/flightsql" _ "github.com/databricks/databricks-sql-go" _ "github.com/exasol/exasol-driver-go" _ "github.com/go-sql-driver/mysql" @@ -85,7 +86,7 @@ type Connection interface { ExecMultiContext(ctx context.Context, sqls ...string) (result sql.Result, err error) GenerateDDL(table Table, data iop.Dataset, temporary bool) (string, error) GenerateInsertStatement(tableName string, cols iop.Columns, numRows int) string - GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) + GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) GetAnalysis(string, map[string]interface{}) (string, error) GetColumns(tableFName string, fields ...string) (iop.Columns, error) GetColumnsFull(string) (iop.Dataset, error) @@ -140,7 +141,7 @@ type Connection interface { Template() dbio.Template Tx() Transaction Unquote(string) string - Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error) + Merge(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error) ValidateColumnNames(tgtCols iop.Columns, colNames []string) (newCols iop.Columns, err error) AddMissingColumns(table Table, newCols iop.Columns) (ok bool, err error) } @@ -312,6 +313,8 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &IcebergConn{URL: URL} } else if strings.HasPrefix(URL, "azuretable:") { conn = &AzureTableConn{URL: URL} + } else if strings.HasPrefix(URL, "flightsql:") { + conn = &AzureTableConn{URL: URL} } else { conn = &BaseConn{URL: URL} } @@ -375,6 +378,8 @@ func getDriverName(conn Connection) (driverName string) { driverName = "proton" case dbio.TypeDbExasol: driverName = "exasol" + case dbio.TypeDbArrowFlight: + driverName = "flightsql" default: driverName = dbType.String() } @@ -2274,60 +2279,6 @@ func (conn *BaseConn) GenerateInsertStatement(tableName string, cols iop.Columns return statement } -// Upsert inserts / updates from a srcTable into a target table. -// Assuming the srcTable has some or all of the tgtTable fields with matching types -func (conn *BaseConn) Upsert(srcTable string, tgtTable string, primKeys []string) (rowAffCnt int64, err error) { - var cnt int64 - if conn.tx != nil { - cnt, err = Upsert(conn.Self(), conn.tx, srcTable, tgtTable, primKeys) - } else { - cnt, err = Upsert(conn.Self(), nil, srcTable, tgtTable, primKeys) - } - if err != nil { - err = g.Error(err, "could not upsert") - } - return cast.ToInt64(cnt), err -} - -// SwapTable swaps two table -func (conn *BaseConn) SwapTable(srcTable string, tgtTable string) (err error) { - - tgtTableTemp := tgtTable + "_tmp" + g.RandString(g.AlphaRunesLower, 2) - conn.DropTable(tgtTableTemp) - - sql := g.R( - conn.GetTemplateValue("core.rename_table"), - "table", tgtTable, - "new_table", tgtTableTemp, - ) - _, err = conn.Exec(sql) - if err != nil { - return g.Error(err, "could not rename table "+tgtTable) - } - - sql = g.R( - conn.GetTemplateValue("core.rename_table"), - "table", srcTable, - "new_table", tgtTable, - ) - _, err = conn.Exec(sql) - if err != nil { - return g.Error(err, "could not rename table "+srcTable) - } - - sql = g.R( - conn.GetTemplateValue("core.rename_table"), - "table", tgtTableTemp, - "new_table", srcTable, - ) - _, err = conn.Exec(sql) - if err != nil { - return g.Error(err, "could not rename table "+tgtTableTemp) - } - - return -} - // GetNativeType returns the native column type from generic func (conn *BaseConn) GetNativeType(col iop.Column) (nativeType string, err error) { var ct iop.ColumnTyping @@ -2605,10 +2556,75 @@ func (conn *BaseConn) BulkExportFlowCSV(table Table) (df *iop.Dataflow, err erro return } -// GenerateUpsertSQL returns a sql for upsert -func (conn *BaseConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// MergeStrategy is for incremental loading +type MergeStrategy string + +const ( + MergeStrategyNone MergeStrategy = "" + MergeStrategyInsert MergeStrategy = "insert" + MergeStrategyUpdate MergeStrategy = "update" + MergeStrategyUpdateInsert MergeStrategy = "update_insert" + MergeStrategyDeleteInsert MergeStrategy = "delete_insert" +) - upsertMap, err := conn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) +// Merge inserts / updates from a srcTable into a target table. +// Assuming the srcTable has some or all of the tgtTable fields with matching types +func (conn *BaseConn) Merge(srcTable string, tgtTable string, primKeys []string) (rowAffCnt int64, err error) { + var cnt int64 + if conn.tx != nil { + cnt, err = Merge(conn.Self(), conn.tx, srcTable, tgtTable, primKeys) + } else { + cnt, err = Merge(conn.Self(), nil, srcTable, tgtTable, primKeys) + } + if err != nil { + err = g.Error(err, "could not upsert") + } + return cast.ToInt64(cnt), err +} + +// SwapTable swaps two table +func (conn *BaseConn) SwapTable(srcTable string, tgtTable string) (err error) { + + tgtTableTemp := tgtTable + "_tmp" + g.RandString(g.AlphaRunesLower, 2) + conn.DropTable(tgtTableTemp) + + sql := g.R( + conn.GetTemplateValue("core.rename_table"), + "table", tgtTable, + "new_table", tgtTableTemp, + ) + _, err = conn.Exec(sql) + if err != nil { + return g.Error(err, "could not rename table "+tgtTable) + } + + sql = g.R( + conn.GetTemplateValue("core.rename_table"), + "table", srcTable, + "new_table", tgtTable, + ) + _, err = conn.Exec(sql) + if err != nil { + return g.Error(err, "could not rename table "+srcTable) + } + + sql = g.R( + conn.GetTemplateValue("core.rename_table"), + "table", tgtTableTemp, + "new_table", srcTable, + ) + _, err = conn.Exec(sql) + if err != nil { + return g.Error(err, "could not rename table "+tgtTableTemp) + } + + return +} + +// GenerateMergeSQL returns a sql for upsert +func (conn *BaseConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { + + mc, err := conn.GenerateMergeConfig(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return @@ -2623,19 +2639,25 @@ func (conn *BaseConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFiel sqlTemplate, "src_table", srcTable, "tgt_table", tgtTable, - "src_tgt_pk_equal", upsertMap["src_tgt_pk_equal"], - "src_upd_pk_equal", strings.ReplaceAll(upsertMap["src_tgt_pk_equal"], "tgt.", "upd."), - "pk_fields", upsertMap["pk_fields"], - "set_fields", upsertMap["set_fields"], - "insert_fields", upsertMap["insert_fields"], - "src_fields", upsertMap["src_fields"], + "src_tgt_pk_equal", mc.Map["src_tgt_pk_equal"], + "src_upd_pk_equal", strings.ReplaceAll(mc.Map["src_tgt_pk_equal"], "tgt.", "upd."), + "pk_fields", mc.Map["pk_fields"], + "set_fields", mc.Map["set_fields"], + "insert_fields", mc.Map["insert_fields"], + "src_fields", mc.Map["src_fields"], ) return } -// GenerateUpsertExpressions returns a map with needed expressions -func (conn *BaseConn) GenerateUpsertExpressions(srcTable string, tgtTable string, pkFields []string) (exprs map[string]string, err error) { +type MergeConfig struct { + Strategy MergeStrategy + Template string + Map map[string]string +} + +// GenerateMergeConfig returns the merge config +func (conn *BaseConn) GenerateMergeConfig(srcTable string, tgtTable string, pkFields []string) (mc MergeConfig, err error) { srcColumns, err := conn.GetColumns(srcTable) if err != nil { @@ -2686,7 +2708,7 @@ func (conn *BaseConn) GenerateUpsertExpressions(srcTable string, tgtTable string srcCol := g.PtrVal(srcColumns.GetColumn(tgtColName)) // should be found tgtCol := tgtColumns.GetColumn(tgtColName) if tgtCol == nil { - return nil, g.Error("did not find target column: %s (has %s)", tgtColName, g.Marshal(tgtColumns.Names())) + return mc, g.Error("did not find target column: %s (has %s)", tgtColName, g.Marshal(tgtColumns.Names())) } // don't normalize, use raw name @@ -2710,22 +2732,50 @@ func (conn *BaseConn) GenerateUpsertExpressions(srcTable string, tgtTable string // cast into the correct type srcFields := conn.Self().CastColumnsForSelect(srcColumns, tgtColumns) - exprs = map[string]string{ - "src_tgt_pk_equal": strings.Join(pkEqualFields, " and "), - "src_upd_pk_equal": strings.ReplaceAll(strings.Join(pkEqualFields, ", "), "tgt.", "upd."), - "src_fields": strings.Join(srcFields, ", "), - "tgt_fields": strings.Join(tgtFields, ", "), - "insert_fields": strings.Join(insertFields, ", "), - "pk_fields": strings.Join(pkFields, ", "), - "src_pk_fields": strings.Join(srcPkFields, ", "), - "tgt_pk_fields": strings.Join(tgtPkFields, ", "), - "set_fields": strings.Join(setFields, ", "), - "placeholder_fields": strings.Join(placeholderFields, ", "), + mc = MergeConfig{ + Strategy: MergeStrategy(conn.GetTemplateValue("variable.merge_strategy")), + Template: "", + Map: map[string]string{ + "src_tgt_pk_equal": strings.Join(pkEqualFields, " and "), + "src_upd_pk_equal": strings.ReplaceAll(strings.Join(pkEqualFields, ", "), "tgt.", "upd."), + "src_fields": strings.Join(srcFields, ", "), + "tgt_fields": strings.Join(tgtFields, ", "), + "insert_fields": strings.Join(insertFields, ", "), + "pk_fields": strings.Join(pkFields, ", "), + "src_pk_fields": strings.Join(srcPkFields, ", "), + "tgt_pk_fields": strings.Join(tgtPkFields, ", "), + "set_fields": strings.Join(setFields, ", "), + "placeholder_fields": strings.Join(placeholderFields, ", "), + }, + } + + switch mc.Strategy { + case MergeStrategyInsert, MergeStrategyUpdate, MergeStrategyUpdateInsert, MergeStrategyDeleteInsert: + case MergeStrategyNone: + return mc, g.Error("no default merge strategy specified for %s", conn.GetType()) + default: + return mc, g.Error("invalid merge strategy %s", mc.Strategy) + } + + key := g.F("core.merge_%s", mc.Strategy) + mc.Template = conn.GetTemplateValue(key) + + if mc.Template == "" { + return mc, g.Error("merge strategy `%s` not supported for %s (did not find SQL template key `%s`)", mc.Strategy, conn.GetType(), key) } return } +// GenerateMergeExpressions returns a map with needed expressions +func (conn *BaseConn) GenerateMergeExpressions(srcTable string, tgtTable string, pkFields []string) (upsertMap map[string]string, err error) { + mc, err := conn.GenerateMergeConfig(srcTable, tgtTable, pkFields) + if err != nil { + return nil, err + } + return mc.Map, nil +} + // GetColumnStats analyzes the table and returns the column statistics func (conn *BaseConn) GetColumnStats(tableName string, fields ...string) (columns iop.Columns, err error) { diff --git a/core/dbio/database/database_arrow_flight.go b/core/dbio/database/database_arrow_flight.go new file mode 100644 index 000000000..d5fd93b8f --- /dev/null +++ b/core/dbio/database/database_arrow_flight.go @@ -0,0 +1,25 @@ +package database + +import ( + "github.com/slingdata-io/sling-cli/core/dbio" +) + +// ArrowFlightConn is an Arrow FlightSQL connection +type ArrowFlightConn struct { + BaseConn + URL string +} + +// Init initiates the connection +func (conn *ArrowFlightConn) Init() error { + conn.BaseConn.URL = conn.URL + conn.BaseConn.Type = dbio.TypeDbExasol + conn.BaseConn.defaultPort = 12345 + + // exasol driver is auto-registered on import + + instance := Connection(conn) + conn.BaseConn.instance = &instance + + return conn.BaseConn.Init() +} diff --git a/core/dbio/database/database_athena.go b/core/dbio/database/database_athena.go index aa0d27978..d61d3c269 100644 --- a/core/dbio/database/database_athena.go +++ b/core/dbio/database/database_athena.go @@ -827,9 +827,9 @@ func (conn *AthenaConn) BulkImportStream(tableFName string, ds *iop.Datastream) return conn.BulkImportFlow(tableFName, df) } -// GenerateUpsertSQL generates the upsert SQL -func (conn *AthenaConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) +// GenerateMergeSQL generates the upsert SQL +func (conn *AthenaConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_azuretable.go b/core/dbio/database/database_azuretable.go index acbda8128..2c114c9fd 100644 --- a/core/dbio/database/database_azuretable.go +++ b/core/dbio/database/database_azuretable.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "net/url" "regexp" "strings" "time" @@ -27,6 +28,7 @@ type AzureTableConn struct { AccountKey string SASToken string ConnectionStr string + TableName string // Specific table name for table-specific SAS tokens } // Init initiates the object @@ -56,6 +58,17 @@ func (conn *AzureTableConn) getNewClient(timeOut ...int) (client *aztables.Servi return client, nil } else if conn.SASToken != "" { // Use SAS token + // Check if the SAS token contains a table name in the URL + if strings.Contains(conn.SASToken, "tn=") { + // Extract table name from SAS token if present + if u, err := url.Parse("?" + strings.TrimPrefix(conn.SASToken, "?")); err == nil { + if tn := u.Query().Get("tn"); tn != "" { + conn.TableName = tn + g.Debug("Extracted table name from SAS token: %s", tn) + } + } + } + sasURL := serviceURL + "?" + strings.TrimPrefix(conn.SASToken, "?") client, err = aztables.NewServiceClientWithNoCredential(sasURL, nil) if err != nil { @@ -100,6 +113,7 @@ func (conn *AzureTableConn) Connect(timeOut ...int) error { } conn.SASToken = conn.GetProp("sas_token") conn.ConnectionStr = conn.GetProp("conn_str") + conn.TableName = conn.GetProp("table_name") var err error conn.Client, err = conn.getNewClient(timeOut...) @@ -107,15 +121,28 @@ func (conn *AzureTableConn) Connect(timeOut ...int) error { return g.Error(err, "Failed to connect to client") } - // Test connection by listing tables - ctx, cancel := context.WithTimeout(conn.BaseConn.Context().Ctx, 5*time.Second) - defer cancel() - - pager := conn.Client.NewListTablesPager(nil) - if pager.More() { - _, err = pager.NextPage(ctx) - if err != nil { - return g.Error(err, "Failed to list tables") + // Test connection + // For SAS tokens, we may not have permissions to list tables + // Only test listing if we're not using a SAS token or if it's likely an account-level SAS + if conn.SASToken == "" || strings.Contains(conn.SASToken, "ss=t") { + // ss=t indicates table service permissions in account SAS + ctx, cancel := context.WithTimeout(conn.BaseConn.Context().Ctx, 5*time.Second) + defer cancel() + + pager := conn.Client.NewListTablesPager(nil) + if pager.More() { + _, err = pager.NextPage(ctx) + if err != nil { + // If using SAS token and we get authorization error, that's ok + // The SAS token might be table-specific + if conn.SASToken != "" && strings.Contains(err.Error(), "AuthorizationFailure") { + g.Debug("SAS token does not have list permissions, assuming table-specific SAS") + } else if strings.Contains(err.Error(), "AuthorizationResourceTypeMismatch") { + g.Debug("SAS token does not have list permissions: AuthorizationResourceTypeMismatch") + } else { + g.Warn("Failed to list tables: %s", err.Error()) + } + } } } @@ -645,13 +672,42 @@ func (conn *AzureTableConn) TableExists(table Table) (exists bool, err error) { // Sanitize table name for Azure Table Storage sanitizedName := sanitizeAzureTableName(table.Name) - // List tables and check if our table exists + // If we have a specific table name from SAS token, check if it matches + if conn.TableName != "" { + return sanitizedName == conn.TableName, nil + } + + // Try to list tables and check if our table exists ctx := conn.Context().Ctx pager := conn.Client.NewListTablesPager(nil) for pager.More() { page, err := pager.NextPage(ctx) if err != nil { + // Handle authorization errors for SAS tokens + if conn.SASToken != "" && strings.Contains(err.Error(), "AuthorizationFailure") { + // Can't list tables, try to access the table directly + tableClient := conn.Client.NewClient(sanitizedName) + // Try to get a single entity to check if table exists + queryOptions := &aztables.ListEntitiesOptions{ + Top: g.Ptr(int32(1)), + } + pager := tableClient.NewListEntitiesPager(queryOptions) + if pager.More() { + _, err := pager.NextPage(ctx) + if err == nil { + return true, nil + } + // Check if error is table not found + if strings.Contains(err.Error(), "TableNotFound") || strings.Contains(err.Error(), "ResourceNotFound") { + return false, nil + } + // Some other error (could be auth or network issue) + return false, g.Error(err, "could not verify table existence") + } + // Assume table exists if we can't verify otherwise + return true, nil + } return false, g.Error(err, "could not list tables") } @@ -700,14 +756,27 @@ func (conn *AzureTableConn) GetSchemas() (data iop.Dataset, err error) { // GetTables returns tables func (conn *AzureTableConn) GetTables(schema string) (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("table_name")) + + // If we have a specific table name (from table-specific SAS), return just that + if conn.TableName != "" { + data.Append([]any{conn.TableName}) + return data, nil + } + + // Try to list tables ctx := conn.Context().Ctx pager := conn.Client.NewListTablesPager(nil) - data = iop.NewDataset(iop.NewColumnsFromFields("table_name")) - for pager.More() { page, err := pager.NextPage(ctx) if err != nil { + // Handle authorization errors gracefully for SAS tokens + if conn.SASToken != "" && strings.Contains(err.Error(), "AuthorizationFailure") { + g.Debug("Cannot list tables with current SAS token permissions") + // Return empty dataset instead of error + return data, nil + } return data, g.Error(err, "could not list tables") } diff --git a/core/dbio/database/database_bigquery.go b/core/dbio/database/database_bigquery.go index 6cf930d3f..061f53f47 100755 --- a/core/dbio/database/database_bigquery.go +++ b/core/dbio/database/database_bigquery.go @@ -614,7 +614,6 @@ func (conn *BigQueryConn) importStreamConfig(columns iop.Columns) (config iop.St } else if hasNumeric { config.MaxDecimals = 9 } - config.SetMaxDecimals(config.MaxDecimals) } return config @@ -1087,10 +1086,10 @@ func (conn *BigQueryConn) CastColumnForSelect(srcCol iop.Column, tgtCol iop.Colu return selectStr } -// GenerateUpsertSQL generates the upsert SQL -func (conn *BigQueryConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *BigQueryConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_clickhouse.go b/core/dbio/database/database_clickhouse.go index 1bbb71dae..9157cd685 100755 --- a/core/dbio/database/database_clickhouse.go +++ b/core/dbio/database/database_clickhouse.go @@ -549,9 +549,9 @@ func (conn *ClickhouseConn) GenerateInsertStatement(tableName string, cols iop.C return statement } -// GenerateUpsertSQL generates the upsert SQL -func (conn *ClickhouseConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) +// GenerateMergeSQL generates the upsert SQL +func (conn *ClickhouseConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_d1.go b/core/dbio/database/database_d1.go index 9b5354524..c1adb1ba1 100644 --- a/core/dbio/database/database_d1.go +++ b/core/dbio/database/database_d1.go @@ -584,10 +584,10 @@ func (conn *D1Conn) GetSchemata(level SchemataLevel, schemaName string, tableNam return schemata, nil } -// GenerateUpsertSQL generates the upsert SQL -func (conn *D1Conn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *D1Conn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_databricks.go b/core/dbio/database/database_databricks.go index e72183814..59b6a4d58 100644 --- a/core/dbio/database/database_databricks.go +++ b/core/dbio/database/database_databricks.go @@ -728,9 +728,9 @@ func (conn *DatabricksConn) CastColumnForSelect(srcCol iop.Column, tgtCol iop.Co return selectStr } -// GenerateUpsertSQL generates the upsert SQL -func (conn *DatabricksConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) +// GenerateMergeSQL generates the upsert SQL +func (conn *DatabricksConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_duckdb.go b/core/dbio/database/database_duckdb.go index bfb6a3888..f6acecf12 100644 --- a/core/dbio/database/database_duckdb.go +++ b/core/dbio/database/database_duckdb.go @@ -133,7 +133,7 @@ func (conn *DuckDbConn) Connect(timeOut ...int) (err error) { // add extensions if conn.GetProp("copy_method") == "arrow_http" { - conn.duck.AddExtension("arrow") + conn.duck.AddExtension("arrow from community") } return nil @@ -303,10 +303,10 @@ func (conn *DuckDbConn) generateCsvColumns(columns iop.Columns) (colStr string) return conn.duck.GenerateCsvColumns(columns) } -// GenerateUpsertSQL generates the upsert SQL -func (conn *DuckDbConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *DuckDbConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_ducklake.go b/core/dbio/database/database_ducklake.go index ee31b073a..bf28e5993 100644 --- a/core/dbio/database/database_ducklake.go +++ b/core/dbio/database/database_ducklake.go @@ -53,6 +53,10 @@ func (conn *DuckLakeConn) Init() error { return g.Error("did not provide 'catalog_conn_string'") } + if _, ok := conn.properties["copy_method"]; !ok { + conn.SetProp("copy_method", "arrow_http") // default to arrow_http + } + // Set base properties conn.BaseConn.URL = conn.URL conn.BaseConn.Type = dbio.TypeDbDuckLake @@ -188,11 +192,11 @@ func (conn *DuckLakeConn) GetURL(newURL ...string) string { return connURL } -// GenerateUpsertSQL generates the upsert SQL for DuckLake -func (conn *DuckLakeConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL for DuckLake +func (conn *DuckLakeConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { // DuckLake supports transactions and proper ACID operations // For now, use the same approach as DuckDB - return conn.DuckDbConn.GenerateUpsertSQL(srcTable, tgtTable, pkFields) + return conn.DuckDbConn.GenerateMergeSQL(srcTable, tgtTable, pkFields) } func (conn *DuckLakeConn) SubmitTemplate(level string, templateMap map[string]string, name string, values map[string]any) (data iop.Dataset, err error) { diff --git a/core/dbio/database/database_exasol.go b/core/dbio/database/database_exasol.go index abccbe327..bafa7db8a 100644 --- a/core/dbio/database/database_exasol.go +++ b/core/dbio/database/database_exasol.go @@ -582,9 +582,9 @@ func (conn *ExasolConn) writeDataflowToCSV(ds *iop.Datastream, folderPath string return nil } -// GenerateUpsertSQL generates the upsert SQL for Exasol -func (conn *ExasolConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) +// GenerateMergeSQL generates the upsert SQL for Exasol +func (conn *ExasolConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_mysql.go b/core/dbio/database/database_mysql.go index c0c93a33f..9bfd43746 100755 --- a/core/dbio/database/database_mysql.go +++ b/core/dbio/database/database_mysql.go @@ -314,10 +314,10 @@ func (conn *MySQLConn) LoadDataInFile(tableFName string, ds *iop.Datastream) (co // UPSERT // https://vladmihalcea.com/how-do-upsert-and-merge-work-in-oracle-sql-server-postgresql-and-mysql/ -// GenerateUpsertSQL generates the upsert SQL -func (conn *MySQLConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *MySQLConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_oracle.go b/core/dbio/database/database_oracle.go index 54befb01b..a5085ad16 100755 --- a/core/dbio/database/database_oracle.go +++ b/core/dbio/database/database_oracle.go @@ -58,9 +58,6 @@ func (conn *OracleConn) Init() error { instance := Connection(conn) conn.BaseConn.instance = &instance - // set MAX_DECIMALS to import for numeric types - conn.SetProp("MAX_DECIMALS", "9") - return conn.BaseConn.Init() } @@ -526,10 +523,10 @@ func (conn *OracleConn) writeCsv(ds *iop.Datastream, writer io.Writer, pu *cmap. return } -// GenerateUpsertSQL generates the upsert SQL -func (conn *OracleConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *OracleConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_postgres.go b/core/dbio/database/database_postgres.go index 193b3e933..4f9a9d227 100755 --- a/core/dbio/database/database_postgres.go +++ b/core/dbio/database/database_postgres.go @@ -270,88 +270,47 @@ func (conn *PostgresConn) CastColumnForSelect(srcCol iop.Column, tgtCol iop.Colu return selectStr } -// GenerateUpsertSQL generates the upsert SQL -func (conn *PostgresConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *PostgresConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + mc, err := conn.BaseConn.GenerateMergeConfig(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return } pkFieldsQ := lo.Map(pkFields, func(f string, i int) string { return conn.Quote(f) }) - indexSQL := g.R( - conn.GetTemplateValue("core.create_unique_index"), - "index", strings.Join(pkFields, "_")+"_idx", - "table", tgtTable, - "cols", strings.Join(pkFieldsQ, ", "), - ) - - // in order to use on conflict, the target table needs - // a unique index on the PK. We will not use it since - // it complicates matters - sqlTempl := ` - {indexSQL}; - insert into {tgt_table} - ({insert_fields}) - select {src_fields} from {src_table} src - on conflict ({tgt_pk_fields}) - DO UPDATE - SET {set_fields} - ` tempTable := g.RandSuffix("temp", 5) - tempIndexSQL := g.R( + tempTableIndexSQL := g.R( conn.GetTemplateValue("core.create_unique_index"), "index", tempTable+"_idx", "table", tempTable, "cols", strings.Join(pkFieldsQ, ", "), ) - sqlTempl = ` - create temporary table {temp_table} as - with src_table as ( - select {src_fields} from {src_table} - ) - , updates as ( - update {tgt_table} tgt - set {set_fields} - from src_table src - where {src_tgt_pk_equal} - returning tgt.* - ) - select * from updates; - - {tempIndexSQL}; - - with src_table as ( - select {src_fields} from {src_table} - ) - insert into {tgt_table} - ({insert_fields}) - select {src_fields} from src_table src - where not exists ( - select 1 - from {temp_table} upd - where {src_upd_pk_equal} + finalTableIndexSQL := g.R( + conn.GetTemplateValue("core.create_unique_index"), + "index", strings.Join(pkFields, "_")+"_idx", + "table", tgtTable, + "cols", strings.Join(pkFieldsQ, ", "), ) - ` sql = g.R( - sqlTempl, + mc.Template, "src_table", srcTable, "tgt_table", tgtTable, "temp_table", tempTable, - "indexSQL", indexSQL, - "tempIndexSQL", tempIndexSQL, - "src_tgt_pk_equal", upsertMap["src_tgt_pk_equal"], - "src_upd_pk_equal", strings.ReplaceAll(upsertMap["src_tgt_pk_equal"], "tgt.", "upd."), - "src_fields", upsertMap["src_fields"], - "tgt_pk_fields", upsertMap["tgt_pk_fields"], + "final_table_index_sql", finalTableIndexSQL, + "temp_table_index_sql", tempTableIndexSQL, + "src_tgt_pk_equal", mc.Map["src_tgt_pk_equal"], + "src_upd_pk_equal", strings.ReplaceAll(mc.Map["src_tgt_pk_equal"], "tgt.", "upd."), + "src_fields", mc.Map["src_fields"], + "tgt_pk_fields", mc.Map["tgt_pk_fields"], // "set_fields", strings.ReplaceAll(upsertMap["set_fields"], "src.", "excluded."), - "set_fields", upsertMap["set_fields"], - "insert_fields", upsertMap["insert_fields"], + "set_fields", mc.Map["set_fields"], + "insert_fields", mc.Map["insert_fields"], ) return diff --git a/core/dbio/database/database_prometheus.go b/core/dbio/database/database_prometheus.go index a9b95e9b9..773c98650 100644 --- a/core/dbio/database/database_prometheus.go +++ b/core/dbio/database/database_prometheus.go @@ -104,7 +104,45 @@ func (conn *PrometheusConn) NewTransaction(ctx context.Context, options ...*sql. return } func (conn *PrometheusConn) GetSQLColumns(table Table) (columns iop.Columns, err error) { - return iop.Columns{{Name: "metric"}}, nil + // For Prometheus, we need to examine the query to determine expected columns + // Default columns for time series data + columns = iop.Columns{ + {Name: "__name__", Type: iop.StringType, Position: 1}, + {Name: "job", Type: iop.StringType, Position: 2}, + {Name: "instance", Type: iop.StringType, Position: 3}, + {Name: "timestamp", Type: iop.BigIntType, Position: 4}, + {Name: "value", Type: iop.DecimalType, Position: 5}, + } + + // Try to get actual columns by querying with a very small time range + if table.SQL != "" { + // Extract the base query without options + baseQuery := table.SQL + if idx := strings.Index(baseQuery, "#"); idx != -1 { + baseQuery = strings.TrimSpace(baseQuery[:idx]) + } + + // Query with a minimal time range to get column structure + testOpts := g.M( + "start", "now-1m", + "end", "now", + "step", "1m", + "limit", 1, + "get_columns", true, + ) + + ds, err := conn.StreamRowsContext(conn.Context().Ctx, baseQuery, testOpts) + if err == nil && ds != nil { + // Wait for columns to be initialized + ds.WaitReady() + if len(ds.Columns) > 0 { + columns = ds.Columns + } + ds.Close() + } + } + + return columns, nil } // NewTransaction creates a new transaction @@ -125,7 +163,7 @@ func (conn *PrometheusConn) GetTableColumns(table *Table, fields ...string) (col return nil, g.Error("did not find collection %s", table.FullName()) } - ds, err := conn.StreamRows(table.FullName(), g.M("limit", 10)) + ds, err := conn.StreamRows(table.FullName(), g.M("limit", 10, "silent", true, "get_columns", true)) if err != nil { return columns, g.Error("could not query to get columns") } @@ -151,6 +189,11 @@ func (conn *PrometheusConn) BulkExportFlow(table Table) (df *iop.Dataflow, err e g.Debug("query options: %s", g.Marshal(options)) } + // Pass table columns for fallback when no data is returned + if len(table.Columns) > 0 { + options["columns"] = table.Columns + } + ds, err := conn.StreamRowsContext(conn.Context().Ctx, table.SQL, options) if err != nil { return df, g.Error(err, "could start datastream") @@ -505,6 +548,17 @@ func (conn *PrometheusConn) StreamRowsContext(ctx context.Context, query string, return nil, g.Error("invalid result: %#v", result) } + // If no columns were detected (no data), use columns from options + if len(data.Columns) == 0 && opts["columns"] != nil { + switch cols := opts["columns"].(type) { + case iop.Columns: + data.Columns = cols + default: + // Try to convert from options + g.JSONConvert(opts["columns"], &data.Columns) + } + } + ds = data.Stream(conn.Props()) return @@ -824,8 +878,17 @@ func (conn *PrometheusConn) StreamRowsChunked(queryContext *g.Context, query str time.Sleep(100 * time.Millisecond) } - // If no columns were initialized (no data), create empty columns and set ready + // If no columns were initialized (no data), use columns from options if available if !columnsInitialized { + if opts["columns"] != nil { + switch cols := opts["columns"].(type) { + case iop.Columns: + ds.Columns = cols + default: + // Try to convert from options + g.JSONConvert(opts["columns"], &ds.Columns) + } + } ds.SetReady() } }() diff --git a/core/dbio/database/database_proton.go b/core/dbio/database/database_proton.go index a67c9f6d0..ed142790e 100755 --- a/core/dbio/database/database_proton.go +++ b/core/dbio/database/database_proton.go @@ -320,9 +320,9 @@ func (conn *ProtonConn) GenerateInsertStatement(tableName string, cols iop.Colum return statement } -// GenerateUpsertSQL generates the upsert SQL -func (conn *ProtonConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) +// GenerateMergeSQL generates the upsert SQL +func (conn *ProtonConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_redshift.go b/core/dbio/database/database_redshift.go index e1a7ef7bd..6b843b6aa 100755 --- a/core/dbio/database/database_redshift.go +++ b/core/dbio/database/database_redshift.go @@ -389,10 +389,10 @@ func (conn *RedshiftConn) BulkImportStream(tableFName string, ds *iop.Datastream return conn.BulkImportFlow(tableFName, df) } -// GenerateUpsertSQL generates the upsert SQL -func (conn *RedshiftConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *RedshiftConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_snowflake.go b/core/dbio/database/database_snowflake.go index 5fe74a605..515f70eda 100755 --- a/core/dbio/database/database_snowflake.go +++ b/core/dbio/database/database_snowflake.go @@ -1002,10 +1002,10 @@ func (conn *SnowflakeConn) StagePUT(fileURI string, internalStagePath string) (e return } -// GenerateUpsertSQL generates the upsert SQL -func (conn *SnowflakeConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *SnowflakeConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_sqlite.go b/core/dbio/database/database_sqlite.go index 8a05cfeb8..1035cfd12 100644 --- a/core/dbio/database/database_sqlite.go +++ b/core/dbio/database/database_sqlite.go @@ -250,10 +250,10 @@ func (conn *SQLiteConn) BulkImportStream(tableFName string, ds *iop.Datastream) return ds.Count, nil } -// GenerateUpsertSQL generates the upsert SQL -func (conn *SQLiteConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *SQLiteConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/database_sqlserver.go b/core/dbio/database/database_sqlserver.go index 582b871b4..c95d83356 100755 --- a/core/dbio/database/database_sqlserver.go +++ b/core/dbio/database/database_sqlserver.go @@ -680,10 +680,10 @@ func (conn *MsSQLServerConn) BcpExport() (err error) { //UPSERT // https://vladmihalcea.com/how-do-upsert-and-merge-work-in-oracle-sql-server-postgresql-and-mysql/ -// GenerateUpsertSQL generates the upsert SQL -func (conn *MsSQLServerConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { +// GenerateMergeSQL generates the upsert SQL +func (conn *MsSQLServerConn) GenerateMergeSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error) { - upsertMap, err := conn.BaseConn.GenerateUpsertExpressions(srcTable, tgtTable, pkFields) + upsertMap, err := conn.BaseConn.GenerateMergeExpressions(srcTable, tgtTable, pkFields) if err != nil { err = g.Error(err, "could not generate upsert variables") return diff --git a/core/dbio/database/dbx.go b/core/dbio/database/dbx.go index 1dd888b98..39ffb8139 100644 --- a/core/dbio/database/dbx.go +++ b/core/dbio/database/dbx.go @@ -441,8 +441,8 @@ func (x *DbX) Update(o interface{}, fields ...string) (cnt int, err error) { return } -// Upsert upserts from object or slice -func (x *DbX) Upsert(o interface{}, fields ...string) (cnt int, err error) { +// Merge upserts from object or slice +func (x *DbX) Merge(o interface{}, fields ...string) (cnt int, err error) { defer x.Where() // reset where clause return } diff --git a/core/dbio/database/transaction.go b/core/dbio/database/transaction.go index 43b3c9e0e..649ac8ad1 100644 --- a/core/dbio/database/transaction.go +++ b/core/dbio/database/transaction.go @@ -223,9 +223,9 @@ func (t *BaseTransaction) InsertBatchStream(tableFName string, ds *iop.Datastrea return } -// Upsert does an upsert from source table into target table -func (t *BaseTransaction) Upsert(sourceTable, targetTable string, pkFields []string) (count uint64, err error) { - cnt, err := Upsert(t.Conn, t, sourceTable, targetTable, pkFields) +// Merge does an upsert from source table into target table +func (t *BaseTransaction) Merge(sourceTable, targetTable string, pkFields []string) (count uint64, err error) { + cnt, err := Merge(t.Conn, t, sourceTable, targetTable, pkFields) if err != nil { err = g.Error(err, "Could not upsert from %s into %s", sourceTable, targetTable) } @@ -448,8 +448,8 @@ func InsertBatchStream(conn Connection, tx Transaction, tableFName string, ds *i return count, nil } -// Upsert upserts from source table into target table -func Upsert(conn Connection, tx Transaction, sourceTable, targetTable string, pkFields []string) (count int64, err error) { +// Merge upserts from source table into target table +func Merge(conn Connection, tx Transaction, sourceTable, targetTable string, pkFields []string) (count int64, err error) { srcTable, err := ParseTableName(sourceTable, conn.GetType()) if err != nil { @@ -463,7 +463,7 @@ func Upsert(conn Connection, tx Transaction, sourceTable, targetTable string, pk return } - q, err := conn.GenerateUpsertSQL(srcTable.FullName(), tgtTable.FullName(), pkFields) + q, err := conn.Self().GenerateMergeSQL(srcTable.FullName(), tgtTable.FullName(), pkFields) if err != nil { err = g.Error(err, "could not generate upsert sql") return diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index efb632f23..ed1e5f803 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -84,6 +84,7 @@ const ( TypeDbIceberg Type = "iceberg" TypeDbAzureTable Type = "azuretable" TypeDbExasol Type = "exasol" + TypeDbArrowFlight Type = "flightsql" ) var AllType = []struct { diff --git a/core/dbio/filesys/fs.go b/core/dbio/filesys/fs.go index 3d899eb96..9be1cfb30 100755 --- a/core/dbio/filesys/fs.go +++ b/core/dbio/filesys/fs.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "net/url" "os" "path" "path/filepath" @@ -224,7 +225,7 @@ func NormalizeURI(fs FileSysClient, uri string) string { path = strings.TrimPrefix(path, u.U.User.Username()) path = strings.TrimPrefix(path, ":") password, _ := u.U.User.Password() - path = strings.TrimPrefix(path, password) + path = strings.TrimPrefix(path, url.QueryEscape(password)) path = strings.TrimPrefix(path, "@") path = strings.TrimPrefix(path, u.U.Host) path = strings.TrimPrefix(path, "/") @@ -238,7 +239,7 @@ func NormalizeURI(fs FileSysClient, uri string) string { path = strings.TrimPrefix(path, u.U.User.Username()) path = strings.TrimPrefix(path, ":") password, _ := u.U.User.Password() - path = strings.TrimPrefix(path, password) + path = strings.TrimPrefix(path, url.QueryEscape(password)) path = strings.TrimPrefix(path, "@") path = strings.TrimPrefix(path, u.U.Host) path = strings.TrimPrefix(path, "/") @@ -959,10 +960,12 @@ func (fs *BaseFileSysClient) WriteDataflowReady(df *iop.Dataflow, url string, fi localCtx.Wg.Read.Wait() } - err = Delete(fsClient, url) - if err != nil { - err = g.Error(err, "Could not delete url") - return + if sc.DeleteFile { + err = Delete(fsClient, url) + if err != nil { + err = g.Error(err, "Could not delete url") + return + } } if !singleFile && g.In(fsClient.FsType(), dbio.TypeFileLocal, dbio.TypeFileSftp, dbio.TypeFileFtp) { @@ -1300,6 +1303,17 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw fileFormat = InferFileFormat(uri) } + if deleteURI := uri; sc.DeleteFile { + if strings.Contains(deleteURI, "*") { + deleteURI = GetDeepestParent(deleteURI) // get target folder, since split by files + } + err = Delete(fs, deleteURI) + if err != nil { + err = g.Error(err, "Could not delete uri") + return bw, err + } + } + for streamPart := range streamPartChn { copyOptions := iop.DuckDbCopyOptions{ Format: fileFormat, @@ -1323,17 +1337,15 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw } // create the parent folder if needed - if fs.FsType() == dbio.TypeFileLocal { - parent := path.Dir(localPath) - if err = os.MkdirAll(parent, 0755); err != nil { - err = g.Error(err, "Could not create output folder") - return bw, err - } - if duckSc.FileMaxRows > 0 { - copyOptions.FileSizeBytes = 0 // since we are splitting by rows already - os.MkdirAll(localPath, 0755) // make root dir first - localPath = g.F("%s/data_%03d.parquet", localPath, streamPart.Index+1) - } + parent := path.Dir(localPath) + if err = os.MkdirAll(parent, 0755); err != nil { + err = g.Error(err, "Could not create output folder") + return bw, err + } + if duckSc.FileMaxRows > 0 { + copyOptions.FileSizeBytes = 0 // since we are splitting by rows already + os.MkdirAll(localPath, 0755) // make root dir first + localPath = g.F("%s/data_%03d.parquet", localPath, streamPart.Index+1) } sql, err := duck.GenerateCopyStatement(streamPart.FromExpr, localPath, copyOptions) @@ -1383,12 +1395,6 @@ func WriteDataflowViaDuckDB(fs FileSysClient, df *iop.Dataflow, uri string) (bw uri = GetDeepestParent(uri) // get target folder, since split by files } - err = Delete(fs, uri) - if err != nil { - err = g.Error(err, "Could not delete uri") - return bw, err - } - written, err := CopyFromLocalRecursive(fs, localRoot, uri) if err != nil { err = g.Error(err, "Could not write to file") diff --git a/core/dbio/filesys/fs_file_node.go b/core/dbio/filesys/fs_file_node.go index 7770bba42..d5b4472fb 100644 --- a/core/dbio/filesys/fs_file_node.go +++ b/core/dbio/filesys/fs_file_node.go @@ -286,7 +286,7 @@ func ParseURLType(uri string) (uType dbio.Type, host string, path string, err er host = u.Hostname() path = strings.TrimPrefix(uri, g.F("%s://%s/", scheme, u.U.Host)) - // g.Info("uri => %s, host => %s, path => %s (%s)", uri, host, path, u.U.Path) + // g.Info("uri => %s | host => %s | path => %s (%s)", uri, host, path, u.U.Path) if scheme == "" || host == "" { err = errors.New("Invalid URL: " + uri) diff --git a/core/dbio/filesys/fs_ftp.go b/core/dbio/filesys/fs_ftp.go index a6283b6ba..b681e641b 100644 --- a/core/dbio/filesys/fs_ftp.go +++ b/core/dbio/filesys/fs_ftp.go @@ -204,7 +204,7 @@ func (fs *FtpFileSysClient) List(url string) (nodes FileNodes, err error) { entries, err := fs.client.List(path) if err != nil { - return nodes, g.Error(err, "error listing path") + return nodes, g.Error(err, "error listing path: %#v", path) } for _, file := range entries { diff --git a/core/dbio/filesys/fs_sftp.go b/core/dbio/filesys/fs_sftp.go index 13c3ddefb..e7734ab2e 100644 --- a/core/dbio/filesys/fs_sftp.go +++ b/core/dbio/filesys/fs_sftp.go @@ -167,7 +167,7 @@ func (fs *SftpFileSysClient) List(url string) (nodes FileNodes, err error) { } files, err = fs.client.ReadDir(path) if err != nil { - return nodes, g.Error(err, "error listing path") + return nodes, g.Error(err, "error listing path: %#v", path) } if path == "/" { @@ -223,7 +223,7 @@ func (fs *SftpFileSysClient) ListRecursive(uri string) (nodes FileNodes, err err path = strings.TrimSuffix(path, "/") files, err = fs.client.ReadDir(path) if err != nil { - return nodes, g.Error(err, "error listing path") + return nodes, g.Error(err, "error listing path: %#v", path) } for _, file := range files { @@ -237,7 +237,7 @@ func (fs *SftpFileSysClient) ListRecursive(uri string) (nodes FileNodes, err err if file.IsDir() { subNodes, err := fs.ListRecursive(node.Path() + "/") if err != nil { - return nil, g.Error(err, "error listing sub path") + return nil, g.Error(err, "error listing sub path: %#v", node.Path()) } nodes.AddWhere(pattern, ts, subNodes...) } else { diff --git a/core/dbio/iop/dataset.go b/core/dbio/iop/dataset.go index 4f0b318fb..e522ac55f 100644 --- a/core/dbio/iop/dataset.go +++ b/core/dbio/iop/dataset.go @@ -16,6 +16,7 @@ import ( "github.com/jedib0t/go-pretty/table" "github.com/jmoiron/sqlx" "github.com/samber/lo" + "github.com/shopspring/decimal" "github.com/spf13/cast" ) @@ -383,7 +384,7 @@ func (data *Dataset) RecordsString(lower ...bool) []map[string]string { if row[j] == nil { rec[field] = "" } else { - rec[field] = cast.ToString(row[j]) + rec[field] = data.Sp.CastToString(row[j]) } } records[i] = rec @@ -404,7 +405,7 @@ func (data *Dataset) RecordsCasted(lower ...bool) []map[string]any { if row[j] == nil { rec[field] = nil } else { - rec[field] = data.Sp.ParseString(cast.ToString(row[j]), j) + rec[field] = data.Sp.ParseString(data.Sp.CastToString(row[j]), j) } } records[i] = rec @@ -458,7 +459,7 @@ func (data *Dataset) InferColumnTypes() { valStr := "" if val != nil { - valStr = cast.ToString(val) + valStr = data.Sp.CastToString(val) // Handle maps by marshaling them to JSON if reflect.TypeOf(val) != nil && reflect.TypeOf(val).Kind() == reflect.Map { valStr = g.Marshal(val) @@ -506,23 +507,54 @@ func (data *Dataset) InferColumnTypes() { if val0 < columns[j].Stats.Min { columns[j].Stats.Min = val0 } - case float32, float64: + case decimal.Decimal: + columns[j].Stats.DecCnt++ - val0 := cast.ToInt64(val) - if val0 > columns[j].Stats.Max { - columns[j].Stats.Max = val0 - } - if val0 < columns[j].Stats.Min { - columns[j].Stats.Min = val0 - } - if parts := strings.Split(cast.ToString(val), "."); len(parts) == 2 { + valStr = strings.Replace(valStr, ",", ".", 1) // when comma is used a a decimal point + if parts := strings.Split(valStr, "."); len(parts) == 2 { + + val0 := cast.ToInt64(parts[0]) + if val0 > columns[j].Stats.Max { + columns[j].Stats.Max = val0 + } + if val0 < columns[j].Stats.Min { + columns[j].Stats.Min = val0 + } + decLen := len(parts[1]) if decLen > columns[j].Stats.MaxDecLen { columns[j].Stats.MaxDecLen = decLen } + } else { + val0 := cast.ToInt64(valStr) + if val0 > columns[j].Stats.Max { + columns[j].Stats.Max = val0 + } + if val0 < columns[j].Stats.Min { + columns[j].Stats.Min = val0 + } } + // float64 can only handle up to 15 total digits + // case float32, float64: + // columns[j].Stats.DecCnt++ + // val0 := cast.ToInt64(val) + // if val0 > columns[j].Stats.Max { + // columns[j].Stats.Max = val0 + // } + // if val0 < columns[j].Stats.Min { + // columns[j].Stats.Min = val0 + // } + + // if parts := strings.Split(data.Sp.CastToString(val), "."); len(parts) == 2 { + // g.Warn("%#v", parts) + // decLen := len(parts[1]) + // if decLen > columns[j].Stats.MaxDecLen { + // columns[j].Stats.MaxDecLen = decLen + // } + // } + case bool: columns[j].Stats.BoolCnt++ case string, []uint8: diff --git a/core/dbio/iop/datastream.go b/core/dbio/iop/datastream.go index 6a2c56f78..0e5e723b5 100644 --- a/core/dbio/iop/datastream.go +++ b/core/dbio/iop/datastream.go @@ -733,12 +733,26 @@ loop: // determine if column type changed for i, oldVal := range row { + col := ds.Columns[i] newVal := newRow[i] - colType := ds.Columns[i].Type + colType := col.Type oldType := ds.Sp.CheckType(oldVal) newType := ds.Sp.CheckType(newVal) if oldType != newType && colType != newType { - ds.Sp.ds.ChangeColumn(i, newType) + switch { + case colType.IsDatetime() && TimestampzType.IsDatetime(): // leave as is on orig column + case colType.IsDecimal() && newType.IsNumber(): // leave decimal on orig column + case colType.IsString() && newType.IsString(): // leave string as is + default: + // set directly since we're still profiling + g.Debug("column type change from transforms for %s (from %s to %s)", col.Name, col.Type, newType) + ds.Columns[i].Type = newType + if ds.df != nil { + if col := ds.df.Columns.GetColumn(ds.Columns[i].Name); col != nil { + col.Type = newType + } + } + } } } } @@ -1106,7 +1120,7 @@ func (ds *Datastream) ConsumeXmlReader(reader io.Reader) (err error) { reader2 = newReader } - decoder := xml.NewDecoder(reader2) + decoder := NewXMLDecoder(reader2) js := NewJSONStream(ds, decoder, ds.Sp.Config.Flatten, ds.Sp.Config.Jmespath) ds.it = ds.NewIterator(ds.Columns, js.NextFunc) diff --git a/core/dbio/iop/datatype.go b/core/dbio/iop/datatype.go index a4f597d9b..625fb85d0 100755 --- a/core/dbio/iop/datatype.go +++ b/core/dbio/iop/datatype.go @@ -797,7 +797,7 @@ func InferFromStats(columns []Column, safe bool, noDebug bool) []Column { col.Type = BoolType col.goType = reflect.TypeOf(true) colStats.Min = 0 - } else if colStats.IntCnt > 0 && colStats.IntCnt+colStats.NullCnt == colStats.TotalCnt { + } else if colStats.IntCnt > 0 && colStats.IntCnt+colStats.NullCnt == colStats.TotalCnt && col.Type != DecimalType { // Check if the values are too large for a regular int if colStats.Min < -2147483648 || colStats.Max > 2147483647 { col.Type = BigIntType @@ -810,6 +810,11 @@ func InferFromStats(columns []Column, safe bool, noDebug bool) []Column { // cast as bigint for safety col.Type = BigIntType } + } else if (colStats.DecCnt > 0 && colStats.DecCnt+colStats.IntCnt+colStats.NullCnt == colStats.TotalCnt) || col.Type == DecimalType { // keep as decimal if already set from mapping + col.Type = DecimalType + col.goType = reflect.TypeOf(float64(0.0)) + col.DbPrecision = lo.Ternary(col.DbPrecision == 0, colStats.MaxLen, col.DbPrecision) + col.DbScale = lo.Ternary(col.DbScale == 0, colStats.MaxDecLen, col.DbScale) } else if colStats.DateCnt > 0 && colStats.DateCnt+colStats.NullCnt == colStats.TotalCnt { col.Type = DateType col.goType = reflect.TypeOf(time.Now()) @@ -822,11 +827,6 @@ func InferFromStats(columns []Column, safe bool, noDebug bool) []Column { } col.goType = reflect.TypeOf(time.Now()) colStats.Min = 0 - } else if colStats.DecCnt > 0 && colStats.DecCnt+colStats.IntCnt+colStats.NullCnt == colStats.TotalCnt { - col.Type = DecimalType - col.goType = reflect.TypeOf(float64(0.0)) - col.DbPrecision = lo.Ternary(col.DbPrecision == 0, colStats.MaxLen, col.DbPrecision) - col.DbScale = lo.Ternary(col.DbScale == 0, colStats.MaxDecLen, col.DbScale) } else { // Mixed types or unrecognized - default to string/text if colStats.MaxLen >= 4000 { @@ -1546,6 +1546,7 @@ type DecimalColumnTyping struct { func (dct *DecimalColumnTyping) Apply(col *Column) (precision, scale int) { + minPrecision := col.Stats.MaxLen precision = col.DbPrecision scale = col.DbScale @@ -1555,7 +1556,6 @@ func (dct *DecimalColumnTyping) Apply(col *Column) (precision, scale int) { scale = lo.Ternary(scale < env.DdlMinDecScale, env.DdlMinDecScale, scale) scale = lo.Ternary(scale < minScale, minScale, scale) } - minPrecision := col.Stats.MaxLen + scale precision = lo.Ternary(precision < (scale*2), scale*2, precision) precision = lo.Ternary(precision < env.DdlMinDecLength, env.DdlMinDecLength, precision) precision = lo.Ternary(precision < minPrecision, minPrecision, precision) @@ -1572,6 +1572,10 @@ func (dct *DecimalColumnTyping) Apply(col *Column) (precision, scale int) { precision = *dct.MinPrecision } + if precision < minPrecision { + precision = minPrecision + } + if dct.MaxPrecision > 0 && precision > dct.MaxPrecision { precision = dct.MaxPrecision } diff --git a/core/dbio/iop/duckdb_test.go b/core/dbio/iop/duckdb_test.go index 8146ba99a..88e353db6 100644 --- a/core/dbio/iop/duckdb_test.go +++ b/core/dbio/iop/duckdb_test.go @@ -274,7 +274,7 @@ func TestDuckDbDataflowToHttpStream(t *testing.T) { // Create DuckDB instance duck := NewDuckDb(ctx) - duck.AddExtension("arrow") + duck.AddExtension("arrow from community") defer duck.Close() // Test DataflowToHttpStream with Arrow format diff --git a/core/dbio/iop/json.go b/core/dbio/iop/json.go index bba59f0b6..69d5919a4 100644 --- a/core/dbio/iop/json.go +++ b/core/dbio/iop/json.go @@ -93,6 +93,8 @@ func (js *jsonStream) NextFunc(it *Iterator) bool { } switch payloadV := payload.(type) { + case nil: + recordsInterf = nil case map[string]any: // is one record recordsInterf = js.extractNestedArray(payloadV) @@ -181,6 +183,10 @@ func (js *jsonStream) addColumn(cols ...Column) { } func (js *jsonStream) parseRecords(records []map[string]any) { + if records == nil { + js.buffer <- nil + return + } for _, rec := range records { if js.flatten < 0 { diff --git a/core/dbio/iop/stream_processor.go b/core/dbio/iop/stream_processor.go index f314078e2..ceba08624 100644 --- a/core/dbio/iop/stream_processor.go +++ b/core/dbio/iop/stream_processor.go @@ -32,13 +32,12 @@ import ( type StreamProcessor struct { N uint64 dateLayoutCache string - stringTypeCache map[int]string + stringTypeCache map[int]parseStringType colStats map[int]*ColumnStats rowChecksum []uint64 unrecognizedDate string warn bool skipCurrent bool // whether to skip current row (for constraints) - parseFuncs map[string]func(s string) (any, error) decReplRegex *regexp.Regexp ds *Datastream dateLayouts []string @@ -46,35 +45,34 @@ type StreamProcessor struct { rowBlankValCnt int transformers Transformers digitString map[int]string - transformEG g.ErrorGroup } type StreamConfig struct { - EmptyAsNull bool `json:"empty_as_null"` - Header bool `json:"header"` - Compression CompressorType `json:"compression"` // AUTO | ZIP | GZIP | SNAPPY | NONE - NullIf string `json:"null_if"` - NullAs string `json:"null_as"` - DatetimeFormat string `json:"datetime_format"` - SkipBlankLines bool `json:"skip_blank_lines"` - Format dbio.FileType `json:"format"` - Delimiter string `json:"delimiter"` - Escape string `json:"escape"` - Quote string `json:"quote"` - FileMaxRows int64 `json:"file_max_rows"` - FileMaxBytes int64 `json:"file_max_bytes"` - BatchLimit int64 `json:"batch_limit"` - MaxDecimals int `json:"max_decimals"` - Flatten int `json:"flatten"` - FieldsPerRec int `json:"fields_per_rec"` - Jmespath string `json:"jmespath"` - Sheet string `json:"sheet"` - ColumnCasing ColumnCasing `json:"column_casing"` - TargetType dbio.Type `json:"target_type"` - BoolAsInt bool `json:"-"` - Columns Columns `json:"columns"` // list of column types. Can be partial list! likely is! - transforms Transform - maxDecimalsFormat string `json:"-"` + EmptyAsNull bool `json:"empty_as_null"` + Header bool `json:"header"` + Compression CompressorType `json:"compression"` // AUTO | ZIP | GZIP | SNAPPY | NONE + NullIf string `json:"null_if"` + NullAs string `json:"null_as"` + DatetimeFormat string `json:"datetime_format"` + SkipBlankLines bool `json:"skip_blank_lines"` + Format dbio.FileType `json:"format"` + Delimiter string `json:"delimiter"` + Escape string `json:"escape"` + Quote string `json:"quote"` + FileMaxRows int64 `json:"file_max_rows"` + FileMaxBytes int64 `json:"file_max_bytes"` + BatchLimit int64 `json:"batch_limit"` + MaxDecimals int `json:"max_decimals"` + Flatten int `json:"flatten"` + FieldsPerRec int `json:"fields_per_rec"` + Jmespath string `json:"jmespath"` + Sheet string `json:"sheet"` + ColumnCasing ColumnCasing `json:"column_casing"` + TargetType dbio.Type `json:"target_type"` + DeleteFile bool `json:"delete"` // whether to delete before writing + BoolAsInt bool `json:"-"` + Columns Columns `json:"columns"` // list of column types. Can be partial list! likely is! + transforms Transform Map map[string]string `json:"-"` } @@ -85,13 +83,6 @@ func (sc *StreamConfig) ToMap() map[string]string { return g.ToMapString(m) } -func (sc *StreamConfig) SetMaxDecimals(val int) { - sc.MaxDecimals = val - if sc.MaxDecimals > -1 { - sc.maxDecimalsFormat = "%." + cast.ToString(sc.MaxDecimals) + "f" - } -} - type Transformers struct { Accent transform.Transformer @@ -202,7 +193,7 @@ func NewTransformers() Transformers { // NewStreamProcessor returns a new StreamProcessor func NewStreamProcessor() *StreamProcessor { sp := StreamProcessor{ - stringTypeCache: map[int]string{}, + stringTypeCache: map[int]parseStringType{}, colStats: map[int]*ColumnStats{}, decReplRegex: regexp.MustCompile(`^(\d*[\d.]*?)\.?0*$`), transformers: NewTransformers(), @@ -211,33 +202,9 @@ func NewStreamProcessor() *StreamProcessor { sp.ResetConfig() if val := os.Getenv("MAX_DECIMALS"); val != "" && val != "-1" { - sp.Config.SetMaxDecimals(cast.ToInt(val)) + sp.Config.MaxDecimals = cast.ToInt(val) } - // if val is '0400', '0401'. Such as codes. - hasZeroPrefix := func(s string) bool { return len(s) >= 2 && s[0] == '0' && s[1] != '.' } - - sp.parseFuncs = map[string]func(s string) (any, error){ - "int": func(s string) (any, error) { - if hasZeroPrefix(s) { - return s, errors.New("number has zero prefix, treat as string") - } - // return fastfloat.ParseInt64(s) - return strconv.ParseInt(s, 10, 64) - }, - "float": func(s string) (any, error) { - if hasZeroPrefix(s) { - return s, errors.New("number has zero prefix, treat as string") - } - return strconv.ParseFloat(strings.Replace(s, ",", ".", 1), 64) - }, - "time": func(s string) (any, error) { - return sp.ParseTime(s) - }, - "bool": func(s string) (any, error) { - return cast.ToBoolE(s) - }, - } sp.dateLayouts = []string{ "2006-01-02", "2006-01-02 15:04:05", @@ -392,7 +359,7 @@ func (sp *StreamProcessor) SetConfig(configMap map[string]string) { if err != nil { sp.Config.MaxDecimals = -1 } else { - sp.Config.SetMaxDecimals(val) + sp.Config.MaxDecimals = val } } @@ -424,6 +391,10 @@ func (sp *StreamProcessor) SetConfig(configMap map[string]string) { sp.Config.SkipBlankLines = cast.ToBool(val) } + if val, ok := configMap["delete_file"]; ok { + sp.Config.DeleteFile = cast.ToBool(val) + } + if val, ok := configMap["column_casing"]; ok { sp.Config.ColumnCasing = ColumnCasing(val) } @@ -588,7 +559,7 @@ func (sp *StreamProcessor) CheckType(v any) (typ ColumnType) { // Float types case float32, float64: - return FloatType + return DecimalType // Decimal types case decimal.Decimal: @@ -600,7 +571,7 @@ func (sp *StreamProcessor) CheckType(v any) (typ ColumnType) { // Time types case time.Time, *time.Time: - return TimestampType + return TimestampzType // String/Text types case string: @@ -634,7 +605,7 @@ func (sp *StreamProcessor) CheckType(v any) (typ ColumnType) { case sql.NullBool: return BoolType case sql.NullTime: - return TimestampType + return TimestampzType // Default case default: @@ -958,10 +929,12 @@ func (sp *StreamProcessor) CastVal(i int, val any, col *Column) any { cs.DecCnt++ } + // use string to keep accuracy, replace comma as decimal point + sVal := strings.Replace(sp.CastToString(val), ",", ".", 1) if sp.Config.MaxDecimals > -1 && !isInt { - nVal = g.F(sp.Config.maxDecimalsFormat, fVal) + nVal = sp.TruncateDecimalString(sVal, sp.Config.MaxDecimals) } else { - nVal = strings.Replace(cast.ToString(val), ",", ".", 1) // use string to keep accuracy, replace comma as decimal point + nVal = sVal } case col.Type.IsBool(): @@ -1048,6 +1021,56 @@ func (sp *StreamProcessor) bytesToHexEscape(b []byte) string { return string(result) } +func (sp *StreamProcessor) CountDigits(number string) (precision, scale int) { + inDecimal := false + for _, c := range number { + if c == '.' { + inDecimal = true + continue + } else if inDecimal { + scale++ // only decimal digits + } + precision++ // total number of digits + } + return +} + +// TruncateDecimalString return up to specified scale, without converting +func (sp *StreamProcessor) TruncateDecimalString(number string, decCount int) (newNumber string) { + + var precision, scale int + var inDecimal bool + newNumber = number + + for i, c := range newNumber { + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if inDecimal { + scale++ // count decimal digits + if scale > decCount { + return newNumber[0:i] + } + } + case '-': + case '.': + if inDecimal { + // Multiple decimal points - treat subsequent ones as the end of valid decimal + return newNumber[0:i] + } else { + inDecimal = true + // If decCount is 0 or negative, truncate right at the decimal point + if decCount <= 0 { + return newNumber[0:i] + } + } + default: + return number // if not a digit, return as is (is a string?) + } + precision++ // total number of digits + } + return newNumber +} + func (sp *StreamProcessor) CastToString(val any) (valString string) { valString, _ = sp.CastToStringE(val) return @@ -1056,8 +1079,12 @@ func (sp *StreamProcessor) CastToString(val any) (valString string) { func (sp *StreamProcessor) CastToStringE(val any) (valString string, err error) { switch v := val.(type) { + case string: + valString = v case []uint8: valString = string(v) + case *string: + valString = *v case chJSON: // Clickhouse JSON / Variant var sBytes []byte sBytes, err = v.MarshalJSON() @@ -1071,10 +1098,6 @@ func (sp *StreamProcessor) CastToStringE(val any) (valString string, err error) decCount = sp.Config.MaxDecimals } valString = v.FloatString(decCount) - case *string: - valString = *v - case string: - valString = v case map[string]string, map[string]any, map[any]any, []any, []string: valString = g.Marshal(v) default: @@ -1115,12 +1138,8 @@ func (sp *StreamProcessor) CastToStringCSV(i int, val any, valType ...ColumnType if RemoveTrailingDecZeros { // attempt to remove trailing zeros, but is 10 times slower return sp.decReplRegex.ReplaceAllString(cast.ToString(val), "$1") - } else if sp.Config.maxDecimalsFormat != "" { - if sp.Config.MaxDecimals <= 10 { - if fVal, err := cast.ToFloat64E(val); err == nil { - return g.F(sp.Config.maxDecimalsFormat, fVal) - } - } + } else if sp.Config.MaxDecimals > -1 { + return sp.TruncateDecimalString(sp.CastToString(val), sp.Config.MaxDecimals) } return sp.CastToString(val) // return fmt.Sprintf("%v", val) @@ -1316,6 +1335,16 @@ func (sp *StreamProcessor) ParseTime(i any) (t time.Time, err error) { return } +type parseStringType string + +const ( + parseStringTypeInt parseStringType = "int" + parseStringTypeDecimal parseStringType = "decimal" + parseStringTypeFloat parseStringType = "float" + parseStringTypeTime parseStringType = "time" + parseStringTypeBool parseStringType = "bool" +) + // ParseString return an interface // string: "varchar" // integer: "integer" @@ -1324,7 +1353,7 @@ func (sp *StreamProcessor) ParseTime(i any) (t time.Time, err error) { // datetime: "timestamp" // timestamp: "timestamp" // text: "text" -func (sp *StreamProcessor) ParseString(s string, jj ...int) any { +func (sp *StreamProcessor) ParseString(s string, jj ...int) (val any) { if s == "" { return nil } @@ -1334,52 +1363,149 @@ func (sp *StreamProcessor) ParseString(s string, jj ...int) any { j = jj[0] } - stringTypeCache := sp.stringTypeCache[j] - - if stringTypeCache != "" { - i, err := sp.parseFuncs[stringTypeCache](s) - if err == nil { - return i - } + var err error + switch sp.stringTypeCache[j] { + case parseStringTypeInt: + val, err = sp.parseStringInt(s) + case parseStringTypeDecimal: + val, err = sp.parseStringDecimal(s) + case parseStringTypeFloat: + val, err = sp.parseStringFloat(s) + case parseStringTypeTime: + val, err = sp.parseStringTime(s) + case parseStringTypeBool: + val, err = sp.parseStringBool(s) + } + if val != nil && err == nil { + return val } // int - i, err := sp.parseFuncs["int"](s) + i, err := sp.parseStringInt(s) if err == nil { // if s = 0100, casting to int64 will return 64 // need to mitigate by when s starts with 0 if len(s) > 1 && s[0] == '0' { return s } - sp.stringTypeCache[j] = "int" + sp.stringTypeCache[j] = parseStringTypeInt return i } - // float - f, err := sp.parseFuncs["float"](s) + // // float + // f, err := sp.parseStringFloat(s) + // if err == nil { + // sp.stringTypeCache[j] = parseStringTypeFloat + // return f + // } + + // decimal + d, err := sp.parseStringDecimal(s) if err == nil { - sp.stringTypeCache[j] = "float" - return f + sp.stringTypeCache[j] = parseStringTypeDecimal + return d } // date/time - t, err := sp.parseFuncs["time"](s) + t, err := sp.parseStringTime(s) if err == nil { - sp.stringTypeCache[j] = "time" + sp.stringTypeCache[j] = parseStringTypeTime return t } // boolean // FIXME: causes issues in SQLite and Oracle, needed for correct boolean parsing - b, err := sp.parseFuncs["bool"](s) + b, err := sp.parseStringBool(s) if err == nil { - sp.stringTypeCache[j] = "bool" + sp.stringTypeCache[j] = parseStringTypeBool return b } return s } +// if val is '0400', '0401'. Such as codes. +func (sp *StreamProcessor) hasZeroPrefix(s string) bool { + return len(s) >= 2 && s[0] == '0' && s[1] != '.' +} + +func (sp *StreamProcessor) parseStringInt(s string) (newVal any, err error) { + if sp.hasZeroPrefix(s) { + return s, errors.New("number has zero prefix, treat as string") + } + // return fastfloat.ParseInt64(s) + return strconv.ParseInt(s, 10, 64) +} + +func (sp *StreamProcessor) parseStringFloat(s string) (newVal any, err error) { + if sp.hasZeroPrefix(s) { + return s, errors.New("number has zero prefix, treat as string") + } + return strconv.ParseFloat(strings.Replace(s, ",", ".", 1), 64) +} + +func (sp *StreamProcessor) parseStringBigFloat(s string) (newVal any, err error) { + if sp.hasZeroPrefix(s) { + return s, errors.New("number has zero prefix, treat as string") + } + var ok bool + newVal, ok = new(big.Float).SetString(strings.Replace(s, ",", ".", 1)) + if !ok { + return nil, errors.New("could not cast to big.Rat, treat as string") + } + return newVal, nil +} + +func (sp *StreamProcessor) parseStringIsDecimal(s string) bool { + if sp.hasZeroPrefix(s) { + return false + } + + var inDecimal bool + for _, c := range s { + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + case '.': + if inDecimal { + return false // can only have 1 dot + } + inDecimal = true + default: + return false + } + } + return true +} + +func (sp *StreamProcessor) parseStringDecimal(s string) (newVal any, err error) { + if sp.hasZeroPrefix(s) { + return s, errors.New("number has zero prefix, treat as string") + } + + return decimal.NewFromString(strings.Replace(s, ",", ".", 1)) +} + +func (sp *StreamProcessor) parseStringRational(s string) (newVal any, err error) { + if sp.hasZeroPrefix(s) { + return s, errors.New("number has zero prefix, treat as string") + } + + var ok bool + newVal, ok = new(big.Rat).SetString(strings.Replace(s, ",", ".", 1)) + if !ok { + return nil, errors.New("could not cast to big.Rat, treat as string") + } + return newVal, nil +} + +func (sp *StreamProcessor) parseStringTime(s string) (newVal any, err error) { + return sp.ParseTime(s) +} + +func (sp *StreamProcessor) parseStringBool(s string) (newVal any, err error) { + return cast.ToBoolE(s) +} + // ProcessVal processes a value func (sp *StreamProcessor) ProcessVal(val any) any { var nVal any diff --git a/core/dbio/iop/stream_processor_test.go b/core/dbio/iop/stream_processor_test.go new file mode 100644 index 000000000..40a2c30b1 --- /dev/null +++ b/core/dbio/iop/stream_processor_test.go @@ -0,0 +1,401 @@ +package iop + +import ( + "testing" +) + +// go test -benchmem -run='^$ github.com/slingdata-io/sling-cli/core/dbio/iop' -bench '^BenchmarkProcessDecimal' +func BenchmarkProcessDecimal1(b *testing.B) { + sp := NewStreamProcessor() + for n := 0; n < b.N; n++ { + sp.CountDigits("1273192311231.9382471941") + } +} +func BenchmarkProcessDecimal2(b *testing.B) { + sp := NewStreamProcessor() + for n := 0; n < b.N; n++ { + sp.parseStringFloat("1273192311231.9382471941") + } +} +func BenchmarkProcessDecimal3(b *testing.B) { + sp := NewStreamProcessor() + for n := 0; n < b.N; n++ { + sp.parseStringDecimal("1273192311231.9382471941") + } +} +func BenchmarkProcessDecimal4(b *testing.B) { + sp := NewStreamProcessor() + for n := 0; n < b.N; n++ { + sp.parseStringRational("1273192311231.9382471941") + } +} +func BenchmarkProcessDecimal5(b *testing.B) { + sp := NewStreamProcessor() + for n := 0; n < b.N; n++ { + sp.parseStringBigFloat("1273192311231.9382471941") + } +} +func BenchmarkProcessDecimal6(b *testing.B) { + sp := NewStreamProcessor() + for n := 0; n < b.N; n++ { + sp.parseStringIsDecimal("1273192311231.9382471941") + } +} +func BenchmarkProcessDecimal7(b *testing.B) { + sp := NewStreamProcessor() + for n := 0; n < b.N; n++ { + sp.TruncateDecimalString("1273192311231.9382471941", 9) + } +} + +func TestStreamProcessor_countDigits(t *testing.T) { + sp := NewStreamProcessor() + + tests := []struct { + name string + number string + precision int + scale int + }{ + // Integer cases + { + name: "single digit", + number: "5", + precision: 1, + scale: 0, + }, + { + name: "multiple digits", + number: "12345", + precision: 5, + scale: 0, + }, + { + name: "zero", + number: "0", + precision: 1, + scale: 0, + }, + { + name: "large integer", + number: "9876543210", + precision: 10, + scale: 0, + }, + + // Decimal cases + { + name: "simple decimal", + number: "123.45", + precision: 5, // total digits: 1+2+3+4+5 = 5 + scale: 2, // digits after decimal: 4+5 = 2 + }, + { + name: "decimal with zero integer part", + number: "0.123", + precision: 4, // total digits: 0+1+2+3 = 4 + scale: 3, // digits after decimal: 1+2+3 = 3 + }, + { + name: "decimal with trailing zeros", + number: "123.450", + precision: 6, // total digits: 1+2+3+4+5+0 = 6 + scale: 3, // digits after decimal: 4+5+0 = 3 + }, + { + name: "decimal with leading zeros", + number: "00123.45", + precision: 7, // total digits: 0+0+1+2+3+4+5 = 7 + scale: 2, // digits after decimal: 4+5 = 2 + }, + { + name: "very small decimal", + number: "0.000001", + precision: 7, // total digits: 0+0+0+0+0+0+1 = 7 + scale: 6, // digits after decimal: 0+0+0+0+0+1 = 6 + }, + { + name: "decimal with many places", + number: "123.456789012345", + precision: 15, // total digits: 3 before + 12 after = 15 + scale: 12, // digits after decimal: 12 + }, + + // Edge cases + { + name: "empty string", + number: "", + precision: 0, + scale: 0, + }, + { + name: "just decimal point", + number: ".", + precision: 0, + scale: 0, + }, + { + name: "decimal point at start", + number: ".123", + precision: 3, // total digits: 1+2+3 = 3 + scale: 3, // digits after decimal: 1+2+3 = 3 + }, + { + name: "decimal point at end", + number: "123.", + precision: 3, // total digits: 1+2+3 = 3 + scale: 0, // no digits after decimal + }, + { + name: "multiple decimal points", // This is an edge case - function will count digits after first dot + number: "12.34.56", + precision: 6, // total digits: 1+2+3+4+5+6 = 6 (excluding decimal points) + scale: 4, // digits after first decimal: 3+4+5+6 = 4 + }, + + // Scientific notation (treated as string) + { + name: "scientific notation", + number: "1.23e+10", + precision: 7, // total digits: 1+2+3+e(letter not counted)+1+0 = 7 + scale: 6, // digits after decimal: 2+3+e+1+0 = 6 (e treated as character) + }, + + // Financial numbers + { + name: "currency amount", + number: "1234567.89", + precision: 9, // total digits: 1+2+3+4+5+6+7+8+9 = 9 + scale: 2, // digits after decimal: 8+9 = 2 + }, + { + name: "percentage", + number: "99.99", + precision: 4, // total digits: 9+9+9+9 = 4 + scale: 2, // digits after decimal: 9+9 = 2 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + precision, scale := sp.CountDigits(tt.number) + + if precision != tt.precision { + t.Errorf("countDigits(%q) precision = %d, want %d", tt.number, precision, tt.precision) + } + + if scale != tt.scale { + t.Errorf("countDigits(%q) scale = %d, want %d", tt.number, scale, tt.scale) + } + }) + } +} + +func TestStreamProcessor_TruncateDecimalString(t *testing.T) { + sp := NewStreamProcessor() + + tests := []struct { + name string + number string + decCount int + expected string + }{ + // Basic decimal rounding + { + name: "round to 2 decimals", + number: "123.456789", + decCount: 2, + expected: "123.45", + }, + { + name: "round to 1 decimal", + number: "123.456789", + decCount: 1, + expected: "123.4", + }, + { + name: "round to 0 decimals", + number: "123.456789", + decCount: 0, + expected: "123", + }, + + // Exact decimal count matches + { + name: "exact decimal count match", + number: "123.45", + decCount: 2, + expected: "123.45", + }, + { + name: "fewer decimals than requested", + number: "123.4", + decCount: 3, + expected: "123.4", + }, + + // Integer cases + { + name: "integer with decimal count", + number: "123", + decCount: 2, + expected: "123", + }, + + // Edge cases with zeros + { + name: "trailing zeros", + number: "123.450000", + decCount: 2, + expected: "123.45", + }, + { + name: "leading zeros in decimal", + number: "123.001234", + decCount: 3, + expected: "123.001", + }, + + // Zero values + { + name: "zero value", + number: "0.123456", + decCount: 2, + expected: "0.12", + }, + { + name: "zero integer", + number: "0", + decCount: 2, + expected: "0", + }, + + // Large numbers + { + name: "large number", + number: "1234567890.123456789", + decCount: 4, + expected: "1234567890.1234", + }, + + // Small decimals + { + name: "very small decimal", + number: "0.000001234", + decCount: 6, + expected: "0.000001", + }, + + // Decimal point at end + { + name: "decimal point at end", + number: "123.", + decCount: 2, + expected: "123.", + }, + + // Decimal point at start + { + name: "decimal point at start", + number: ".123456", + decCount: 2, + expected: ".12", + }, + + // Empty string + { + name: "empty string", + number: "", + decCount: 2, + expected: "", + }, + + // Just decimal point + { + name: "just decimal point", + number: ".", + decCount: 2, + expected: ".", + }, + + // Non-numeric characters (should return original) + { + name: "non-numeric string", + number: "abc", + decCount: 2, + expected: "abc", + }, + { + name: "mixed characters", + number: "123.45abc", + decCount: 2, + expected: "123.45abc", // returns original when encountering non-digit + }, + + // Multiple decimal points (truncate at second decimal point) + { + name: "multiple decimal points", + number: "12.34.56", + decCount: 1, + expected: "12.3", // truncate at second decimal point + }, + + // Negative numbers (note: the method doesn't handle signs, treats as non-digit) + { + name: "negative number", + number: "-123.456", + decCount: 2, + expected: "-123.45", + }, + + // Scientific notation (treated as string) + { + name: "scientific notation", + number: "1.23e+10", + decCount: 2, + expected: "1.23e+10", // returns original due to 'e' being non-digit + }, + + // Very high decimal count + { + name: "very high decimal count", + number: "123.456", + decCount: 100, + expected: "123.456", + }, + + // Negative decimal count behavior + { + name: "negative decimal count", + number: "123.456", + decCount: -1, + expected: "123", // truncates at decimal point since -1 <= 0 + }, + + // Edge case: zero decimal count + { + name: "zero decimal count with integer", + number: "123", + decCount: 0, + expected: "123", + }, + + // Very long decimal precision + { + name: "very long decimal", + number: "1.123456789012345678901234567890", + decCount: 10, + expected: "1.1234567890", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := sp.TruncateDecimalString(tt.number, tt.decCount) + + if result != tt.expected { + t.Errorf("RoundDecimalString(%q, %d) = %q, want %q", tt.number, tt.decCount, result, tt.expected) + } + }) + } +} diff --git a/core/dbio/iop/transforms.go b/core/dbio/iop/transforms.go index dd4c103ab..2d2bf4345 100644 --- a/core/dbio/iop/transforms.go +++ b/core/dbio/iop/transforms.go @@ -14,6 +14,7 @@ import ( "github.com/flarco/g" "github.com/google/uuid" + "github.com/jinzhu/copier" "github.com/jmespath/go-jmespath" "github.com/maja42/goval" "github.com/spf13/cast" @@ -611,10 +612,10 @@ func (t transformsNS) BinaryToHex(val string) string { func (t transformsNS) Replace0x00(sp *StreamProcessor, val string) (string, error) { // Remove null bytes and common null representations - val = strings.ReplaceAll(val, "\x00", "") // actual null byte - val = strings.ReplaceAll(val, "\\u0000", "") // escaped unicode null - val = strings.ReplaceAll(val, "\\x00", "") // escaped hex null - val = strings.ReplaceAll(val, "\u0000", "") // unicode null character + val = strings.ReplaceAll(val, "\x00", "") // actual null byte + val = strings.ReplaceAll(val, "\\u0000", "") // escaped unicode null + val = strings.ReplaceAll(val, "\\x00", "") // escaped hex null + val = strings.ReplaceAll(val, "\u0000", "") // unicode null character return val, nil } @@ -667,7 +668,7 @@ func (t transformsNS) ReplaceNonPrintable(val string) string { case 160: // NO-BREAK SPACE newVal.WriteRune(' ') // replace with space continue - + // Zero-width characters - remove these case 0x200B: // ZERO WIDTH SPACE continue @@ -679,7 +680,7 @@ func (t transformsNS) ReplaceNonPrintable(val string) string { continue case 0xFEFF: // ZERO WIDTH NO-BREAK SPACE (BOM) continue - + // Directional formatting characters - remove these case 0x200E: // LEFT-TO-RIGHT MARK continue @@ -695,7 +696,7 @@ func (t transformsNS) ReplaceNonPrintable(val string) string { continue case 0x202E: // RIGHT-TO-LEFT OVERRIDE continue - + // Other problematic characters case 0x00AD: // SOFT HYPHEN continue // remove @@ -860,7 +861,12 @@ func (e *Evaluator) RenderAny(input any, extras ...map[string]any) (output any, output = input noCompute := false // especially in SQL queries - stateMap := e.State + stateMap := g.M() + err = copier.CopyWithOption(&stateMap, &e.State, copier.Option{DeepCopy: true}) + if err != nil { + return nil, g.Error(err, "could not deep copy for evaluation") + } + for _, extra := range extras { for k, v := range extra { stateMap[k] = v @@ -1088,3 +1094,46 @@ func (e *Evaluator) RenderPayload(val any, extras ...map[string]any) (newVal any return val, nil } } + +func (e *Evaluator) Check(expr string) (err error) { + inDouble := false + parenCount := 0 + + runes := []rune(expr) + for i, c := range runes { + if c == '\'' && !inDouble { + return g.Error("cannot use single quotes (') for strings in expression, use double quotes (\"): %s", expr) + } else if c == '"' { + // Check if this quote is escaped by counting preceding backslashes + backslashCount := 0 + for j := i - 1; j >= 0 && runes[j] == '\\'; j-- { + backslashCount++ + } + // If even number of backslashes (including 0), the quote is not escaped + if backslashCount%2 == 0 { + inDouble = !inDouble + } + } else if !inDouble { + // Only track parentheses when not inside double quotes + switch c { + case '(': + parenCount++ + case ')': + parenCount-- + if parenCount < 0 { + return g.Error("unmatched closing parenthesis ')' in expression: %s", expr) + } + } + } + } + + if inDouble { + return g.Error("unclosed double quote in expression: %s", expr) + } + + if parenCount > 0 { + return g.Error("unclosed parenthesis '(' in expression: %s", expr) + } + + return nil +} diff --git a/core/dbio/iop/transforms_test.go b/core/dbio/iop/transforms_test.go index 254fed2bd..9c4b21853 100755 --- a/core/dbio/iop/transforms_test.go +++ b/core/dbio/iop/transforms_test.go @@ -996,7 +996,7 @@ func TestEvaluatorRenderPayload(t *testing.T) { } } -func TestEvaluatortExtractVars(t *testing.T) { +func TestEvaluatorExtractVars(t *testing.T) { type testCase struct { name string expr string @@ -1096,3 +1096,357 @@ func TestEvaluatortExtractVars(t *testing.T) { }) } } + +func TestEvaluatorCheckExpression(t *testing.T) { + tests := []struct { + name string + expression string + expectError bool + errorMsg string + }{ + { + name: "empty_expression", + expression: "", + expectError: false, + }, + { + name: "simple_expression_no_quotes", + expression: "state.counter > 10", + expectError: false, + }, + { + name: "valid_double_quotes", + expression: `state.name == "John"`, + expectError: false, + }, + { + name: "multiple_valid_double_quotes", + expression: `state.firstName == "John" && state.lastName == "Doe"`, + expectError: false, + }, + { + name: "nested_double_quotes_with_escape", + expression: `state.message == "He said \"Hello\""`, + expectError: false, + }, + { + name: "single_quote_error", + expression: `state.name == 'John'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "single_quote_in_middle", + expression: `state.name == "John" && state.title == 'Mr'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "single_quote_at_beginning", + expression: `'test' == state.value`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "apostrophe_outside_double_quotes", + expression: `state.name == "John's car"`, + expectError: false, + }, + { + name: "apostrophe_and_single_quote_mix", + expression: `state.name == "John's car" && state.other == 'test'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "escaped_double_quote", + expression: `state.quote == "She said \"hi\""`, + expectError: false, + }, + { + name: "multiple_escaped_double_quotes", + expression: `state.json == "{\"name\": \"John\", \"age\": 30}"`, + expectError: false, + }, + { + name: "single_quote_after_escaped_double_quote", + expression: `state.text == "He said \"hello\"" && state.bad == 'world'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "complex_valid_expression", + expression: `response.status == 200 && response.data.message == "Success" && len(response.items) > 0`, + expectError: false, + }, + { + name: "backslash_before_single_quote_still_error", + expression: `state.test == "valid" && state.invalid == \'bad\'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "double_backslash_before_double_quote", + expression: `state.path == "C:\\Program Files\\"`, + expectError: false, + }, + { + name: "single_quote_inside_double_quotes_is_valid", + expression: `state.message == "Don't do that"`, + expectError: false, + }, + { + name: "unclosed_double_quote_with_single_quote", + expression: `state.name == "John && state.other == 'test'`, + expectError: true, + errorMsg: "unclosed double quote", + }, + { + name: "only_single_quotes", + expression: `'hello world'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "only_double_quotes", + expression: `"hello world"`, + expectError: false, + }, + { + name: "mixed_quotes_complex", + expression: `state.a == "test" && state.b == 'invalid' && state.c == "valid"`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "unicode_characters_with_double_quotes", + expression: `state.emoji == "Hello 👋 world"`, + expectError: false, + }, + { + name: "unicode_characters_with_single_quotes", + expression: `state.emoji == 'Hello 👋 world'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "json_like_string", + expression: `state.config == "{\"timeout\": 30, \"retries\": 3}"`, + expectError: false, + }, + { + name: "regex_like_pattern_double_quotes", + expression: `state.pattern == "^[a-zA-Z0-9]+$"`, + expectError: false, + }, + { + name: "regex_like_pattern_single_quotes", + expression: `state.pattern == '^[a-zA-Z0-9]+$'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "empty_string_double_quotes", + expression: `state.value == ""`, + expectError: false, + }, + { + name: "empty_string_single_quotes", + expression: `state.value == ''`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "multiple_consecutive_escapes", + expression: `state.path == "C:\\\\server\\\\path\\\\"`, + expectError: false, + }, + { + name: "quote_at_end_of_string", + expression: `state.sql == "SELECT * FROM table WHERE name = \"John\""`, + expectError: false, + }, + { + name: "single_quote_at_very_end", + expression: `someexpression'`, + expectError: true, + errorMsg: "cannot use single quotes", + }, + { + name: "double_quote_at_very_end", + expression: `someexpression"`, + expectError: true, + errorMsg: "unclosed double quote", + }, + // Parentheses tests + { + name: "valid_parentheses_simple", + expression: `func(arg1, arg2)`, + expectError: false, + }, + { + name: "valid_parentheses_nested", + expression: `outer(inner(value), other)`, + expectError: false, + }, + { + name: "valid_parentheses_multiple", + expression: `func1() && func2(arg) || func3(a, b)`, + expectError: false, + }, + { + name: "unclosed_parenthesis_simple", + expression: `func(arg1, arg2`, + expectError: true, + errorMsg: "unclosed parenthesis", + }, + { + name: "unclosed_parenthesis_nested", + expression: `outer(inner(value), other`, + expectError: true, + errorMsg: "unclosed parenthesis", + }, + { + name: "extra_closing_parenthesis", + expression: `func(arg1, arg2))`, + expectError: true, + errorMsg: "unmatched closing parenthesis", + }, + { + name: "parentheses_inside_double_quotes_valid", + expression: `state.value == "text with (parentheses)"`, + expectError: false, + }, + { + name: "unclosed_quote_simple", + expression: `state.name == "John`, + expectError: true, + errorMsg: "unclosed double quote", + }, + { + name: "unclosed_quote_at_beginning", + expression: `"unclosed string and other code`, + expectError: true, + errorMsg: "unclosed double quote", + }, + { + name: "mixed_unclosed_quote_and_paren", + expression: `func("unclosed string and missing paren`, + expectError: true, + errorMsg: "unclosed double quote", // Quote error should be caught first + }, + { + name: "mixed_unclosed_paren_after_quote", + expression: `func("closed string" and missing paren`, + expectError: true, + errorMsg: "unclosed parenthesis", + }, + { + name: "complex_valid_expression_with_quotes_and_parens", + expression: `len(state.items) > 0 && state.name == "John" && func(state.age)`, + expectError: false, + }, + { + name: "parentheses_with_escaped_quotes", + expression: `func(state.message == "He said \"Hello\"")`, + expectError: false, + }, + { + name: "multiple_unclosed_parentheses", + expression: `outer(inner(deep(value)`, + expectError: true, + errorMsg: "unclosed parenthesis", + }, + { + name: "parentheses_only_closing", + expression: `)`, + expectError: true, + errorMsg: "unmatched closing parenthesis", + }, + { + name: "parentheses_only_opening", + expression: `(`, + expectError: true, + errorMsg: "unclosed parenthesis", + }, + { + name: "empty_parentheses", + expression: `func()`, + expectError: false, + }, + // Additional edge cases for escape handling + { + name: "odd_escapes_before_quote", + expression: `state.text == "He said \\\"Hello\\\""`, + expectError: false, + }, + { + name: "escaped_backslash_before_quote", + expression: `state.path == "C:\\\\"`, + expectError: false, + }, + { + name: "parentheses_and_quotes_complex", + expression: `func(state.name == "value") && other("test")`, + expectError: false, + }, + { + name: "nested_parens_with_unclosed_quote", + expression: `func(inner(state.name == "unclosed))`, + expectError: true, + errorMsg: "unclosed double quote", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eval := NewEvaluator(nil) + err := eval.Check(tt.expression) + + if tt.expectError { + assert.Error(t, err) + if tt.errorMsg != "" && err != nil { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +// go test -benchmem -run='^$ github.com/slingdata-io/sling-cli/core/dbio/iop' -bench '^BenchmarkPtEvaluatorRenderPayload' +func BenchmarkPtEvaluatorRenderPayload(b *testing.B) { + + eState := g.M("value1", 1, "value2", g.M("nested1", "a")) + eval := NewEvaluator(g.ArrStr("state", "store", "env", "run", "target", "source", "stream", "object", "timestamp", "execution", "loop"), eState) + + // Render the payload + var err error + expr := "value1={state.value1} | value2={state.value2.nested1}" + for n := 0; n < b.N; n++ { + newState := g.M("value1", 11) + _, err = eval.RenderPayload(expr, newState) + if err != nil { + return + } + } +} + +// go test -benchmem -run='^$ github.com/slingdata-io/sling-cli/core/dbio/iop' -bench '^BenchmarkPtEvaluatorRenderPayload' +func BenchmarkPtEvaluatorEval(b *testing.B) { + + eState := g.M("value1", 1, "value2", g.M("nested1", "a")) + eval := NewEvaluator(g.ArrStr("state", "store", "env", "run", "target", "source", "stream", "object", "timestamp", "execution", "loop"), eState) + + // Render the payload + var err error + expr := "value1={state.value1} | value2={state.value2.nested1}" + for n := 0; n < b.N; n++ { + newState := g.M("value1", 1, "value2", g.M("nested1", "a")) + _, err = eval.Eval.Evaluate(expr, newState, GlobalFunctionMap) + if err != nil { + return + } + } +} diff --git a/core/dbio/iop/xml.go b/core/dbio/iop/xml.go new file mode 100644 index 000000000..fe6f05321 --- /dev/null +++ b/core/dbio/iop/xml.go @@ -0,0 +1,176 @@ +package iop + +import ( + "encoding/xml" + "io" + "strconv" + "strings" + + "github.com/flarco/g" +) + +type xmlDecoder struct { + reader io.Reader + decoder *xml.Decoder + exhausted bool + discovered bool + rootTag string + recordTag string +} + +func NewXMLDecoder(reader io.Reader) *xmlDecoder { + return &xmlDecoder{ + reader: reader, + decoder: xml.NewDecoder(reader), + } +} + +func (xd *xmlDecoder) Decode(obj any) error { + if xd.exhausted { + return io.EOF + } + + if !xd.discovered { + // Discover root and first record tag, parse first record. + var err error + xd.rootTag, xd.recordTag, err = xd.discoverAndParseFirst(obj) + if err != nil { + return err + } + xd.discovered = true + return nil + } + + // For subsequent records: advance to next matching StartElement. + for { + token, err := xd.decoder.Token() + if err != nil { + if err == io.EOF { + xd.exhausted = true + return io.EOF + } + return g.Error(err, "token read error") + } + + if se, ok := token.(xml.StartElement); ok && se.Name.Local == xd.recordTag { + // Parse this record. + recordMap, err := xd.parseRecord(&se) + if err != nil { + return err + } + // Set to obj (assuming *interface{} like original). + if i, ok := obj.(*interface{}); ok { + *i = recordMap + } else { + return g.Error(nil, "obj must be *interface{}") + } + return nil + } + + if ee, ok := token.(xml.EndElement); ok && ee.Name.Local == xd.rootTag { + // End of root: no more records. + xd.exhausted = true + return io.EOF + } + } +} + +// discoverAndParseFirst skips prologue, finds root, then first record, parses it, and sets tags. +func (xd *xmlDecoder) discoverAndParseFirst(obj any) (rootTag, recordTag string, err error) { + // Skip to root StartElement. + for { + token, err := xd.decoder.Token() + if err != nil { + return "", "", g.Error(err, "could not find root") + } + if se, ok := token.(xml.StartElement); ok { + rootTag = se.Name.Local + break + } + // Skip ProcInst, etc. + } + + // Now find first child StartElement (record). + var firstSE xml.StartElement + for { + token, err := xd.decoder.Token() + if err != nil { + return "", "", g.Error(err, "could not find first record") + } + if se, ok := token.(xml.StartElement); ok { + firstSE = se + recordTag = se.Name.Local + break + } + } + + // Parse the first record. + recordMap, err := xd.parseRecord(&firstSE) + if err != nil { + return "", "", err + } + + // Set to obj. + if i, ok := obj.(*interface{}); ok { + *i = recordMap + } else { + return "", "", g.Error(nil, "obj must be *interface{}") + } + + return rootTag, recordTag, nil +} + +// parseRecord builds map for the subtree starting at 'start', until matching EndElement. +func (xd *xmlDecoder) parseRecord(start *xml.StartElement) (map[string]interface{}, error) { + m := make(map[string]interface{}) + + // Handle attributes if any (prefix with "@"). + for _, attr := range start.Attr { + m["@"+attr.Name.Local] = attr.Value + } + + for { + token, err := xd.decoder.Token() + if err != nil { + return nil, g.Error(err, "parse record error") + } + + if ee, ok := token.(xml.EndElement); ok && ee.Name.Local == start.Name.Local { + return m, nil + } + + if se, ok := token.(xml.StartElement); ok { + // For this child: collect CharData until its End. + var textBuilder strings.Builder + childKey := se.Name.Local + for { + tt, err := xd.decoder.Token() + if err != nil { + return nil, g.Error(err, "parse child error") + } + if cd, ok := tt.(xml.CharData); ok { + textBuilder.Write(cd) + } else if ee, ok := tt.(xml.EndElement); ok && ee.Name.Local == childKey { + break + } else { + // Unexpected (e.g., nested StartElement); for simple flat, error or extend. + return nil, g.Error(nil, "nested content not supported in simple mode") + } + } + text := strings.TrimSpace(textBuilder.String()) + if text == "" { + continue // Empty element. + } + + // Type conversion like mxj: try float64, then bool, else string. + if f, err := strconv.ParseFloat(text, 64); err == nil { + m[childKey] = f + } else if b, err := strconv.ParseBool(text); err == nil { + m[childKey] = b + } else { + m[childKey] = text + } + } + // Ignore other tokens (e.g., comments). + } +} diff --git a/core/dbio/templates/base.yaml b/core/dbio/templates/base.yaml index 177af4d5b..6991b5b15 100755 --- a/core/dbio/templates/base.yaml +++ b/core/dbio/templates/base.yaml @@ -46,6 +46,7 @@ core: incremental_select_limit_offset: select {fields} from {table} where ({incremental_where_cond}){where_and} order by {update_key} asc limit {limit} offset {offset} incremental_where: '{update_key} {gt} {value}' backfill_where: '{update_key} >= {start_value} and {update_key} <= {end_value}' + merge_update_insert: 'select error /* merge_update_insert not implemented */' analysis: # table level @@ -392,6 +393,9 @@ variable: timestampz_layout: '2006-01-02 15:04:05.000000 -0700' error_filter_table_exists: already bool_as: bool + + # default merge strategy + merge_strategy: update_insert error_filter: table_not_exist: exist diff --git a/core/dbio/templates/postgres.yaml b/core/dbio/templates/postgres.yaml index 987da6ab6..a74eb779e 100755 --- a/core/dbio/templates/postgres.yaml +++ b/core/dbio/templates/postgres.yaml @@ -25,6 +25,48 @@ core: rename_table: ALTER TABLE {table} RENAME TO {new_table} modify_column: alter column {column} type {type} use_database: SET search_path TO {database} + + # in order to use on conflict, the target table needs + # a unique index on the PK. We will not use it since + # it complicates matters + # merge_update_insert: | + # {final_table_index_sql}; + # insert into {tgt_table} + # ({insert_fields}) + # select {src_fields} from {src_table} src + # on conflict ({tgt_pk_fields}) + # do update + # SET {set_fields} + + merge_update_insert: | + create temporary table {temp_table} as + with src_table as ( + select {src_fields} from {src_table} + ) + , updates as ( + update {tgt_table} tgt + set {set_fields} + from src_table src + where {src_tgt_pk_equal} + returning tgt.* + ) + select * from updates; + + {temp_table_index_sql}; + + with src_table as ( + select {src_fields} from {src_table} + ) + insert into {tgt_table} + ({insert_fields}) + select {src_fields} from src_table src + where not exists ( + select 1 + from {temp_table} upd + where {src_upd_pk_equal} + ) + + metadata: diff --git a/core/sling/config.go b/core/sling/config.go index 883134f8b..653125856 100644 --- a/core/sling/config.go +++ b/core/sling/config.go @@ -1459,6 +1459,7 @@ type TargetOptions struct { PreSQL *string `json:"pre_sql,omitempty" yaml:"pre_sql,omitempty"` PostSQL *string `json:"post_sql,omitempty" yaml:"post_sql,omitempty"` IsolationLevel *database.IsolationLevel `json:"isolation_level,omitempty" yaml:"isolation_level,omitempty"` + MergeStrategy *database.MergeStrategy `json:"merge_strategy,omitempty" yaml:"merge_strategy,omitempty"` } var SourceFileOptionsDefault = SourceOptions{ @@ -1635,6 +1636,9 @@ func (o *TargetOptions) SetDefaults(targetOptions TargetOptions) { if o.IsolationLevel == nil { o.IsolationLevel = targetOptions.IsolationLevel } + if o.MergeStrategy == nil { + o.MergeStrategy = targetOptions.MergeStrategy + } if o.TableTmp == "" { o.TableTmp = targetOptions.TableTmp } diff --git a/core/sling/task.go b/core/sling/task.go index 66af700c8..629e76de4 100644 --- a/core/sling/task.go +++ b/core/sling/task.go @@ -528,6 +528,9 @@ func (t *TaskExecution) getTargetOptionsMap() (options map[string]any) { // set target type for column casing, name length validation options["target_type"] = string(t.Config.TgtConn.Type) + // set to delete file/folder + options["delete_file"] = true + return } diff --git a/core/sling/task_run.go b/core/sling/task_run.go index 25c4ef444..5dffd4864 100644 --- a/core/sling/task_run.go +++ b/core/sling/task_run.go @@ -2,6 +2,7 @@ package sling import ( "context" + "maps" "net/http" "os" "runtime" @@ -277,6 +278,14 @@ func (t *TaskExecution) getSrcApiConn(ctx context.Context) (conn *api.APIConnect return } + // set replication store + state, err := t.Replication.RuntimeState() + if err != nil { + g.Warn("could not get replication store to put into API state: " + err.Error()) + } else { + conn.SetReplicationStore(state.Store) + } + if err := conn.Authenticate(); err != nil { return nil, g.Error(err, "could not authenticate") } @@ -565,7 +574,7 @@ func (t *TaskExecution) runApiToDb() (err error) { err = g.Error(err, "Could not get incremental value") return err } - var syncState map[string]map[string]any + var syncState map[string]any if t.Config.IncrementalVal != nil { err = g.Unmarshal(cast.ToString(t.Config.IncrementalVal), &syncState) if err != nil { @@ -596,6 +605,11 @@ skipGetState: return } + // sync back replication store + if state, err := t.Replication.RuntimeState(); err == nil { + maps.Copy(state.Store, srcConn.GetReplicationStore()) + } + elapsed := int(time.Since(start).Seconds()) if len(t.df.Columns) > 0 { t.SetProgress("inserted %d rows into %s in %d secs [%s r/s]", cnt, t.getTargetObjectValue(), elapsed, getRate(cnt)) @@ -633,7 +647,7 @@ skipGetState: func (t *TaskExecution) runApiToFile() (err error) { start = time.Now() - t.SetProgress("connecting to source api (%s)", t.Config.SrcConn.Type) + t.SetProgress("connecting to source api (%s)", strings.ToLower(t.Config.SrcConn.Name)) srcConn, err := t.getSrcApiConn(t.Context.Ctx) if err != nil { @@ -651,7 +665,7 @@ func (t *TaskExecution) runApiToFile() (err error) { return err } - var syncState map[string]map[string]any + var syncState map[string]any if t.Config.IncrementalVal != nil { err = g.Unmarshal(cast.ToString(t.Config.IncrementalVal), &syncState) if err != nil { @@ -686,6 +700,11 @@ skipGetState: return } + // sync back replication store + if state, err := t.Replication.RuntimeState(); err == nil { + maps.Copy(state.Store, srcConn.GetReplicationStore()) + } + elapsed := int(time.Since(start).Seconds()) t.SetProgress("wrote %d rows to %s in %d secs [%s r/s]", cnt, t.getTargetObjectValue(), elapsed, getRate(cnt)) diff --git a/core/sling/task_run_write.go b/core/sling/task_run_write.go index 8e4e8ab12..a0fde595b 100644 --- a/core/sling/task_run_write.go +++ b/core/sling/task_run_write.go @@ -103,7 +103,7 @@ func (t *TaskExecution) WriteToFile(cfg *Config, df *iop.Dataflow) (cnt uint64, sc := df.StreamConfig() if ct := cfg.Target.Options.ColumnTyping; ct != nil { if dec := ct.MaxDecimals(); dec > 0 { - sc.SetMaxDecimals(dec) // set max decimals if specified + sc.MaxDecimals = dec // set max decimals if specified } } @@ -925,6 +925,12 @@ func writeDataflowViaTempDuckDB(t *TaskExecution, df *iop.Dataflow, fs filesys.F FileSizeBytes: g.PtrVal(t.Config.Target.Options.FileMaxBytes), } + // if one wishes to not write the partition columns + // see https://github.com/slingdata-io/sling-cli/issues/634 + if val := os.Getenv("DUCKDB_WRITE_PARTITION_COLS"); val != "" { + copyOptions.WritePartitionCols = cast.ToBool(val) + } + if len(copyOptions.PartitionFields) > 0 && copyOptions.PartitionKey == "" { return bw, g.Error("missing update_key in order to partition") } @@ -987,7 +993,7 @@ func performUpsert(tgtConn database.Connection, tableTmp, targetTable database.T } g.Debug("performing upsert from temporary table %s to target table %s with primary keys %v", tableTmp.FullName(), targetTable.FullName(), tgtPrimaryKey) - rowAffCnt, err := tgtConn.Upsert(tableTmp.FullName(), targetTable.FullName(), tgtPrimaryKey) + rowAffCnt, err := tgtConn.Merge(tableTmp.FullName(), targetTable.FullName(), tgtPrimaryKey) if err != nil { err = g.Error(err, "could not perform upsert from temp") return err diff --git a/go.mod b/go.mod index 40393010a..fd78c007a 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/exasol/exasol-driver-go v1.0.14 github.com/fatih/color v1.18.0 github.com/flarco/bigquery v0.0.9 - github.com/flarco/g v0.1.157 + github.com/flarco/g v0.1.158 github.com/getsentry/sentry-go v0.27.0 github.com/go-sql-driver/mysql v1.9.1 github.com/gobwas/glob v0.2.3 @@ -216,6 +216,7 @@ require ( github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect diff --git a/go.sum b/go.sum index f3df95370..1421369fe 100644 --- a/go.sum +++ b/go.sum @@ -574,6 +574,8 @@ github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJk github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=