V1.4.18

cmd/sling/tests/replications/r.48.bigquery_time_datatype.yaml

@@ -0,0 +1,63 @@
+source: POSTGRES
+target: BIGQUERY
+
+defaults:
+  mode: truncate
+
+hooks:
+  start:
+    - type: query
+      connection: '{source.name}'
+      query: |
+        DROP TABLE IF EXISTS public.test_time_datatype;
+        CREATE TABLE public.test_time_datatype (
+          id INTEGER,
+          time_col TIME,
+          time_tz_col TIME WITH TIME ZONE,
+          time_precision_col TIME(3)
+        );
+        INSERT INTO public.test_time_datatype VALUES 
+          (1, '08:30:45', '08:31:45+02:00', '08:30:45.123'),
+          (2, '12:15:30', '12:11:30-05:00', '12:15:30.456'),
+          (3, '23:59:59', '23:51:59+00:00', '23:59:59.789');
+
+  end:
+    - type: query
+      connection: '{target.name}'
+      query: |
+        SELECT 
+          column_name, 
+          data_type
+        FROM `public`.INFORMATION_SCHEMA.COLUMNS 
+        WHERE table_name = 'test_time_datatype2' 
+        ORDER BY ordinal_position
+      into: column_info
+
+    - type: log
+      message: |
+        Target table column info:
+        { pretty_table(store.column_info) }
+
+    - type: query
+      connection: '{target.name}'
+      query: SELECT * FROM public.test_time_datatype2 ORDER BY id
+      into: result_data
+
+    - type: log
+      message: |
+        Target table data:
+        { pretty_table(store.result_data) }
+
+    # Clean up
+    - type: query
+      connection: '{source.name}'
+      query: DROP TABLE IF EXISTS public.test_time_datatype
+
+    - type: query
+      connection: '{target.name}'
+      query: DROP TABLE IF EXISTS public.test_time_datatype2
+
+streams:
+  public.test_time_datatype:
+    object: public.test_time_datatype2
+    mode: full-refresh

cmd/sling/tests/suite.cli.yaml

@@ -788,13 +788,29 @@
     - 'execution succeeded'
 
 - id: 104
-  name: Test PostgreSQL TIME datatype preservation
-  run: sling run -d -r cmd/sling/tests/replications/r.46.postgres_time_datatype.yaml
-  streams: 1
-  rows: 3
+  name: Test PostgreSQL & BigQury TIME datatype preservation
+  run: |
+    sling run -d -r cmd/sling/tests/replications/r.46.postgres_time_datatype.yaml
+    sling run -d -r cmd/sling/tests/replications/r.48.bigquery_time_datatype.yaml
   output_contains:
     - '| time without time zone |'
     - '| time with time zone    |'
+    - | # pg output
+      +----+----------+--------------------+----------------+
+      | ID | TIME_COL | TIME_PRECISION_COL | TIME_TZ_COL    |
+      +----+----------+--------------------+----------------+
+      | 1  | 08:30:45 | 08:30:45.123       | 08:30:45+02:00 |
+      | 2  | 12:15:30 | 12:15:30.456       | 12:15:30-05:00 |
+      | 3  | 23:59:59 | 23:59:59.789       | 23:59:59+00:00 |
+      +----+----------+--------------------+----------------+
+    - | # bq output
+      +----+----------+--------------------+----------------+
+      | ID | TIME_COL | TIME_PRECISION_COL | TIME_TZ_COL    |
+      +----+----------+--------------------+----------------+
+      | 1  | 08:30:45 | 08:30:45.123       | 08:31:45+02:00 |
+      | 2  | 12:15:30 | 12:15:30.456       | 12:11:30-05:00 |
+      | 3  | 23:59:59 | 23:59:59.789       | 23:51:59+00:00 |
+      +----+----------+--------------------+----------------+
 
 - id: 105
   name: Test MSSQL uniqueidentifier datatype preservation

core/dbio/connection/connection_local.go

@@ -202,6 +202,14 @@ func GetLocalConns(options ...any) ConnEntries {
 				g.Warn(e)
 				continue
 			}
+		} else if url, ok := payload["url"]; ok && err == nil {
+			cType := SchemeType(cast.ToString(url))
+			conn, err = NewConnectionFromMap(g.M("name", key, "type", cType, "data", payload))
+			if err != nil {
+				e := g.F("could not parse env payload %s: %s", key, g.ErrMsgSimple(err))
+				g.Warn(e)
+				continue
+			}
 		} else {
 			// Parse URL
 			if !strings.Contains(val, "://") || strings.Contains(val, "{") || strings.Contains(val, "[") {

core/dbio/database/database.go

@@ -3151,8 +3151,11 @@ func (conn *BaseConn) credsProvided(provider string) bool {
 }
 
 func (conn *BaseConn) makeTlsConfig() (tlsConfig *tls.Config, err error) {
-	if cast.ToBool(conn.GetProp("tls")) {
+	if val := strings.ToLower(conn.GetProp("tls")); g.In(val, "true", "skip-verify", "custom") {
 		tlsConfig = &tls.Config{}
+		if val == "skip-verify" {
+			tlsConfig.InsecureSkipVerify = true
+		}
 	}
 
 	// legacy only handles files

core/dbio/database/database_bigquery.go

@@ -530,6 +530,7 @@ func getBqSchema(columns iop.Columns) (schema bigquery.Schema) {
 		iop.DecimalType: bigquery.NumericFieldType,
 		// "decimal":   bigquery.FloatFieldType,
 		iop.TimeType:       bigquery.StringFieldType,
+		iop.TimezType:      bigquery.StringFieldType,
 		iop.TimestampType:  bigquery.TimestampFieldType,
 		iop.TimestampzType: bigquery.TimestampFieldType,
 	}