add some unit tests

Author: joroscoSF

src/datacustomcode/client.py

@@ -120,6 +120,7 @@ def __new__(
         if cls._instance is None:
             cls._instance = super().__new__(cls)
 
+            spark = None
             # Initialize Readers and Writers from config
             # and/or provided reader and writer
             if reader is None or writer is None:
@@ -138,6 +139,22 @@ def __new__(
                     provider = DefaultSparkSessionProvider()
 
                 spark = provider.get_session(config.spark_config)
+            elif (
+                proxy is None
+                and config.proxy_config is not None
+                and config.spark_config is not None
+            ):
+                # Both reader and writer provided; we still need spark for proxy init
+                provider = (
+                    spark_provider
+                    if spark_provider is not None
+                    else (
+                        config.spark_provider_config.to_object()
+                        if config.spark_provider_config is not None
+                        else DefaultSparkSessionProvider()
+                    )
+                )
+                spark = provider.get_session(config.spark_config)
 
             if config.reader_config is None and reader is None:
                 raise ValueError(
@@ -155,9 +172,12 @@ def __new__(
                 reader_init = config.reader_config.to_object(spark)  # type: ignore
             else:
                 reader_init = reader
-                if config.proxy_config is None:
+                if proxy is not None:
+                    proxy_init = proxy
+                elif config.proxy_config is None:
                     raise ValueError("Proxy config is required when reader is provided")
-                proxy_init = config.proxy_config.to_object(spark)
+                else:
+                    proxy_init = config.proxy_config.to_object(spark)
             if config.writer_config is None and writer is None:
                 raise ValueError(
                     "Writer config is required when writer is not provided"

tests/test_client.py

@@ -17,6 +17,7 @@
 )
 from datacustomcode.io.reader.base import BaseDataCloudReader
 from datacustomcode.io.writer.base import BaseDataCloudWriter, WriteMode
+from datacustomcode.proxy.client.base import BaseProxyClient
 
 
 class MockDataCloudReader(BaseDataCloudReader):
@@ -75,6 +76,13 @@ def mock_config(mock_spark):
     )
 
 
+@pytest.fixture
+def mock_proxy():
+    """Mock proxy client to avoid starting Spark when reader/writer are provided."""
+    proxy = MagicMock(spec=BaseProxyClient)
+    return proxy
+
+
 @pytest.fixture
 def reset_client():
     """Reset the Client singleton between tests."""
@@ -85,12 +93,12 @@ def reset_client():
 
 class TestClient:
 
-    def test_singleton_pattern(self, reset_client, mock_spark):
+    def test_singleton_pattern(self, reset_client, mock_spark, mock_proxy):
         """Test that Client behaves as a singleton."""
         reader = MockDataCloudReader(mock_spark)
         writer = MockDataCloudWriter(mock_spark)
 
-        client1 = Client(reader=reader, writer=writer)
+        client1 = Client(reader=reader, writer=writer, proxy=mock_proxy)
         client2 = Client()
 
         assert client1 is client2
@@ -136,38 +144,38 @@ def test_initialization_with_config(self, mock_config, reset_client, mock_spark)
             assert client._reader is mock_reader
             assert client._writer is mock_writer
 
-    def test_read_dlo(self, reset_client, mock_spark):
+    def test_read_dlo(self, reset_client, mock_spark, mock_proxy):
         reader = MagicMock(spec=BaseDataCloudReader)
         writer = MagicMock(spec=BaseDataCloudWriter)
         mock_df = MagicMock(spec=DataFrame)
         reader.read_dlo.return_value = mock_df
 
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
         result = client.read_dlo("test_dlo")
 
         reader.read_dlo.assert_called_once_with("test_dlo")
         assert result is mock_df
         assert "test_dlo" in client._data_layer_history[DataCloudObjectType.DLO]
 
-    def test_read_dmo(self, reset_client, mock_spark):
+    def test_read_dmo(self, reset_client, mock_spark, mock_proxy):
         reader = MagicMock(spec=BaseDataCloudReader)
         writer = MagicMock(spec=BaseDataCloudWriter)
         mock_df = MagicMock(spec=DataFrame)
         reader.read_dmo.return_value = mock_df
 
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
         result = client.read_dmo("test_dmo")
 
         reader.read_dmo.assert_called_once_with("test_dmo")
         assert result is mock_df
         assert "test_dmo" in client._data_layer_history[DataCloudObjectType.DMO]
 
-    def test_write_to_dlo(self, reset_client, mock_spark):
+    def test_write_to_dlo(self, reset_client, mock_spark, mock_proxy):
         reader = MagicMock(spec=BaseDataCloudReader)
         writer = MagicMock(spec=BaseDataCloudWriter)
         mock_df = MagicMock(spec=DataFrame)
 
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
         client._record_dlo_access("some_dlo")
 
         client.write_to_dlo("test_dlo", mock_df, WriteMode.APPEND, extra_param=True)
@@ -176,12 +184,12 @@ def test_write_to_dlo(self, reset_client, mock_spark):
             "test_dlo", mock_df, WriteMode.APPEND, extra_param=True
         )
 
-    def test_write_to_dmo(self, reset_client, mock_spark):
+    def test_write_to_dmo(self, reset_client, mock_spark, mock_proxy):
         reader = MagicMock(spec=BaseDataCloudReader)
         writer = MagicMock(spec=BaseDataCloudWriter)
         mock_df = MagicMock(spec=DataFrame)
 
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
         client._record_dmo_access("some_dmo")
 
         client.write_to_dmo("test_dmo", mock_df, WriteMode.OVERWRITE, extra_param=True)
@@ -190,42 +198,42 @@ def test_write_to_dmo(self, reset_client, mock_spark):
             "test_dmo", mock_df, WriteMode.OVERWRITE, extra_param=True
         )
 
-    def test_mixed_dlo_dmo_raises_exception(self, reset_client, mock_spark):
+    def test_mixed_dlo_dmo_raises_exception(self, reset_client, mock_spark, mock_proxy):
         """Test that mixing DLOs and DMOs raises an exception."""
         reader = MagicMock(spec=BaseDataCloudReader)
         writer = MagicMock(spec=BaseDataCloudWriter)
         mock_df = MagicMock(spec=DataFrame)
 
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
         client._record_dlo_access("test_dlo")
 
         with pytest.raises(DataCloudAccessLayerException) as exc_info:
             client.write_to_dmo("test_dmo", mock_df, WriteMode.APPEND)
 
         assert "test_dlo" in str(exc_info.value)
 
-    def test_mixed_dmo_dlo_raises_exception(self, reset_client, mock_spark):
+    def test_mixed_dmo_dlo_raises_exception(self, reset_client, mock_spark, mock_proxy):
         """Test that mixing DMOs and DLOs raises an exception (converse case)."""
         reader = MagicMock(spec=BaseDataCloudReader)
         writer = MagicMock(spec=BaseDataCloudWriter)
         mock_df = MagicMock(spec=DataFrame)
 
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
         client._record_dmo_access("test_dmo")
 
         with pytest.raises(DataCloudAccessLayerException) as exc_info:
             client.write_to_dlo("test_dlo", mock_df, WriteMode.APPEND)
 
         assert "test_dmo" in str(exc_info.value)
 
-    def test_read_pattern_flow(self, reset_client, mock_spark):
+    def test_read_pattern_flow(self, reset_client, mock_spark, mock_proxy):
         """Test a complete flow of reading and writing within the same object type."""
         reader = MagicMock(spec=BaseDataCloudReader)
         writer = MagicMock(spec=BaseDataCloudWriter)
         mock_df = MagicMock(spec=DataFrame)
         reader.read_dlo.return_value = mock_df
 
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
 
         df = client.read_dlo("source_dlo")
         client.write_to_dlo("target_dlo", df, WriteMode.APPEND)
@@ -239,7 +247,7 @@ def test_read_pattern_flow(self, reset_client, mock_spark):
 
         # Reset for DMO test
         Client._instance = None
-        client = Client(reader=reader, writer=writer)
+        client = Client(reader=reader, writer=writer, proxy=mock_proxy)
         reader.read_dmo.return_value = mock_df
 
         df = client.read_dmo("source_dmo")