diff --git a/LICENSE.txt b/LICENSE.txt index 43e0d3b..d49f2bd 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -218,8 +218,8 @@ Software License for q.so KX Confidential -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -231,7 +231,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/README.md b/README.md index 555c2c5..89b0a00 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Then install the latest version of PyKX with the following command: pip install pykx ``` -To install a specific version of PyKX run the following command replacing with a specific released semver version of the interface +To install a specific version of PyKX run the following command replacing `` with a specific released semver version of the interface ```bash pip install pykx== @@ -55,37 +55,20 @@ pip install pykx== ### PyKX License access and enablement -Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](docs/user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a kdb+ license, this can be achieved either through use of a personal evaluation license or receipt of a commercial license. +Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](docs/user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a license. -#### Personal Evaluation License +If you do not have a license go to the [KX Developer Center](https://developer.kx.com/products/kdb-x/install) to obtain a KDB-X license. +You will need to update to KDB-X Python `pykx>=4.0.0` to use the provided license. -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX - -1. Visit https://kx.com/kdb-insights-sdk-personal-edition-download/ and fill in the attached form following the instructions provided. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` - -#### Commercial Evaluation License - -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX - -1. Contact you KX sales representative or sales@kx.com requesting a trial license for PyKX evaluation. Alternately apply through https://kx.com/book-demo. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` - -__Note:__ PyKX will not operate with a vanilla or legacy kdb+ license which does not have access to specific feature flags embedded within the license. In the absence of a license with appropriate feature flags PyKX will fail to initialise with full feature functionality. +Full instructions on license installation are covered on the [Installing page](docs/getting-started/installing.md). ### Supported Environments KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX installed from wheel files. Support for user-built installations of PyKX (e.g. built from the source distribution) is only provided on a best-effort basis. Currently, PyKX provides wheels for the following environments: -- Linux (`manylinux_2_17_x86_64`) with CPython 3.8-3.11 -- macOS (`macosx_10_10_x86_64`) with CPython 3.8-3.11 -- Windows (`win_amd64`) with CPython 3.8-3.11 +- Linux (`manylinux2014_x86_64`, `manylinux2014_aarch64`) with CPython 3.9-3.14 +- macOS (`macosx_10_15_x86_64`, `macosx_10_15_arm64`) with CPython 3.9-3.14 +- Windows (`win_amd64`) with CPython 3.9-3.14 ### Dependencies @@ -93,11 +76,10 @@ KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX PyKX depends on the following third-party Python packages: -- `pandas>=1.2, <2.0; python_version=='3.8'` -- `pandas>=1.2, <=2.3.0; python_version>'3.8'` -- `numpy~=1.22; python_version<'3.11'` -- `numpy~=1.23, <2.3.0; python_version=='3.11'` -- `numpy~=1.26, <2.3.0; python_version>'3.11'` +- `pandas>=1.2, <3.0; python_version>'3.8'` +- `numpy>=1.22; python_version<'3.11'` +- `numpy>=1.23; python_version=='3.11'` +- `numpy>=1.26; python_version>'3.11'` - `pytz>=2022.1` - `toml~=0.10.2` - `dill>=0.2.0` @@ -105,7 +87,7 @@ PyKX depends on the following third-party Python packages: They are installed automatically by `pip` when PyKX is installed. -PyKX also has an optional Python dependency of `pyarrow>=3.0.0, <19.0.0`, which can be included by installing the `pyarrow` extra, e.g. `pip install pykx[pyarrow]` +PyKX also has an optional Python dependency of `pyarrow>=3.0.0`, which can be included by installing the `pyarrow` extra, e.g. `pip install pykx[pyarrow]` When using PyKX with KX Dashboards users will be required to install `ast2json~=0.3` this can be installed using the `dashboards` extra, e.g. `pip install pykx[dashboards]` @@ -169,7 +151,7 @@ cd pykx pip3 install -U '.[all]' ``` -To run PyKX in licensed mode ensure to follow the steps to receive a [Personal Evaluation License](https://code.kx.com/pykx/getting-started/installing.html#personal-evaluation-license) +To run in licensed mode see [here](docs/getting-started/installing.md). Now you can run/test PyKX: diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 7d39e2b..5f2fff5 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,7 +1,7 @@ python: - - 3.8 - 3.9 - 3.10 - 3.11 - 3.12 - 3.13 + - 3.14 diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 5b08776..205de72 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -16,28 +16,30 @@ requirements: build: - git - python - - setuptools>=68.0 + - setuptools>=68.0, <81.0 - setuptools_scm[toml]>=8.0.0 - - cython==3.0.* - - numpy==2.*, <2.3.0 # [py==313] + - cython==3.0.* # [py<314] + - cython>=3.1.3 # [py>=314] + - numpy>=2.3.0 # [py==314] + - numpy==2.* # [py==313] - numpy==2.0.* # [py==312] - numpy==2.0.* # [py==311] - numpy==2.0.* # [py==310] - numpy==2.0.* # [py==39] - - numpy==1.22.* # [py==38] - - numpy==1.20.* # [py==37] - tomli>=2.0.1 - wheel>=0.36 - - sysroot_linux-64 2.17 # [linux64] - - kernel-headers_linux-64 3.10.0 # [linux64] + - sysroot_linux-64 ==2.17 # [linux and py<314] + - kernel-headers_linux-64==3.10.0 # [linux and py<314] + - sysroot_linux-64>=2.28 # [linux and py>=314] + - kernel-headers_linux-64 # [linux and py>=314] run: - python - - numpy>=1.20 # [py==37] - - numpy>=1.22, <2.3.0 # [py>37] - - pandas>=1.2, <=2.3.0 # [py>38] - - pandas<2.0 # [py==38] - - pyarrow>=3.0.0, <19.0.0 + - numpy>=1.22 # [py>38 , py<314] + - numpy>=2.3.0 # [py==314] + - pandas>=1.2 , <3.0 # [py>38 , py<314] + - pandas>=2.3.3, <3.0 # [py==314] + - pyarrow>=3.0.0 - pytz>=2022.1 - toml>=0.10.2 - dill>=0.2.0 diff --git a/custom_theme/partials/header.html.tpl b/custom_theme/partials/header.html.tpl index e412c5a..5ede20a 100644 --- a/custom_theme/partials/header.html.tpl +++ b/custom_theme/partials/header.html.tpl @@ -126,5 +126,5 @@ } - + diff --git a/docs/examples/interface-overview.ipynb b/docs/examples/interface-overview.ipynb index bd7c79a..e75bec0 100644 --- a/docs/examples/interface-overview.ipynb +++ b/docs/examples/interface-overview.ipynb @@ -574,7 +574,7 @@ "source": [ "### 3.5 Query external processes via IPC\n", "\n", - "One of the most common usage patterns in organizations with access to data in kdb+/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://kx.com/kdb-insights-sdk-personal-edition-download/).\n", + "One of the most common usage patterns in organizations with access to data in kdb+/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://code.kx.com/q/).\n", "\n", "First, set up a q/kdb+ server. Set it on port 5050 and populate it with some data in the form of a table `tab`:" ] diff --git a/docs/extras/glossary.md b/docs/extras/glossary.md index 9b18431..a24c45c 100644 --- a/docs/extras/glossary.md +++ b/docs/extras/glossary.md @@ -74,7 +74,7 @@ A database schema is a fundamental concept in database management. It describes The `#!python sorted` attribute ensures that all items in the `#!python Vector`/`#!python Table` column are sorted in ascending order. ## Time-series analysis -Time-series analysis is a specific way of analyzing a sequence of data points collected over an interval of time. Unlike sporadic or random data collection, time-series analysis involves recording data points at consistent intervals within a set period. Use cases include finance, Internet of Things (IoT), and other domains where data evolves over time. PyKX time-series analysis uses q’s query language (qSQL) against kdb+. Learn more about [time-series models](https://code.kx.com/insights/1.9/api/machine-learning/q/analytics/api/variadic/timeseries.html). +Time-series analysis is a specific way of analyzing a sequence of data points collected over an interval of time. Unlike sporadic or random data collection, time-series analysis involves recording data points at consistent intervals within a set period. Use cases include finance, Internet of Things (IoT), and other domains where data evolves over time. PyKX time-series analysis uses q’s query language (qSQL) against kdb+. Learn more about [time-series models](https://code.kx.com/insights/api/machine-learning/q/analytics/api/variadic/timeseries.html). ## Thread A thread is an independent sequence of instructions within a program that can be executed independently of other code. Threads share the same memory space as the process they belong to, allowing them to communicate and share data efficiently. In Python, the threading module provides an intuitive API for working with threads. Learn about [PyKX calling into q from multiple threads](../examples/threaded_execution/threading.md). diff --git a/docs/getting-started/installing.md b/docs/getting-started/installing.md index b394b00..340b961 100644 --- a/docs/getting-started/installing.md +++ b/docs/getting-started/installing.md @@ -21,7 +21,7 @@ _This page explains how to install PyKX on your machine._ Before you start, make sure you have: -- [**Python**](https://www.python.org/downloads/) (versions 3.8-3.13) +- [**Python**](https://www.python.org/downloads/) (versions 3.9-3.14) - [**pip**](https://pypi.org/project/pip/) Recommended: a virtual environment with packages such as [venv](https://docs.python.org/3/library/venv.html) from the standard library. @@ -30,9 +30,9 @@ Recommended: a virtual environment with packages such as [venv](https://docs.pyt KX only supports versions of PyKX built by KX (installed from wheel files) for: -- **Linux** (`manylinux_2_17_x86_64`, `linux-arm64`) with CPython 3.8-3.13 -- **macOS** (`macosx_10_10_x86_64`, `macosx_10_10_arm`) with CPython 3.8-3.13 -- **Windows** (`win_amd64`) with CPython 3.8-3.13 +- **Linux** (`manylinux2014_x86_64`, `manylinux2014_aarch64`) with CPython 3.9-3.14 +- **macOS** (`macosx_10_15_x86_64`, `macosx_10_15_arm64`) with CPython 3.9-3.14 +- **Windows** (`win_amd64`) with CPython 3.9-3.14 We provide assistance to user-built installations of PyKX only on a best-effort basis. @@ -87,22 +87,47 @@ You can install PyKX from three sources: ``` +=== "Install PyKX from UV" + + Ensure you have `#!bash uv` installed, instructions can be found [here.](https://docs.astral.sh/uv/getting-started/installation/) + + Then install the latest version of PyKX with the following command: + + ```sh + uv pip install pykx + + ``` + +=== "Install PyKX from pipx" + + Ensure you have `#!bash pipx` installed, installation guide can be found [here.](https://pipx.pypa.io/stable/installation/) + + Then install PyKX using the flag to install dependencies: + + ```sh + pipx install pykx --include-deps + + ``` + + This will create a virtual environment to run PyKX in, make sure you activate this to access the package. + + At this point you have [partial access to PyKX](../user-guide/advanced/modes.md#1a-running-in-unlicensed-mode). To gain access to all PyKX features, follow the steps in the next section, otherwise go straight to [3. Verify PyKX Installation](#3-verify-pykx-installation). -## 2. Install a kdb Insights license +## 2. Install a license + +To use all PyKX functionalities, you need to download and install a license. -To use all PyKX functionalities, you need to download and install a kdb Insights license. +If you do not have a license, it is recommended to upgrade to KDB-X Python `pykx>=4.0` and obtain a license from the [KX Developer Center](https://developer.kx.com/products/kdb-x/install). !!! Warning "Legacy kdb+/q licenses do not support all PyKX features." -There are two types of kdb Insights licenses for PyKX: personal and commercial. For either of them, you have two installation options: - - a) from Python - - b) using environment variables +!!! Warning "Newer KDB-X Python `pykx>=4.0` licenses will not work with PyKX `pykx<4.0`. Refer to [troubleshooting](../help/troubleshooting.md) for more." -### 2.a Install license in Python +### 2.a Install an existing license in Python -Follow the steps below to install a kdb Insights license for PyKX from Python: +Follow the steps below to install a license for PyKX from Python: 1. Start your Python session: @@ -121,79 +146,45 @@ Follow the steps below to install a kdb Insights license for PyKX from Python: Would you like to install a license? [Y/n]: ``` -3. Indicate whether you have access to an existing PyKX enabled license or not, type `#!python N` or press `#!python Enter` to continue with accessing a new license: +3. Indicate whether you have access to an existing PyKX enabled license or not, type `#!python Y` if you have a license: ```python Do you have access to an existing license for PyKX that you would like to use? [N/y]: ``` -4. Choose whether you wish to install a personal or commercial license, type `#!python Y` or press `#!python Enter` to choose a personal license - - ```python - Is the intended use of this software for: - [1] Personal use (Default) - [2] Commercial use - Enter your choice here [1/2]: - ``` - -5. When asked if you would like to apply for a license, type `#!python Y` or press `#!python Enter`: - - === "Personal license" - - ```bash - To apply for a PyKX license, navigate to https://kx.com/kdb-insights-sdk-personal-edition-download - Shortly after you submit your license application, you will receive a welcome email containing your license information. - Would you like to open this page? [Y/n]: - ``` - - === "Commercial license" - - ```bash - To apply for your PyKX license, contact your KX sales representative or sales@kx.com. - Alternately apply through https://kx.com/book-demo. - Would you like to open this page? [Y/n]: - ``` - -6. For personal use, complete the form to receive your welcome email. For commercial use, the license will be provided over email after the commercial evaluation process has been followed with the support of your sales representative. - -7. Choose the desired method to activate your license by typing `#!python 1`, `#!python 2`, or `#!python 3` as appropriate: +4. Choose the desired method to activate your license by typing `#!python 1` or `#!python 2` as appropriate: ```bash - Select the method you wish to use to activate your license: - [1] Download the license file provided in your welcome email and input the file path (Default) - [2] Input the activation key (base64 encoded string) provided in your welcome email - [3] Proceed with unlicensed mode - Enter your choice here [1/2/3]: + Please select the method you wish to use to activate your license: + [1] Provide the location of your license file + [2] Paste the license key ``` -8. Depending on your choice (`#!python 1`, `#!python 2`, or `#!python 3`), complete the installation by following the final step as below: +5. Depending on your choice (`#!python 1` or `#!python 2`), complete the installation by following the final steps as below: === "1" - === "Personal license" - - ```bash - Provide the download location of your license (for example, ~/path/to/kc.lic): - ``` - - === "Commercial license" - - ```bash - Provide the download location of your license (for example, ~/path/to/k4.lic): - ``` + ```bash + Provide the download location of your license (for example, ~/path/to/kc.lic): + ``` === "2" + Specify the license type: + ```bash - Provide your activation key (base64 encoded string) provided with your welcome email: + Please confirm the license type: + [1] kc.lic - The default + [2] k4.lic - Used in some scenarios ``` - === "3" + + Then paste your license key and hit enter: ```bash - No further actions needed. + Provide your kc.lic license key (base64 encoded string) : ``` -9. Validate the correct installation of your license: +6. Validate the correct installation of your license: ```python >>> kx.q.til(10) @@ -202,29 +193,25 @@ Follow the steps below to install a kdb Insights license for PyKX from Python: ### 2.b Install license with environment variables -For environment-specific flexibility, there are two ways to install your license: by using a file or by copying text. Both are sourced in your welcome email. Click on the tabs below, read the instructions, and choose the method you wish to follow: +For environment-specific flexibility, there are two ways to install your license: by using a file or by copying text. Click on the tabs below, read the instructions, and choose the method you wish to follow: !!! Note "" === "Using a file" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. + 1. Download your license file. - 2. On receipt of an email from KX, download and save the license file to a secure location on your computer. - - 3. Set an environment variable pointing to the folder with the license file. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). + 2. Set an environment variable pointing to the folder with the license file. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). * **Variable Name**: `#!bash QLIC` * **Variable Value**: `#!bash /user/path/to/folder` === "Using text" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. - - 2. On receipt of an email from KX, copy the `#!bash base64` encoded contents of your license provided in plain-text within the email. + 1. Copy the `#!bash base64` encoded contents of your license key provided. - 3. On your computer, set an environment variable `#!bash KDB_LICENSE_B64` when using a personal license or `KDB_K4LICENSE_B64` for a commercial license, pointing with the value copied in step 2. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). + 2. On your computer, set an environment variable with the contents of your `kc.lic` license key named `#!bash KDB_LICENSE_B64`. Or use `KDB_K4LICENSE_B64` if you have an older `k4.lic` license key. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). * **Variable Name**: `KDB_LICENSE_B64` / `KDB_K4LICENSE_B64` - * **Variable Value**: `` + * **Variable Value**: `` To validate if you successfully installed your license with environment variables, start Python and import PyKX as follows: @@ -266,11 +253,10 @@ This command should display the installed version of PyKX. PyKX depends on the following third-party Python packages: - - `pandas>=1.2, <2.0; python_version=='3.8'` - - `pandas>=1.2, <=2.3.0; python_version>'3.8'` - - `numpy~=1.22; python_version<'3.11'` - - `numpy~=1.23, <2.3.0; python_version=='3.11'` - - `numpy~=1.26, <2.3.0; python_version>='3.12'` + - `pandas>=1.2, <3.0; python_version>'3.8'` + - `numpy>=1.22; python_version<'3.11'` + - `numpy>=1.23; python_version=='3.11'` + - `numpy>=1.26; python_version>='3.12'` - `pytz>=2022.1` - `toml~=0.10.2` - `dill>=0.2.0` @@ -291,12 +277,10 @@ This command should display the installed version of PyKX. **Optional Python dependencies:** - - **`pyarrow >=3.0.0, <19.0.0`**: install `pyarrow` extra, for example `pip install pykx[pyarrow]`. + - **`pyarrow >=3.0.0`**: install `pyarrow` extra, for example `pip install pykx[pyarrow]`. - **`find-libpython ~=0.2`**: install `debug` extra, for example `pip install pykx[debug]`. - **`ast2json ~=0.3`**: install with `dashboards` extra, for example `pip install pykx[dashboards]` - **`dill >=0.2`**: install via pip, with `remote` extra, for example `pip install pykx[remote]` - - **`beautifulsoup4 >=4.10.0`**: install with `help` extra, for example `pip install pykx[help]` - - **`markdown2 >=2.5.0`**: install with `help` extra, for example `pip install pykx[help]` - **`psutil >=5.0.0`**: install via pip, with `streaming` extra, for example `pip install pykx[streaming]` - **`torch >2.1`**: install via pip, with `torch` extra, for example `pip install pykx[torch]` @@ -331,11 +315,11 @@ This command should display the installed version of PyKX. | Mac ARM | kdb+ 4.0 | libq.dylib | 2025.02.18 | | Mac x86 | kdb+ 4.0 | libq.dylib | 2025.02.18 | | Windows | kdb+ 4.0 | q.dll/q.lib | 2025.02.18 | -| Linux ARM | kdb+ 4.1 | libq.so | 2025.11.25 | -| Linux x86 | kdb+ 4.1 | libq.so | 2025.11.25 | -| Mac ARM | kdb+ 4.1 | libq.dylib | 2025.11.25 | -| Mac x86 | kdb+ 4.1 | libq.dylib | 2025.11.25 | -| Windows | kdb+ 4.1 | q.dll/q.lib | 2025.11.25 | +| Linux ARM | kdb+ 4.1 | libq.so | 2026.05.01 | +| Linux x86 | kdb+ 4.1 | libq.so | 2026.05.01 | +| Mac ARM | kdb+ 4.1 | libq.dylib | 2026.05.01 | +| Mac x86 | kdb+ 4.1 | libq.dylib | 2026.05.01 | +| Windows | kdb+ 4.1 | q.dll/q.lib | 2026.05.01 | | Linux ARM | Unlicensed | libe.so | 2023.11.22 | | Linux x86 | Unlicensed | libe.so | 2023.11.22 | | Mac ARM | Unlicensed | libe.so | 2023.11.22 | diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index ec3a1de..16f0cdc 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -15,7 +15,7 @@ _This quickstart guide provides first time users with essential instructions for Before you start, make sure to: - [Install the PyKX library](installing.md#1-install-pykx). -- [Have a kdb Insights license](installing.md#2-install-a-kdb-insights-license). +- [Have a license](installing.md#2-install-a-license). ## 1. Import PyKX @@ -26,6 +26,12 @@ To access PyKX, import it within your Python code using the following syntax: ``` !!! Info "The use of the shortened name `#!python kx` is optional and provides a terse convention for interacting with methods and objects from the PyKX library." +Many examples in this guide make use of NumPy so we import this: + +```python +>>> import numpy as np +``` + ## 2. Generate PyKX objects You can generate PyKX objects in three ways. Click on the tabs below to follow the instructions: @@ -64,7 +70,6 @@ You can generate PyKX objects in three ways. Click on the tabs below to follow t >>> qlist pykx.LongVector(pykx.q('10 20 30')) - >>> import numpy as np >>> nplist = np.arange(0, 10, 2) >>> qlist = kx.toq(nplist) >>> qlist @@ -204,7 +209,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i ```python >>> qfunction = kx.q('{x+til 10}') - >>> qfunction(kx.toq([random() for _ in range(10)], kx.FloatVector)) + >>> qfunction(kx.FloatVector([np.random.random() for _ in range(10)])) pykx.FloatVector(pykx.q('0.3992327 1.726329 2.488636 3.653597 4.028107 5.444905 6.542917 7.00628 8.152..')) ``` @@ -218,7 +223,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i pykx.LongVector(pykx.q('5 8 3 3 10 5 3 1 9 1')) ``` -* Pass PyKX arrays of objects to Numpy functions: +* Pass PyKX arrays of objects to NumPy functions: ```python >>> qarray1 = kx.random.random(10, 1.0) @@ -242,7 +247,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i >>> N = 100 >>> qtable = kx.Table( data={ - 'x': kx.random.random(N, ['a', 'b', 'c'], + 'x': kx.random.random(N, ['a', 'b', 'c']), 'x1': kx.random.random(N, 1.0), 'x2': 5 * kx.random.random(N, 1.0), } @@ -268,7 +273,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i a 0.6212161 3.97236 .. ')) - >>> qtable.select(where=kx.Column('col1')=='a') + >>> qtable.select(where=kx.Column('x')=='a') pykx.Table(pykx.q(' x x1 x2 --------------------- @@ -320,7 +325,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i ## 4. Convert PyKX objects to Python types -To convert the objects generated via the PyKX library to the corresponding `#!python Python`, `#!python Numpy`, `#!python Pandas`, and `#!python PyArrow` types, use `#!python py`, `#!python np`, `#!python pd`, and `#!python pa` methods. Click on the tabs below to go through the examples: +To convert the objects generated via the PyKX library to the corresponding `#!python Python`, `#!python NumPy`, `#!python Pandas`, and `#!python PyArrow` types, use `#!python py`, `#!python np`, `#!python pd`, and `#!python pa` methods. Click on the tabs below to go through the examples: === "Convert to Python" @@ -334,22 +339,18 @@ To convert the objects generated via the PyKX library to the corresponding `#!py ')) >>> qdictionary.py() {'a': 5, 'b': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'c': [0.014509072760120034, 0.9131434387527406, 0.5745006683282554, 0.9619080068077892, 0.7609488749876618]} - >>> >>> qvec = kx.toq(np.random.randint(5, size=10)) >>> qvec.py() [0, 2, 4, 1, 2, 1, 0, 1, 0, 1] ``` -=== "Convert to Numpy" +=== "Convert to NumPy" ```python - >>> import numpy as np - >>> random = np.random.random - >>> randint = np.random.randint >>> qvec = kx.q('10?5') >>> qvec.np() array([0, 2, 4, 1, 2, 1, 0, 1, 0, 1]) - >>> qtab = kx.Table([[random(), randint(0, 5)] for _ in range(5)]) + >>> qtab = kx.Table([[np.random.random(), np.random.randint(0, 5)] for _ in range(5)]) >>> qtab pykx.Table(pykx.q(' x x1 @@ -382,7 +383,7 @@ To convert the objects generated via the PyKX library to the corresponding `#!py 8 0 9 1 dtype: int64 - >>> df = pd.DataFrame(data={'x': [random() for _ in range(5)], 'x1': [randint(0, 4) for _ in range(5)]}) + >>> df = pd.DataFrame(data={'x': [np.random.random() for _ in range(5)], 'x1': [np.random.randint(0, 4) for _ in range(5)]}) >>> qtab = kx.toq(df) >>> qtab.pd() x x1 @@ -410,7 +411,7 @@ To convert the objects generated via the PyKX library to the corresponding `#!py 8 2 9 0 dtype: int64[pyarrow] - >>> df = pd.DataFrame(data={'x': [random() for _ in range(5)], 'x1': [randint(0, 4) for _ in range(5)]}) + >>> df = pd.DataFrame(data={'x': [np.random.random() for _ in range(5)], 'x1': [np.random.randint(0, 4) for _ in range(5)]}) >>> qtab = kx.toq(df) >>> qtab.pd(as_arrow=True) x x1 @@ -443,7 +444,7 @@ To convert the objects generated via the PyKX library to the corresponding `#!py 0, 1 ] - >>> df = pd.DataFrame(data={'x': [random() for _ in range(5)], 'x1': [randint(0, 4) for _ in range(5)]}) + >>> df = pd.DataFrame(data={'x': [np.random.random() for _ in range(5)], 'x1': [np.random.randint(0, 4) for _ in range(5)]}) >>> qtab = kx.toq(df) >>> qtab.pa() pyarrow.Table diff --git a/docs/getting-started/what_is_pykx.md b/docs/getting-started/what_is_pykx.md index f2bce8c..543e39b 100644 --- a/docs/getting-started/what_is_pykx.md +++ b/docs/getting-started/what_is_pykx.md @@ -14,6 +14,10 @@ _This page briefly describes PyKX, its use cases, and its connection with q/kdb+ **PyKX** is a Python-first interface to **kdb+** (the world's fastest time-series database) and **q** (kdb+'s underlying vector programming language). PyKX is the result of 10+ years of integrations between two languages: Python and q. Its aim is to help users query and analyze huge amounts of in-memory and on-disk time-series data, significantly faster than other libraries. +- For **Python developers**, PyKX unlocks the speed and power of kdb+ for data processing and storage from within your Python environment. It enables anyone with Python knowledge to apply analytics against vast amounts of data, both in-memory and on-disk, in a fraction of the time, allowing you to focus on getting the best from your data. + +- For **q developers**, PyKX brings together Python's data science ecosystem and the power of kdb+'s vector and time-series analytics. This makes them available in both q and Python environments. You can use it to run q code within a Python environment or embed Python analytics within your q session. + ## Use cases PyKX supports three main use cases, allowing Python data engineers and data scientists to: diff --git a/docs/help/issues.md b/docs/help/issues.md index 4ce458d..78a9f32 100644 --- a/docs/help/issues.md +++ b/docs/help/issues.md @@ -15,6 +15,8 @@ _This page details known issues and functional limitations when using PyKX eithe * Enabling the NEP-49 NumPy allocators will often segfault when running in a multiprocess setting. * The timeout value is always set to 0 when using PYKX_Q_LOCK. * Enabling PYKX_ALLOCATOR and using PyArrow tables can cause segfaults. +* Multiprocessing is not available on Windows. When using PyKX in a multiprocessing setup, PyKX must be imported and initialized within each subprocess where it is needed, regardless of the multiprocessing start method used (spawn, fork, or forkserver). +* Multithreading mode is not available on Windows. * `#!python kurl` functions require their `#!python options` dictionary to have mixed type values. Add a `#!python None` value to bypass: `#!python {'': None, ...}` * `#!python None` and `#!python pykx.Identity(pykx.q('::'))` do not pass through to single argument Python functions set under q as outlined in this example: ```python diff --git a/docs/help/troubleshooting.md b/docs/help/troubleshooting.md index 3f96ba0..c1eb53d 100644 --- a/docs/help/troubleshooting.md +++ b/docs/help/troubleshooting.md @@ -6,12 +6,11 @@ The following section outlines practical information useful when dealing with ge ### Accessing a license valid for PyKX -A number of trial and enterprise type licenses exist for q/kdb+. Not all licenses for q/kdb+ however are valid for PyKX. In particular users require access to a license which contains the feature flags **pykx** and **embedq** which provide access to the PyKX functionality. The following locations can be used for the retrieval of evaluation/personal licenses +Not all licenses for q/kdb+ are valid for PyKX. In particular users require access to a license which contains the feature flags **pykx** and **embedq** which provide access to the PyKX functionality. +Newer KDB-X licenses use shorter flags **py** and **eq**, these work with KDB-X Python `pykx>=4.0` only and not PyKX `pykx<4.0`. -- For non-commercial personal users you can access a 12 month kdb+ license with PyKX enabled [here](https://kx.com/kdb-insights-sdk-personal-edition-download). -- For commercial evaluation, contact your KX sales representative or sales@kx.com requesting a PyKX trial license. Alternately apply through https://kx.com/book-demo. - -For non-personal or non-commercial usage please contact sales@kx.com. +Go to the [KX Developer Center](https://developer.kx.com/products/kdb-x/install) to obtain a KDB-X license. +For other enquiries email sales@kx.com. Once you have access to your license you can install the license following the steps provided [here](../getting-started/installing.md) or through installation using the function `#!python kx.license.install` as follows @@ -24,6 +23,11 @@ Once you have access to your license you can install the license following the s Failure to initialize PyKX while raising an error `#!python embedq` indicates that the license you are attempting to use for PyKX in [licensed modality](../user-guide/advanced/modes.md) does not have the sufficient feature flags necessary to run PyKX. To access a license which does allow for running PyKX in this modality please following the instructions [here](#accessing-a-license-valid-for-pykx) to get a new license with appropriate feature flags. +This error can also be thrown when attempting to use a KDB-X license with PyKX. +Newer KDB-X licenses use shorter flags **py** and **eq**, these work with KDB-X Python `pykx>=4.0` only and not PyKX `pykx<4.0`. + +Check the output of `.z.l` in a q process to view the flags in your license. + ### Initialization failing with a `#!python kc.lic` error If after initially completing the installation guide for PyKX [here](../getting-started/installing.md) you receive the following error: @@ -34,6 +38,7 @@ pykx.exceptions.PyKXException: Failed to initialize embedded q. Captured output ``` It usually indicates that your license was not correctly written to disk or a license could not be found, to check that the installed license matches the license you expect. +This error can also be thrown when attempting to use a KDB-X license with PyKX. === "License file based checking" diff --git a/docs/index.md b/docs/index.md index 4fbec86..651c04b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,41 +1,67 @@ -# PyKX Documentation +# PyKX Home + +_Welcome to the home of PyKX technical documentation._ + +[PyKX](./getting-started/what_is_pykx.md) is a Python-first interface to kdb+, the world's fastest time-series database, and its underlying vector programming language q. Whether you're a Python developer looking to harness kdb+'s analytics, or a q developer wanting to leverage Python's data science ecosystem, PyKX bridges both worlds. + +## Ready to dive in? + +
+
+

Get Started

+ +
+ +
+

How-to Guides

+ +
+ +
+

Integrations

+ +
+ +
+ +## What's next? + +Discover how PyKX builds upon kdb+ and what's coming soon: + +- Submit a [feature request](https://forum.kx.com/c/ideas-feature-requests/10) +- Stay updated with the latest [release notes](release-notes/changelog.md){target="_blank"} +- Join our [community channels](https://kx-community.slack.com/) to get assistance -## About - -Welcome to [PyKX](./getting-started/what_is_pykx.md)! PyKX is a Python first interface to the world's fastest time-series database kdb+ and its underlying vector programming language, q. - -For Python developers, PyKX unlocks the speed and power of kdb+ for data processing and storage from within your Python environment. It enables anyone with Python knowledge to apply analytics against vast amounts of data, both in-memory and on-disk, in a fraction of the time, allowing you to focus on getting the best from your data. - -For q developers, PyKX brings together Python's data science ecosystem and the power of kdb+'s vector and time-series analytics. This makes them available in both q and Python environments. You can use it to run q code within a Python environment or embed Python analytics within your q session. - -To begin your journey with PyKX, follow the sections below: - -## Documentation overview - -!!! home-page "[Get Started](./getting-started/installing.md)" - - This section is for new users. Includes everything you need to begin your journey with PyKX: [installation instructions](./getting-started/installing.md) and a [quickstart guide](./getting-started/quickstart.md). - -!!! home-page "[Learn](./getting-started/what_is_pykx.md)" - - Explore the core concepts of our technology. This section provides you with detailed explanations and access to [KX Academy](https://learninghub.kx.com/courses/introduction-to-pykx/) materials. - -!!! home-page "[How To](./user-guide/configuration.md)" - - Follow step-by-step guides to take you from initial setup to advanced production systems. You’ll find practical advice and best practices here. - -!!! home-page "[Reference](./api/pykx-execution/q.md)" - - Find detailed technical references for methods, functions, and APIs. This is your go-to source for all technical definitions and usage examples. - -!!! home-page "[Integrations](./user-guide/advanced/numpy.md)" - - Learn how to integrate PyKX with other tools and platforms. This section includes guides and compatibility information to ensure seamless interoperability. - -!!! home-page "[Releases](./release-notes/changelog.md)" - - Stay updated with the latest release notes. You’ll find information on the latest releases/fixes, and previous versions. - -!!! home-page "[Help and Support](./help/troubleshooting.md)" - - Get assistance with any questions or issues you might have. This section includes [troubleshooting](./help/troubleshooting.md) guides, [FAQs](./help/faq.md), and [support](./help/support.md) contact information. diff --git a/docs/license.md b/docs/license.md index 7421799..6114927 100644 --- a/docs/license.md +++ b/docs/license.md @@ -63,8 +63,8 @@ END OF TERMS AND CONDITIONS Software License for q.so -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -76,7 +76,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index 199dad9..e150174 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -1,16 +1,550 @@ # PyKX Changelog +This changelog tracks all PyKX releases and their associated additions, fixes, and improvements. + !!! Note - The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). + For changes to PyKX within a q environment, refer to the [PyKX under q changelog](./underq-changelog.md). + +## PyKX 3.2.0 + +**Release Date** + +2026-06-24 + +**Additions** + +- Added `by` keyword to `merge_asof`. + + ```python + >>> import pandas as pd + >>> left = pd.DataFrame({"a": [1, 5, 10], "b":["a", "a", "b"], "left_val": ["a", "b", "c"]}) + >>> right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "b":["a", "a", "b", "b", "a"], "right_val": [1, 2, 3, 6, 7]}) + >>> import pykx as kx + >>> qleft = kx.toq(left) + >>> qright = kx.toq(right) + >>> kx.merge_asof(qleft, qright, on='a') + pykx.Table(pykx.q(' + a b_x left_val b_y right_val + ----------------------------- + 1 a a a 1 + 5 a b b 3 + 10 b c a 7 + ')) + >>> kx.merge_asof(qleft, qright, on='a', by="b") + pykx.Table(pykx.q(' + a b left_val right_val + ----------------------- + 1 a a 1 + 5 a b 2 + 10 b c 6 + ')) + ``` + +- Added `__getitem__` to `pykx.Column` class. + + ```python + >>> t = kx.q('([] a:5 10 15 20)') + # Equivalent to qsql: select a 1 2 from t + >>> t.select(kx.Column('a')[[1,2]]) + pykx.Table(pykx.q(' + a + -- + 10 + 15 + ')) + >>> kx.q['myvar'] = [1, 2] + # Equivalent to qsql: select a myvar from t + >>> t.select(kx.Column('a')[kx.Variable('myvar')]) + pykx.Table(pykx.q(' + a + -- + 10 + 15 + ')) + >>> kx.q('mylookup:5 10 15 20!`five`ten`fifteen`twenty') + pykx.Identity(pykx.q('::')) + # Equivalent to qsql: update word:mylookup a from t + >>> t.update(kx.Column('word', data=kx.Variable('mylookup'))[kx.Column('a')]) + pykx.Table(pykx.q(' + a word + ---------- + 5 five + 10 ten + 15 fifteen + 20 twenty + ')) + ``` + +- Added `.enlist()` method to `pykx.Column` class to enlist data in a `pykx.Column` object. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.exec(kx.Column('n').enlist(iterator='each')) + pykx.List(pykx.q(' + 2 + 4 + ')) + ``` + +- Added `.all()` method to `pykx.Column` class to return `1b` when all items are nonzero. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 5#1b)') + >>> t.exec(kx.Column('n').all()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + + +- Added `.any()` method to `pykx.Column` class which returns `1b` when at least one item is nonzero. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 0010b)') + >>> t.exec(kx.Column('n').any()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + +- Added `._in()` method to `pykx.Column` class to check whether an item is in the column. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 4 5)') + >>> t.select(kx.Column('n')._in([0,2,4], iterator=r'\:')) + pykx.Table(pykx.q(' + n + - + 0 + 1 + 0 + 1 + 0 + ')) + ``` + +- Added `._except()` method to `pykx.Column` class to return column values that are not present in the supplied variable. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: til 5)') + >>> t.exec(kx.Column('n')._except(3)) + pykx.LongVector(pykx.q('0 1 2 4')) + ``` + +- Added `.hsym()` method to `pykx.Column` class which returns a column of symbol objects prefixed with a colon. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: `a`b`c)') + >>> t.exec(kx.Column('n').hsym()) + pykx.SymbolVector(pykx.q('`:a`:b`:c')) + ``` + +- Added `.raze()` method to `pykx.Column` class to collapse column values. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').raze(iterator='over')) + pykx.LongVector(pykx.q('1 2 3 4 5 6 7 8')) + ``` + +- Added `.type()` method to `pykx.Column` class to return type of values in a column. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.ShortVector(pykx.q('7 0 -7 -7h')) + ``` + +- Added `.key()` method to `pykx.Column` class to return keys from a column of dictionaries. + ```python + >>> import pykx as kx + >>> d1 = kx.q('`a`b`c ! 1 2 3') + >>> d2 = kx.q('`d`e`f ! 4 5 6') + >>> t = kx.Table(data={'n':[d1, d2]}) + >>> t.exec(kx.Column('n').key(iterator='each')) + pykx.Table(pykx.q(' + n + ----- + a b c + d e f + ')) + ``` + +- Added `.get()` method to `pykx.Column` class to read from a column of named global variables or file names. + ```python + >>> import pykx as kx + >>> kx.q('`:tb1 set ([] n: `a`b`c)') + pykx.SymbolAtom(pykx.q('`:tb1')) + >>> kx.q('`:tb2 set ([] n: 1 2 3)') + pykx.SymbolAtom(pykx.q('`:tb2')) + >>> t = kx.q('([] n: `:tb1`:tb2)') + >>> t.exec(kx.Column('n').get(iterator='each')) + pykx.List(pykx.q(' + +(,`n)!,`a`b`c + +(,`n)!,1 2 3 + ')) + ``` + +- Added `.read0()` method to `pykx.Column` class to read text from a file or process handle. + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read0(iterator='each')) + pykx.List(pykx.q(' + "hello" + "goodbye" + ')) + ``` + +- Added `.read1()` method to `pykx.Column` class to read bytes from a file or named pipe. + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read1(iterator='each')) + pykx.List(pykx.q(' + 0x68656c6c6f0a + 0x676f6f646279650a + ')) + ``` + +- Added `.next()` method to `pykx.Column` class to return the next item in a list. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:3 6 9 12)') + >>> t.exec(kx.Column('n').call('next')) + pykx.LongVector(pykx.q('6 9 12 0N')) + ``` + +- Added `.prior()` method to `pykx.Column` class to apply a function across current and previous value. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 3 2 1)') + >>> t.update(kx.Column('n').prior(kx.q('<')).name('lessThanPrevious')) + pykx.Table(pykx.q(' + n lessThanPrevious + ------------------ + 1 0 + 2 0 + 3 0 + 3 0 + 2 1 + 1 1 + ')) + ``` + +- Added `.where()` method to `pykx.Column` class to return indexes of True values. + + ```python + >>> import pykx as kx + >>> tab = kx.q('([] a:101010b)') + >>> tab.exec(kx.Column('a').where()) + pykx.LongVector(pykx.q('0 2 4')) + ``` + +- Added `.fills()` method to `pykx.Table`, `pykx.KeyedTable`, and `pykx.Vector` classes to replace nulls with preceding non-nulls. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:0N 2 0N 3 0N;b:``h`j``;g:0N 1 0N 0N 2)') + >>> t + pykx.Table(pykx.q(' + n b g + ----- + + 2 h 1 + j + 3 + 2 + ')) + >>> t.fills() + pykx.Table(pykx.q(' + n b g + ----- + + 2 h 1 + 2 j 1 + 3 j 1 + 3 j 2 + ')) + ``` + +- Added `.ungroup()` method to `pykx.Table` and `pykx.KeyedTable` classes to ungroup nested values. + + ```python + >>> import pykx as kx + >>> t = q('([] n:1 2 3;nn:(1 1;2 2 2;3 3 3))') + >>> t + pykx.Table(pykx.q(' + n nn + ------- + 1 1 1 + 2 2 2 2 + 3 3 3 3 + ')) + >>> t.ungroup() + pykx.Table(pykx.q(' + n nn + ---- + 1 1 + 1 1 + 2 2 + 2 2 + 2 2 + 3 3 + 3 3 + 3 3 + ')) + ``` + +**Fixes and Improvements** + +- Updated 4.1 to 2026.05.01 for all platforms. +- Fixed setting the contents of `Column` objects using `data=` or `.data()` for inputs: `SymbolAtom`, `SymbolVector`, `str`, and lists of `str`. + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:1 2 3)') + >>> t.update(kx.Column('new', data='this')) + Traceback (most recent call last): + ... + pykx.exceptions.QError: this + ``` + + === "Behavior after change" + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:1 2 3)') + >>> t.update(kx.Column('new', data='this')) + pykx.Table(pykx.q(' + a new + ------ + 1 this + 2 this + 3 this + ')) + ``` + +- Fixed dictionary item assignment that could result in errors or incorrect types: + + === "Behavior prior to change" + + ```python + # Single row table saved incorrectly as dictionary + >>> a = kx.q('`a`b!(`;2)') + >>> c = kx.q('([] a:enlist 2;b:enlist 6)') + >>> a['c'] = c + >>> a['c'] + pykx.Dictionary(pykx.q(' + a| 2 + b| 6 + ')) + + # Multiple row table fails + >>> a = kx.q('`a`b!(`;2)') + >>> d = kx.q('([] a:1 2;b:3 6)') + >>> a['d'] = d + Traceback (most recent call last): + File "", line 1, in + a['d'] = d + ~^^^^^ + ... + pykx.exceptions.QError: length + ``` + + === "Behavior after change" + + ```python + >>> a = kx.q('`a`b!(`;2)') + >>> c = kx.q('([] a:enlist 2;b:enlist 6)') + >>> a['c'] = c + >>> a['c'] + pykx.Table(pykx.q(' + a b + --- + 2 6 + ')) + + >>> a = kx.q('`a`b!(`;2)') + >>> d = kx.q('([] a:1 2;b:3 6)') + >>> a['d'] = d + >>> a['d'] + pykx.Table(pykx.q(' + a b + --- + 1 3 + 2 6 + ')) + ``` + +- Removed `.pykx.i.errorList` from `astype` calls to fix `'noupdate` errors when called from a thread. +- Updated `help` on `kx.q` methods to use local documentation instead of making a web request. +- Fixed a memory leak when using `.pykx.getattr` on foreign objects in `q`. +- Improved `.cast('symbol')` on `pykx.Column` to correctly cast `Char` columns to `Symbol`. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:("this";"that"))') + >>> t + pykx.Table(pykx.q(' + a + ------ + "this" + "that" + ')) + >>> t.select(kx.Column('a').cast('symbol')) + pykx.Table(pykx.q(' + a + ---- + this + that + ')) + ``` + +**Version Support Changes** + +- Python 3.8 is no longer supported as it is [end of life](https://devguide.python.org/versions/#unsupported-versions). + +## PyKX 3.1.9 + +**Release Date** + +2026-04-02 + +**Additions** + +- Added `.np()` method to `pykx.serialize` function, this returns the serialized data as a NumPy array. +- Added support for calling `pykx.deserialize` on NumPy arrays of serialized `K` objects. + + ```python + >>> import pykx as kx + >>> vec = kx.q('til 10') + >>> vec + pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) + >>> ser = kx.serialize(vec) + >>> type(ser) + + >>> ser_np = ser.np() + >>> type(ser_np) + + >>> ser_np.dtype + dtype('uint8') + >>> kx.deserialize(ser_np) + pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) + ``` + +- Added `.bin()` method to `pykx.Column` class, to perform a binary search using the data in a `pykx.Column` object. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.select(kx.Column('n').bin([1,3,5])) + pykx.Table(pykx.q(' + n + - + 0 + 1 + ')) + ``` + +**Fixes and Improvements** + +- Updated 4.1 to 2026.04.01 for all platforms. + +## PyKX 3.1.8 + +**Release Date** + +2026-02-18 + +**Additions** + +- Added `connection_timeout` keyword argument to `AsyncQConnection` objects allowing the specification of a timeout for the initial connection to a `q` server. +- Added the usage of `.q.Q.ens` for `sym_enum` argument in `DB` objects. + +## PyKX 3.1.7 + +**Release Date** + +2026-02-10 + +**Additions** + +- Added support for Python 3.14. + +**Fixes and Improvements** + +- Updated 4.1 to 2026.01.23 for all platforms. +- Added more strict type checks for naming `Column` objects, allowing only `str` and `SymbolAtom` values. + + === "Behavior prior to change" + + ```python + >>> kx.Column(name=0, value=[1,2,3]) + pykx.Column(name='0', value=) + + >>> kx.Column(name='a', value=(1,2,3)) + pykx.Column(name='a', value=) + ``` + + === "Behavior after change" + + ```python + >>> kx.Column(name=0, value=[1,2,3]) + TypeError: Column 'name' can only be of type Str and pykx.SymbolAtom + + >>> kx.Column(name='a', value=(1,2,3)) + pykx.Column(name='a', data=) + ``` +- Changed Pandas dependency restriction from `<= 2.3.0` to `<3.0`. +- Removed NumPy dependency restriction `<2.3.0`. +- Removed BeautifulSoup dependency from PyKX. The help system now uses native Python libraries (`urllib` and `re`) for web scraping. +- Added `.time` to `Column` objects to allow for the extraction of time information from temporal columns. + + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={'time': kx.random.random(3, kx.TimestampAtom.inf)}) + >>> tab + pykx.Table(pykx.q(' + time + ----------------------------- + 2118.11.10D05:05:45.270444032 + 2052.01.18D19:21:15.973214208 + 2088.03.13D18:56:07.388233728 + ')) + >>> tab.select(kx.Column('time').time) + pykx.Table(pykx.q(' + time + ------------ + 05:05:45.270 + 19:21:15.973 + 18:56:07.388 + ')) + ``` + +**Deprecations & Removals** + +- Deprecated the `value` keyword when creating `kx.Column` objects, use `data` instead. ## PyKX 3.1.6 -#### Release Date +**Release Date** 2025-12-01 -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.1 to 2025.11.25 for all platforms. - Fixed issue where setting `sort=True` when calling `merge` on two `kx.Tables` ignored the supplied `on` parameter. @@ -41,19 +575,19 @@ pykx.Table(pykx.q(' r_x date k_x r_y k_y -------------------------- - 2 2025.11.25 10 5 13 - 4 2025.11.26 12 6 14 - 3 2025.11.27 11 7 15 + 2 2025.11.25 10 5 13 + 4 2025.11.26 12 6 14 + 3 2025.11.27 11 7 15 ')) ``` ## PyKX 3.1.5 -#### Release Date +**Release Date** 2025-10-21 -### Fixes and Improvements +**Fixes and Improvements** - Availability for Intel Macs on PyPI has been restored. - Paths starting with '~' will now have it expanded to users home path in `PYKX_CONFIGURATION_LOCATION`, `Encrypt`, `DB`, and `system.load` functionality (using `os.path.expanduser`). @@ -66,7 +600,7 @@ >>> db2 = kx.DB(path="tmp/db2") PyKXWarning: Only one DB object exists at a time within a process. Use overwrite=True to overwrite your existing DB object. This warning will error in future releases. >>> db3 = kx.DB(path="tmp/db3", overwrite=True) - >>> + >>> ``` - Use of `Table.astype()` was creating a `.papi.errorList` variable, this has been moved inside the `.pykx` namespace as `.pykx.i.errorList`. @@ -100,7 +634,7 @@ >>> np.asarray(kx.DateAtom(datetime.datetime(2003, 4, 5))).dtype dtype('O') ``` - + === "Behavior post change" ```python @@ -122,7 +656,7 @@ >>> type(qtab.exec(kx.Column('col1').name('symcol'))) pykx.wrappers.SymbolVector ``` - + === "Behavior after change" ```python @@ -132,28 +666,28 @@ pykx.wrappers.Dictionary ``` -### Deprecations & Removals +**Deprecations & Removals** - Creating more than one `DB` instance without specifying `overwrite=True` ## PyKX 3.1.4 -#### Release Date +**Release Date** 2025-07-17 -### Fixes and Improvements +**Fixes and Improvements** - Resolved error on import when `PYKX_THREADING` was set. - Fixed an issue where data could be corrupted in keyed columns of PyArrow backed Pandas dataframes. ## PyKX 3.1.3 -#### Release Date +**Release Date** 2025-06-12 -### Additions +**Additions** - Addition of the utility function `kx.util.delete_q_variable` which can be used to delete a named variable stored q memory within a namespace or in the global namespace @@ -270,7 +804,7 @@ - Added the current number of secondary threads being used by q to `kx.util.debug_environment()` output - Added option to pass `str` or `Path` to `pykx.install_into_QHOME` keyword `to_local_folder` -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.0 to 2025.02.18 for all platforms. - Updated 4.1 to 2025.04.28 for all platforms. @@ -344,7 +878,7 @@ raise ValueError('nyi') ValueError: nyi ``` - + === "Behavior post change" ```python @@ -359,11 +893,11 @@ - Refactored `Vector.index()` from Python to q code to be more performant. - Fixed `PartitionedTable.rename_column()` operation on anymap columns. The `#` and `##` files were not renamed. Now all files are renamed. - + === "Behavior prior to change" ```python - >>> import pykx as kx + >>> import pykx as kx >>>N = 10000000 >>>trade = kx.Table(data={ ... 'date': kx.random.random(N, kx.DateAtom('today') - [1, 2, 3, 4]), @@ -396,7 +930,7 @@ === "Behavior post change" ```python - >>> import pykx as kx + >>> import pykx as kx >>>N = 10000000 >>>trade = kx.Table(data={ ... 'date': kx.random.random(N, kx.DateAtom('today') - [1, 2, 3, 4]), @@ -435,10 +969,10 @@ - Removed `Early garbage collection requires a valid q license.` error thrown on import when loading PyKX in unlicensed mode with `PYKX_GC`. - More helpful error messages in cases of missing/corrupt/incompatible licenses. - `pykx.util.add_to_config` now accepts `bool` and `int` values instead of only `str` objects. - + ```python >>> kx.util.add_to_config({'PYKX_GC': True, 'PYKX_MAX_ERROR_LENGTH': 1}) - + Configuration updated at: /home/user/.pykx-config. Profile updated: default. Successfully added: @@ -446,27 +980,27 @@ - PYKX_MAX_ERROR_LENGTH = 1 ``` -### Deprecations & Removals +**Deprecations & Removals** - Deprecated `kx.q.system.console_size`, use `kx.q.system.display_size` instead. ## PyKX 3.1.2 -#### Release Date +**Release Date** 2025-03-18 -### Fixes and Improvements +**Fixes and Improvements** - Fixes an issue when using `PYKX_THREADING` that could cause a segfault in some scenarios. ## PyKX 3.1.1 -#### Release Date +**Release Date** 2025-02-14 -### Fixes and Improvements +**Fixes and Improvements** - Fixed issue whereby PyKX would prompt for user inputs if a license was not found in a non-interactive session, now correctly falls back to unlicensed mode. - Fixed issue whereby PyKX could list splayed tables loaded outside of initialization of the `kx.DB` module as being supported elements of the database @@ -501,11 +1035,11 @@ ## PyKX 3.1.0 -#### Release Date +**Release Date** 2025-02-11 -### Additions +**Additions** - Added support for `Python 3.13`. - Added support for `NumPy 2.0` @@ -632,7 +1166,7 @@ pykx.LongVector(pykx.q('0 1 2 3 4')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Application of the `#!python str` function on empty PyKX objects could return unexpected results @@ -693,7 +1227,7 @@ - Attempting to create a partitioned databases/add a partition to a database with a `sym_enum` keyword but no `by_field` would result in a `KeyError`. - Improved output when a licence file cannot be found, full paths checked for a license file are now shown and default license installation process is offered to user. -- Users are now given the option to input a base64 encoded string when activating an existing license +- Users are now given the option to input a base64 encoded string when activating an existing license - Using `math.inf` or `-math.inf` when creating numeric values now creates equivalent PyKX types ```python @@ -769,7 +1303,7 @@ >>> kx.TimestampAtom(x) pykx.TimestampAtom(pykx.q('2025.01.28D09:43:06.000000000')) ``` - + - Fixed error when attempting to convert a `pandas.Categorical` to an existing `pykx.EnumVector` via `pykx.toq`. If the `pandas.Categorical` data contained a value outside of the `pykx.EnumVector` an error was thrown. === "Behavior prior to change" @@ -815,7 +1349,7 @@ - Error message when checking a license referenced a function `pykx.util.install_license` which is deprecated, this has now been updated to reference `pykx.license.install` -### Beta Features +**Beta Features** - Added ability for users to convert between PyKX numeric vectors or N-Dimensional Lists and PyTorch Tensor objects using the `pt` method. @@ -834,7 +1368,7 @@ ## PyKX 3.0.1 -#### Release Date +**Release Date** 2024-12-04 @@ -842,11 +1376,11 @@ PyKX 3.0.1 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. To install PyKX 3.0.1 on Mac please install from source [here](https://github.com/kxsystems/pykx). -### Additions +**Additions** - Addition of `.replace()` function to `kx.Vector` and `kx.List` objects to search for and replace items in each collection, retaining typing where appropriate. - ```python + ```python >>> l = kx.q('("a";3;1.3;`b)') >>> l.replace(1.3, "junk") pykx.List(pykx.q(' @@ -869,10 +1403,10 @@ pykx.IntVector(pykx.q('7 10 12..')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Added warning to `kx.q.system.load` and context registration when attempting to load path with a space. Can be suppressed by enabling `PYKX_SUPPRESS_WARNINGS`. -- Changed `%%python` heading to `%%py` when calling Python code during `jupyter_qfirst` mode so as not to conflict with inbuilt Jupyter cell magics. +- Changed `%%python` heading to `%%py` when calling Python code during `jupyter_qfirst` mode so as not to conflict with inbuilt Jupyter cell magics. - Fixed `kx.license.check(format='string')` to remove newline characters during comparison. - Configuration file `.pykx-config` now supports use of boolean toml configuration @@ -914,13 +1448,13 @@ columns datatypes --------------------- a "kx.LongAtom" - b "kx.IntAtom" + b "kx.IntAtom" ')) ``` ## PyKX 3.0.0 -#### Release Date +**Release Date** 2024-11-12 @@ -928,7 +1462,7 @@ PyKX 3.0.0 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. To install PyKX 3.0.0 on Mac please install from source [here](https://github.com/kxsystems/pykx). -### Additions +**Additions** - Addition of functionality to allow for development of end-to-end streaming workflows consisting of data-ingestion, persistence and query. This functionality is outlined in-depth [here](../user-guide/advanced/streaming/index.md) with a worked example [here](../examples/streaming/index.md). @@ -954,7 +1488,7 @@ ... 'sym': kx.random.random(100, ['AAPL', 'GOOG', 'MSFT']), ... 'date': kx.random.random(100, kx.q('2022.01.01') + [0,1,2]), ... 'price': kx.random.random(100, 1000.0), - ... 'size': kx.random.random(100, 100) + ... 'size': kx.random.random(100, 100) ... }) >>> table.select(columns=kx.Column('price').max(), where=kx.Column('size') > 5) >>> table.update(column=kx.Column('price').wavg(kx.Column('size')).rename('vwap'), by=kx.Column('sym')) @@ -1141,7 +1675,7 @@ >>> import pykx as kx >>> tab = kx.q('flip (`a;`$"!";`a;`$"a b")!4 4#16?1f') >>> kx.util.detect_bad_columns(tab) - /usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py:593: UserWarning: + /usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py:593: UserWarning: Duplicate columns or columns with reserved characters detected: Duplicate columns: ['a'] Invalid columns: ['!', 'a b'] @@ -1292,7 +1826,7 @@ - All infinity and null properties on atom classes are now usable in unlicensed mode: `null`, `inf`, `inf_pos`, `inf_neg`, `is_inf`, `is_neg_inf`, `is_pos_inf`. -### Fixes and Improvements +**Fixes and Improvements** - Previously built-in `#!python help()` python function, when called on q keywords such as `#!python kx.q.rand` or `#!python kx.q.max` was displaying the generic `wrappers` object help message. As noted in [the installation docs](../getting-started/installing.md#dependencies), this functionality requires the installation of optional libraries available through `pip install pykx[help]`. @@ -1400,17 +1934,17 @@ >>> kx.util.debug_environment() .. **** PyKX Environment Variables **** - PYKX_IGNORE_QHOME: - PYKX_KEEP_LOCAL_TIMES: - PYKX_ALLOCATOR: - PYKX_GC: - PYKX_LOAD_PYARROW_UNSAFE: - PYKX_MAX_ERROR_LENGTH: - PYKX_NOQCE: - PYKX_Q_LIB_LOCATION: - PYKX_RELEASE_GIL: - PYKX_Q_LOCK: - PYKX_DEFAULT_CONVERSION: + PYKX_IGNORE_QHOME: + PYKX_KEEP_LOCAL_TIMES: + PYKX_ALLOCATOR: + PYKX_GC: + PYKX_LOAD_PYARROW_UNSAFE: + PYKX_MAX_ERROR_LENGTH: + PYKX_NOQCE: + PYKX_Q_LIB_LOCATION: + PYKX_RELEASE_GIL: + PYKX_Q_LOCK: + PYKX_DEFAULT_CONVERSION: .. ``` @@ -1462,7 +1996,7 @@ ... 'price': kx.random.random(100, 10.0)}) >>> tab.groupby('sym')['price'].max() pykx.KeyedTable(pykx.q(' - sym| price + sym| price ---| -------- a | 9.830794 b | 9.761246 @@ -1649,7 +2183,7 @@ >>> import pykx as kx >>> db = kx.DB(path='/tmp/test directory/mydb') >>> db.tables - ['trade'] + ['trade'] ``` - Improved handling of invalid methods for Splayed and Partitioned Tables. @@ -1733,7 +2267,7 @@ ``` - Changed return of `pykx.KeyedTable.keys()` from a list of tuples to a `pykx.Table` to be consistent with `pykx.KeyedTable.values()` and unkeyed `pykx.Table`. - + === "Behavior prior to change" ```python @@ -1863,10 +2397,10 @@ >>> tab.dtypes pykx.Table(pykx.q(' - columns datatypes type + columns datatypes type ----------------------------------- a "kx.LongAtom" "kx.LongAtom" - b "kx.IntAtom" "kx.IntAtom" + b "kx.IntAtom" "kx.IntAtom" ')) >>> tab.select_dtypes(include=['SymbolAtom']) @@ -1895,10 +2429,10 @@ >>> tab.dtypes pykx.Table(pykx.q(' - columns datatypes type + columns datatypes type ----------------------------------- a "kx.LongAtom" "kx.LongAtom" - b "kx.IntAtom" "kx.IntAtom" + b "kx.IntAtom" "kx.IntAtom" ')) >>> tab.select_dtypes(include=['SymbolAtom']) @@ -1930,7 +2464,7 @@ File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory pykx.exceptions.QError: nyi ``` - + === "Behavior post change" @@ -1957,7 +2491,7 @@ /usr/python/site-packages/pykx/wrappers.py:2246: UserWarning: Attempting to call numpy.. warn('Warning: Attempting to call numpy __array_function__ on a ' 9 - ``` + ``` === "Behavior post change" @@ -1980,24 +2514,24 @@ /usr/python/site-packages/pykx/config.py: RuntimeWarning: '-t' argument unsupported in QARGS.. ``` -### Upgrade considerations +**Upgrade considerations** - PyKX 3.0.0 is a major version update which includes changes requiring review before upgrading from 2.5.*. A page details these changes in full [here](../upgrades/2030.md) ## PyKX 2.5.5 -#### Release Date +**Release Date** 2024-11-28 -### Fixes and Improvements +**Fixes and Improvements** - PyKX Pandas dependency has been raised to allow `<=2.2.3` for Python>3.8 - PyKX Pandas dependency for Python 3.8 has been clamped to `<2.0` due to support being dropped for it by Pandas after 2.0.3. ## PyKX 2.5.4 -#### Release Date +**Release Date** 2024-10-22 @@ -2005,7 +2539,7 @@ PyKX 2.5.4 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. -### Fixes and Improvements +**Fixes and Improvements** - Resolved context interface failing to load files on Windows. @@ -2047,18 +2581,18 @@ ``` -### Version Support Changes +**Version Support Changes** - Version 2.5.4 marks the removal of support for releases to PyPI/Anaconda of Python 3.7 supported versions of PyKX ## PyKX 2.5.2 -#### Release Date +**Release Date** 2024-07-05 -### Fixes and Improvements +**Fixes and Improvements** - Converting PyKX generic lists using the keyword parameter `raw=True` would previously return incorrect results, the values received being the memory address of the individual elements of the list, this has now been resolved @@ -2166,11 +2700,11 @@ ## PyKX 2.5.1 -#### Release Date +**Release Date** 2024-06-11 -### Additions +**Additions** - Addition of methods to `pykx.Table` and `pykx.KeyedTable` objects for the application of `sql`, `select`, `exec`, `delete` and `update` queries @@ -2255,7 +2789,7 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Fix to regression in PyKX 2.5.0 where PyKX initialisation on Windows would result in a segmentation fault when using an `k4.lic` license type. - Previously users could not make direct use of `kx.SymbolicFunction` type objects against a remote process, this has been rectified @@ -2340,11 +2874,11 @@ ## PyKX 2.5.0 -#### Release Date +**Release Date** 2024-05-15 -### Additions +**Additions** - Addition of a method for `pykx.Table` objects to apply `xbar` calculations on specified columns names @@ -2500,7 +3034,7 @@ - Added `as_arrow` keyword to the `.pd()` method on PyKX Wrapped objects, using `as_arrow=True` will use PyArrow backed data types instead of the default NumPy backed data types. -### Fixes and Improvements +**Fixes and Improvements** - When importing PyKX from a source file path containing a space initialisation would fail with an `nyi` error message, this has now been resolved @@ -2803,7 +3337,7 @@ dtype: int64 ``` -### Upgrade considerations +**Upgrade considerations** - Since 2.1.0 when using Pandas >= 2.0 dataframe columns of type `datetime64[s]` converted to `DateVector` under `toq`. Now correctly converts to `TimestampVector`. See [conversion condsideratons](../user-guide/fundamentals/conversion_considerations.md#temporal-data-types) for further details. @@ -2840,27 +3374,27 @@ - Configuration option `PYKX_DISABLE_PANDAS_WARNING` has been removed. - Deprecated `.pd(raw_guids)` keyword. -### Beta Features +**Beta Features** - Addition of [streamlit](https://streamlit.io/) connection class `pykx.streamlit.Connection` to allow querying of q processes when building a streamlit application. For an example of this functionality and an introduction to it's usage see [here](../user-guide/advanced/streamlit.md). ## PyKX 2.4.2 -#### Release Date +**Release Date** 2024-04-03 -### Fixes and Improvements +**Fixes and Improvements** - Updated `libq` to 2024.03.28 for all supported OS's. ## PyKX 2.4.1 -#### Release Date +**Release Date** 2024-03-27 -### Fixes and Improvements +**Fixes and Improvements** - Previously calls to `qsql.select`, `qsql.exec`, `qsql.update` and `qsql.delete` would require multiple calls to parse the content of `where`, `columns` and `by` clauses. These have now been removed with all parsing now completed within the functional query when called via IPC or local to the Python process. - Linux x86 and Mac x86/ARM unlicensed mode `e.o` library updated to 2023.11.22. Fixes subnormals issue: @@ -2893,11 +3427,11 @@ ## PyKX 2.4.0 -#### Release Date +**Release Date** 2024-03-20 -### Additions +**Additions** - Support for q/kdb+ `4.1` documentation [here](https://code.kx.com/q/releases/ChangesIn4.1/) added as an opt-in capability, this functionality is enabled through setting `PYKX_4_1_ENABLED` environment variable. @@ -2986,7 +3520,7 @@ pykx.exceptions.QError: type ``` -### Fixes and Improvements +**Fixes and Improvements** - Added instructions for script to install Windows dependencies. - Resolved segfaults on Windows when PyKX calls Python functions under q. @@ -3013,7 +3547,7 @@ pykx.LongVector(pykx.q('24 28 32 36')) ``` -- Updated kdb Insights Core libraries to 4.0.8, see [here](https://code.kx.com/insights/1.8/core/release-notes/latest.html#408) for more information. +- Updated kdb Insights Core libraries to 4.0.8, see [here](https://code.kx.com/insights/core/release-notes/previous.html#408) for more information. - Updated `libq` 4.0 version to 2024.03.04 for all supported OS's. - Fix issue where use of valid C backed q `code` APIs could result in segmentation faults when called. @@ -3051,9 +3585,9 @@ >>> conn('enlist 0Np').py() Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py + File "/home/user/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py converted_vector[i]=q('0Np') - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ raise LicenseException("run q code via 'pykx.q'") pykx.exceptions.LicenseException: A valid q license must be in a known location (e.g. `$QLIC`) to run q code via 'pykx.q'. ``` @@ -3162,7 +3696,7 @@ >>> tab1.merge(tab2_keyed, how='left', q_join=True) ``` -### Beta Features +**Beta Features** - Addition of `Compress` and `Encrypt` classes to allow users to set global configuration and for usage within Database partition persistence. @@ -3205,22 +3739,22 @@ ## PyKX 2.3.2 -#### Release Date +**Release Date** 2024-02-12 -### Fixes and Improvements +**Fixes and Improvements** - Update of PyKX 4.0 linux shared object to version 2024.02.09, this update is to facilitate deployments on more secure linux/linux-arm environments. - Update `Table.rename()` to skip over columns not in table instead of throwing error to match `pandas`. ## PyKX 2.3.1 -#### Release Date +**Release Date** 2024-02-07 -### Fixes and Improvements +**Fixes and Improvements** - Python functions saved to q would error if passed `''` or `'.'`. These now pass without issue. @@ -3234,7 +3768,7 @@ >>> kx.q('func', '') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -3242,7 +3776,7 @@ >>> kx.q('func', '.') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -3311,7 +3845,7 @@ - `df.select_dtypes()` updated to now accept `kx.*Atom` values for `include`/`exclude` params. Use of `kx.CharVector` will return error. - To align with other areas of PyKX the `upsert` and `insert` methods for PyKX tables and keyed tables now support the keyword argument `inplace`, this change will deprecate usage of `replace_self` with the next major release of PyKX. -### Beta Features +**Beta Features** - Addition of the concept of `Remote Function` execution to PyKX, this allows users, from a Python session to define Python functions which will be executed on a remote q/kdb+ server running PyKX under q. The intention with this feature is to allow onboarding of Python first operations within existing or q/kdb+ first infrastructures @@ -3329,11 +3863,11 @@ ## PyKX 2.3.0 -#### Release Date +**Release Date** 2024-01-22 -### Additions +**Additions** - PyKX now supports the use of `KDB_LICENSE_B64` or `KDB_K4LICENSE_B64` configuration values/environment variables to define the content of a `kc.lic` or `k4.lic` license respectively if no license is found on initial usage of PyKX. - Shortcut provided for access to current date, time and timestamp information using `'today'` and `'now'`. @@ -3392,7 +3926,7 @@ - `%%q` Jupyter Notebook magic adds `--debug` option which prints the q backtrace if the cell execution fails. - Release 2.3.0 adds to PyKX the concept of Beta features, these features are available to users through setting the configuration/environment variable `PYKX_BETA_FEATURES`. For more information on Beta features see further documentation [here](../beta-features/index.md) -### Fixes and Improvements +**Fixes and Improvements** - `%%q` Jupyter Notebook magic now returns all outputs up to and including an error when thrown. Previously only the error was returned. - `%%q` Jupyter Notebook magic ignores accidental whitespace in execution options. Below example no longer fails with `Received unknown argument` error: @@ -3424,7 +3958,7 @@ - Improved error message for conversion failures. - Fixes an issue where a user would receive a length error when attempting to apply `min`, `max`, `prod` and `sum` functions on `pykx.KeyedTable` objects. -### Beta Features +**Beta Features** - Database Management functionality has been added for the creation, loading and maintenance of PyKX Partitioned Databases. A full worked example of this functionality can be found [here](../examples/db-management.ipynb) along with full API documentation which includes examples of each function [here](../api/db.md). The API includes but is not limited to the following: @@ -3442,11 +3976,11 @@ ## PyKX 2.2.3 -#### Release Date +**Release Date** 2024-01-11 -### Fixes and Improvements +**Fixes and Improvements** - PyKX now raises an error appropriately when failing to locate `msvcr100.dll` when loading on Windows. - Config values now default to `False` when not set rather than `None`. @@ -3464,11 +3998,11 @@ Please skip this release and use 2.2.3 or newer. This is due to potential segfaults when reading compressed files. -#### Release Date +**Release Date** 2023-12-12 -### Fixes and Improvements +**Fixes and Improvements** - Conversions between `UUID` and `pykx.GUID` types could produce invalid results under various conditions in both licensed and unlicensed mode. - A regression in 2.2.1 resulted in `SIGINT` signals being incorrectly treated as `SIGTERM` style signals, PyKX now resets all signals overwritten by PyKX to their values prior to import. @@ -3480,11 +4014,11 @@ Please skip this release and use 2.2.3 or newer. This is due to potential segfaults when reading compressed files. -#### Release Date +**Release Date** 2023-11-30 -### Fixes and Improvements +**Fixes and Improvements** - Some messages to `stdout` were not being captured when redirecting. Now all are captured. - Deprecation of internally used environment variable `UNDER_PYTHON` which has been replaced by `PYKX_UNDER_PYTHON` to align with other internally used environment variables. @@ -3621,17 +4155,17 @@ ``` -### Upgrade considerations +**Upgrade considerations** - If dependent on the environment variable `UNDER_PYTHON` please upgrade your code to use `PYKX_UNDER_PYTHON` ## PyKX 2.2.0 -#### Release Date +**Release Date** 2023-11-09 -### Additions +**Additions** - Addition of `agg` method for application of aggregation functions on `pykx.Table` and `pykx.GroupbyTable` objects @@ -3847,7 +4381,7 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Fix to allow users to use Python functions when operating on a `pykx.GroupbyTable` with an `apply` function @@ -3941,11 +4475,11 @@ ## PyKX 2.1.2 -#### Release Date +**Release Date** 2023-10-24 -### Fixes and Improvements +**Fixes and Improvements** - Fix to issue where functions retrieved using the Context Interface with names `update/delete/select/exec` would result in an `AttributeError` @@ -3975,11 +4509,11 @@ ## PyKX 2.1.1 -#### Release Date +**Release Date** 2023-10-10 -### Fixes and Improvements +**Fixes and Improvements** - Fix to regression in PyKX 2.1.0 where execution of `from pykx import *` would result in the following behavior @@ -3991,11 +4525,11 @@ ## PyKX 2.1.0 -#### Release Date +**Release Date** 2023-10-09 -### Additions +**Additions** - Added functionality to the CSV Reader to allow for the input of data structures while defining column types. For example, the following reads a CSV file and specifies the types of the three columns named `x1`, `x2` and `x3` to be of type `Integer`, `GUID` and `Timestamp`. @@ -4052,7 +4586,7 @@ the following reads a CSV file and specifies the types of the three columns name the expected type a warning will be raised and the `DateVector` type will be used as a default. -### Fixes and Improvements +**Fixes and Improvements** - Empty PyKX keyed tables can now be converted to Pandas DataFrames, previously this would raise a `ValueError` @@ -4123,17 +4657,17 @@ the following reads a CSV file and specifies the types of the three columns name - Users with [`QINIT`](https://code.kx.com/q/basics/by-topic/#environment) environment variable would previously load twice on initialization within PyKX - Users installing PyKX under q on Windows had been missing installation of required files using `pykx.install_into_QHOME()` -### Dependency Updates +**Dependency Updates** - The version of `Cython` used to build `PyKX` was updated to the full `3.0.x` release version. ## PyKX 2.0.1 -#### Release Date +**Release Date** 2023-09-21 -### Fixes and Improvements +**Fixes and Improvements** - User input based license initialization introduced in 2.0.0 no longer expects user input when operating in a non-interactive modality, use of PyKX in this mode will revert to previous behavior - Use of the environment variables `QARGS='--unlicensed'` or `QARGS='--licensed'` operate correctly following regression in 2.0.0 @@ -4141,7 +4675,7 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 2.0.0 -#### Release Date +**Release Date** 2023-09-18 @@ -4149,7 +4683,7 @@ the following reads a CSV file and specifies the types of the three columns name - Pandas API functionality is enabled permanently which will modify data indexing and retrieval of `pykx.Table` objects. Users should ensure to review and test their codebase before upgrading. - EmbedPy replacement functionality for PyKX under q is now non-beta for Linux and MacOS installations, see [here](underq-changelog.md) for full information on 2.0.0 changelog. -### Additions +**Additions** - [Pandas API](../user-guide/advanced/Pandas_API.ipynb) is enabled by default allowing users to treat PyKX Tables similarly to Pandas Dataframes for a limited subset of Pandas like functionality. As a result of this change the environment variable `PYKX_ENABLE_PANDAS_API` is no longer required. - Addition of file based configuration setting allowing users to define profiles for various PyKX modalities through definition of the file `.pykx.config` see [here](../user-guide/configuration.md) for more information. @@ -4216,7 +4750,7 @@ the following reads a CSV file and specifies the types of the three columns name ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Update to environment variable definitions in all cases to be prefixed with `PYKX_*` - Return of Pandas API functions `dtypes`, `columns`, `empty`, `ndim`, `size` and `shape` return `kx` objects rather than Pythonic objects @@ -4287,21 +4821,21 @@ the following reads a CSV file and specifies the types of the three columns name pykx.q('1b') ``` -### Breaking Changes +**Breaking Changes** - Pandas API functionality is enabled permanently which will modify data indexing and retrieval. Users should ensure to review and test their codebase before upgrading. ## PyKX 1.6.3 -#### Release Date +**Release Date** 2023-08-18 -### Additions +**Additions** - Addition of argument `return_info` to `pykx.util.debug_environment` allowing user to optionally return the result as a `str` rather than to stdout -### Fixes and Improvements +**Fixes and Improvements** - Fixed Pandas API use of `ndim` functionality which should return `2` when interacting with tables following the expected Pandas behavior. - Fixed an error when using the Pandas API to update a column with a `Symbols`, `Characters`, and `Generic Lists`. @@ -4310,17 +4844,17 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.2 -#### Release Date +**Release Date** 2023-08-15 -### Additions +**Additions** - Added `to_local_folder` kwarg to `install_into_QHOME` to enable use of `pykx.q` without write access to `QHOME`. - Added [an example](../examples/threaded_execution/threading.md) that shows how to use `EmbeddedQ` in a multithreaded context where the threads need to modify global state. - Added [PYKX_NO_SIGINT](../user-guide/configuration.md#environment-variables) environment variable. -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue causing a crash when closing `QConnection` instances on Windows. - Updated q 4.0 libraries to 2023.08.11. Note: Mac ARM release remains on 2022.09.30. @@ -4330,18 +4864,18 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.1 -#### Release Date +**Release Date** 2023-07-19 -### Additions +**Additions** - Added `sorted`, `grouped`, `parted`, and `unique`. As methods off of `Tables` and `Vectors`. - Added `PyKXReimport` class to allow subprocesses to reimport `PyKX` safely. - Also includes `.pykx.safeReimport` in `pykx.q` to allows this behavior when running under q as well. - Added environment variables to specify a path to `libpython` in the case `pykx.q` cannot find it. -### Fixes and Improvements +**Fixes and Improvements** - Fixed memory leaks within the various `QConnection` subclasses. - Added deprecation warning around the discontinuing of support for Python 3.7. @@ -4355,11 +4889,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.0 -#### Release Date +**Release Date** 2023-06-16 -### Additions +**Additions** - Added `merge_asof` to the Pandas like API. - See [here](../user-guide/advanced/Pandas_API.ipynb#tablemerge_asof) for details of supported keyword arguments and limitations. @@ -4374,7 +4908,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added the ability to use `kx.RawQConnection` as a Python based `q` server using `kx.RawQConnection(port=x, as_server=True)`. - More documentation around using this functionality can be found [here](../examples/server/server.md). -### Fixes and Improvements +**Fixes and Improvements** - Improved error on Windows if `msvcr100.dll` is not found - Updated q libraries to 2023.04.17 @@ -4407,50 +4941,50 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.5.3 -#### Release Date +**Release Date** 2023-05-18 -### Additions +**Additions** - Added support for Pandas `Float64Index`. - Wheels for ARM64 based Macs are now available for download. ## PyKX 1.5.2 -#### Release Date +**Release Date** 2023-04-30 -### Additions +**Additions** - Added support for ARM 64 Linux. ## PyKX 1.5.1 -#### Release Date +**Release Date** 2023-04-28 -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue with `pykx.q` that caused errors to not be raised properly under q. - Fixed an issue when using `.pykx.get` and `.pykx.getattr` that caused multiple calls to be made. ## PyKX 1.5.0 -#### Release Date +**Release Date** 2023-04-17 -### Additions +**Additions** - Added wrappers around various `q` [system commands](https://code.kx.com/q/basics/syscmds/). - Added `merge` method to tables when using the `Pandas API`. - Added `mean`/`median`/`mode` functions to tables when using the `Pandas API`. - Added various functions around type conversions on tables when using the `Pandas API`. -### Fixes and Improvements +**Fixes and Improvements** - Fix to allow GUIDs to be sent over IPC. - Fix an issue related to IPC connection using compression. @@ -4464,21 +4998,21 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.4.2 -#### Release Date +**Release Date** 2023-03-08 -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue that would cause `EmbeddedQ` to fail to load. ## PyKX 1.4.1 -#### Release Date +**Release Date** 2023-03-06 -### Fixes and Improvements +**Fixes and Improvements** - Added constructors for `Table` and `KeyedTable` objects to allow creation of these objects from dictionaries and list like objects. - Fixed a memory leak around calling wrapped `Foreign` objects in `pykx.q`. @@ -4486,11 +5020,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.4.0 -#### Release Date +**Release Date** 2023-01-23 -### Additions +**Additions** - Addition of a utility function `kx.ssl_info()` to retrieve the SSL configuration when running in unlicensed mode (returns the same info as kx.q('-26!0') with a license). - Addition of a utility function `kx.schema.builder` to allow for the generation of `pykx.Table` and `pykx.KeyedTable` types with a defined schema and zero rows, this provides an alternative to writing q code to create an empty table. @@ -4503,7 +5037,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added environment variable `PYKX_Q_LOCK=1` to enable a Mutex Lock around calls into q, setting this environment variable to a number greater than 0 will set the max length in time to block before raising an error, a value of '-1' will block indefinitely and will not error, any other value will cause an error to be raised immediately if the lock cannot be acquired. - Added `insert` and `upsert` methods to `Table` and `KeyedTable` objects. -### Fixes and Improvements +**Fixes and Improvements** - Fixed `has_nulls` and `has_infs` properties for subclasses of `k.Collection`. - Improved error output of `kx.QConnection` objects when an error is raised within the context interface. @@ -4513,28 +5047,28 @@ the following reads a CSV file and specifies the types of the three columns name - Fixed conversion of null `GUIDAtom`'s to and from Numpy types. - Improved performance of converting `q` enums to pandas Categoricals. -### Beta Features +**Beta Features** - Added support for a Pandas like API around `Table` and `KeyedTable` instances, documentation for the specific functionality can be found [here](../user-guide/advanced/Pandas_API.ipynb). - Added `.pykx.setdefault` to `pykx.q` which allows the default conversion type to be set without using environment variables. ## PyKX 1.3.2 -#### Release Date +**Release Date** 2023-01-06 -### Features and Fixes +**Features and Fixes** - Fixed support for using TLS with `SyncQConnection` instances. ## PyKX 1.3.1 -#### Release Date +**Release Date** 2022-11-16 -### Features and Fixes +**Features and Fixes** - Added environment variable `PYKX_Q_LIB_LOCATION` to specify a path to load the PyKX q libraries from. - Required files in this directory @@ -4553,11 +5087,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.3.0 -#### Release Date +**Release Date** 2022-10-20 -### Features and Fixes +**Features and Fixes** - Support for converting `datetime.datetime` objects with time zone information into `pykx.TimestampAtom`s and `pykx.TimestampVector`s. - Added a magic command to run cells of q code in a Jupyter Notebook. The addition of `%%q` at the start of a Jupyter Notebook cell will allow a user to execute q code locally similarly to loading a q file. @@ -4566,38 +5100,38 @@ the following reads a CSV file and specifies the types of the three columns name - Fix to memory leak seen when converting Pandas Dataframes to q tables. - Removed unnecessary copy when sending `q` objects over IPC. -### Beta Features +**Beta Features** - EmbedPy replacement functionality `pykx.q` updated significantly to provide parity with embedPy from a syntax perspective. Documentation of the interface [here](../pykx-under-q/intro.md) provides API usage. Note that initialization requires the first version of Python to be retrieved on a users `PATH` to have PyKX installed. Additional flexibility with respect to installation location is expected in `1.4.0` please provide any feedback to `pykx@kx.com` ## PyKX 1.2.2 -#### Release Date +**Release Date** 2022-10-01 -### Features and Fixes +**Features and Fixes** - Fixed an issue causing the timeout argument for `QConnection` instances to not work properly. ## PyKX 1.2.1 -#### Release Date +**Release Date** 2022-09-27 -### Features and Fixes +**Features and Fixes** - Added support for OpenSSLv3 for IPC connections created when in 'licensed' mode. - Updated conversion functionality for timestamps to support conversions within Pandas 1.5.0 ## PyKX 1.2.0 -#### Release Date +**Release Date** 2022-09-01 -### Features and Fixes +**Features and Fixes** - Support for converting any python type to a `q` Foreign object has been added. - Support for converting Pandas categorical types into `pykx.EnumVector` type objects. @@ -4613,7 +5147,7 @@ the following reads a CSV file and specifies the types of the three columns name - Fixed conversion of `None` Python objects into analogous null `K` types if a `ktype` is specified. - Added `event_loop` parameter to `pykx.AsyncQConnection` that takes a running event loop as a parameter and allows the event loop to manage `pykx.QFuture` objects. -### Beta Features +**Beta Features** - Added extra functionality to `pykx.q` related to the calling and use of python foreign objects directly within a `q` process. - Support for [NEP-49](https://numpy.org/neps/nep-0049.html), which allows Numpy arrays to be converted into `q` Vectors without copying the underlying data. This behavior is opt-in and you can do so by setting the environment variable `PYKX_ALLOCATOR` to 1, "1" or True or by adding the flag `--pykxalloc` to the `QARGS` environment variable. Note: This feature also requires a python version of at least 3.8. @@ -4621,25 +5155,25 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.1.1 -#### Release Date +**Release Date** 2022-06-13 -### Features & Fixes +**Features & Fixes** - Added ability to skip symlinking `$QHOME` to `PyKX`'s local `$QHOME` by setting the environment variable `IGNORE_QHOME`. ## PyKX 1.1.0 -#### Release Date +**Release Date** 2022-06-07 -### Dependencies +**Dependencies** - The dependency on the system library `libcurl` has been made optional for Linux. If it is missing on Linux, a warning will be emitted instead of an error being raised, and the KX Insights Core library `kurl` will not be fully loaded. Windows and macOS are unaffected, as they don't support the KX Insights Core features to begin with. -### Features & Fixes +**Features & Fixes** - Splayed and partitioned tables no longer emit warnings when instantiated. - Added `pykx.Q.sql`, which is a wrapper around [KXI Core SQL](https://code.kx.com/insights/core/sql.html#sql-language-support). @@ -4666,15 +5200,15 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.0.1 -#### Release Date +**Release Date** 2022-03-18 -### Deprecations & Removals +**Deprecations & Removals** - The `sync` parameter for `pykx.QConnection` and `pykx.QConnection.__call__` has been renamed to the less confusing name `wait`. The `sync` parameter remains, but its usage will result in a `DeprecationWarning` being emitted. The `sync` parameter will be removed in a future version. -### Features & Fixes +**Features & Fixes** - Updated to stable classifier (`Development Status :: 5 - Production/Stable`) in project metadata. Despite this update being done in version 1.0.1, version 1.0.0 is still the first stable release of PyKX. - PyKX now provides source distributions (`sdist`). It can be downloaded from PyPI using `pip download --no-binary=:all: --no-deps pykx`. As noted in [the installation docs](../getting-started/installing.md#supported-environments), installations built from the source will only receive support on a best-effort basis. - Fixed Pandas NaT conversion to q types. Now `pykx.toq(pandas.NaT, ktype=ktype)` produces a null temporal atom for any given `ktype` (e.g. `pykx.TimeAtom`). @@ -4689,11 +5223,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.0.0 -#### Release Date +**Release Date** 2022-02-14 -### Migration Notes +**Migration Notes** To switch from Pykdb to PyKX, you will need to update the name of the dependency from `pykdb` to `pykx` in your `pyproject.toml`/`requirements.txt`/`setup.cfg`/etc. When Pykdb was renamed to PyKX, its version number was reset. The first public release of PyKX has the version number 1.0.0, and will employ [semantic versioning](https://semver.org/). @@ -4727,7 +5261,7 @@ kx.QARGS # List of arguments provided to the embedded q instance at startup, set You can no longer rely on the [context](../api/pykx-execution/ctx.md) being reset to the global context after each call into embedded q, however IPC calls are unaffected. -### Renames +**Renames** - Pykdb has been renamed to PyKX. `Pykdb` -> `PyKX`; `PYKDB` -> `PYKX`; `pykdb` -> `pykx`. - The `adapt` module has been renamed to `toq`, and it can be called directly. Instead of `pykdb.adapt.adapt(x)` one should write `pykx.toq(x)`. - The `k` module has been renamed to `wrappers`. All wrapper classes can be accessed from the top-level, i.e. `pykx.K`, `pykx.SymbolAtom`, etc. @@ -4737,18 +5271,18 @@ You can no longer rely on the [context](../api/pykx-execution/ctx.md) being rese - The `Connection` class in the IPC module has been renamed to `QConnection`, and is now available at the top-level, i.e. `pykx.QConnection`. - The q type wrapper `DynamicLoad` has been renamed to `Foreign` (`pykdb.k.DynamicLoad` -> `pykx.Foreign`). -### Deprecations & Removals +**Deprecations & Removals** - The `pykdb.q.ipc` attribute has been removed. The IPC module can be accessed directly instead at `pykx.ipc`, but generally one will only need to access the `QConnection` class, which can be accessed at the top-level: `pykx.QConnection`. - The `pykdb.q.K` attribute has been removed. Instead, `K` types can be used as constructors for that type by leveraging the `toq` module. For example, instead of `pykdb.q.K(x)` one should write `pykx.K(x)`. Instead of `pykx.q.K(x, k_type=pykx.k.SymbolAtom)` one should write `pykx.SymbolAtom(x)` or `pykx.toq(x, ktype=pykx.SymbolAtom)`. - Most `KdbError`/`QError` subclasses have been removed, as identifying them is error prone, and we are unable to provide helpful error messages for most of them. - The `pykx.kdb` singleton class has been removed. -### Dependencies +**Dependencies** - More Numpy, Pandas, and PyArrow versions are supported. Current `pandas~=1.0`, `numpy~=1.20,<1.22`, and `pyarrow>=3.0.0` are supported. PyArrow remains an optional dependency. - A dependency on `find-libpython~=0.2` was added. This is only used when running PyKX under a q process (see details in the section below about new alpha features). - A dependency on the system library `libcurl` was added for Linux. This dependency will be made optional in a future release. -### Features & Fixes +**Features & Fixes** - The `pykx.Q` class has been added as the base class for `pykx.EmbeddedQ` (the class for `pykx.q`) and `pykx.QConnection`. - The `pykx.EmbeddedQ` process now persists its [context](../api/pykx-execution/ctx.md) between calls. - The console now works over IPC. @@ -4773,13 +5307,13 @@ You can no longer rely on the [context](../api/pykx-execution/ctx.md) being rese - Misc other bug fixes. - Misc doc improvements. -### Performance Improvements +**Performance Improvements** - Converting nested lists from q to Python is much faster. - Internally, PyKX now calls q functions with arguments directly instead of creating a `pykx.Function` instance then calling it. This results in modest performance benefits in some cases. - The context interface no longer loads every element of a context when the context is first accessed, thereby removing the computation spike, which could be particularly intense for large q contexts. -### New Alpha Features +**New Alpha Features** !!! danger "Alpha features are subject to change" diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md index 268120d..e5a0bf6 100644 --- a/docs/release-notes/deprecations.md +++ b/docs/release-notes/deprecations.md @@ -1,9 +1,10 @@ # Deprecations -A list of deprecated behaviors and the version in which they were removed. +A list of deprecated behaviors and the version in which they were removed. For full release notes see [here](changelog.md), and for under q see [here](underq-changelog.md). | Feature | Alternative | Deprecated | Removed | |---------------------------------------------------|---------------------------|---------------|------------| +| `value` keyword for `Column` creation | `data` | 3.1.7 | | | Creating more than one DB object | `overwrite=True` | 3.1.5 | | | `kx.q.system.console_size` | `kx.q.system.display_size`| 3.1.3 | | | `.pykx.console[]` on Windows | | 3.1.3 | 3.1.3 | @@ -21,9 +22,11 @@ A list of deprecated behaviors and the version in which they were removed. | `PYKX_ENABLE_PANDAS_API` | | 3.0.0 | 3.0.0 | | `.pd(raw_guids)` | | 2.5.0 | 2.5.0 | -## PyKX 3.1.5 +## PyKX 3.1.7 + +- Deprecated `value` keyword when creating `Column` objects, use `data` instead -Release Date: 2025-10-21 +## PyKX 3.1.5 ### Fixes and Improvements @@ -56,14 +59,10 @@ Release Date: 2025-02-11 ## PyKX 3.0.1 -Release Date: 2024-12-04 - - Removal of column `type` from the return of `#!python dtypes` method for `#!python kx.Table` objects, previously this had raised a deprecation warning ## PyKX 3.0.0 -Release Date: 2024-11-12 - - Removal of various deprecated keywords used in table operations: - `#!python modify` keyword for `#!python select`, `#!python exec`, `#!python update` and `#!python delete` operations on `#!python pykx.Table` and `#!python pykx.KeyedTable`. This has been permanently changed to use `#!python inplace`. - `#!python replace_self` keyword when attempting to overwrite a `#!python pykx.Table` or `#!python KeyedTable` using insert/upsert functionality. This has been permanently changed to use `#!python inplace`. @@ -84,8 +83,6 @@ Release Date: 2024-11-12 ## PyKX 2.5.0 -Release Date: 2024-05-15 - - Deprecated `.pd(raw_guids)` keyword. - Renamed `labels` parameter in `Table.rename()` to `mapper` to match Pandas. Added deprecation warning to `labels`. - Deprecation of `type` column in `dtypes` output as it is a reserved keyword. Use new `datatypes` column instead. @@ -93,36 +90,26 @@ Release Date: 2024-05-15 ## PyKX 2.3.1 -Release Date: 2024-02-07 - - To align with other areas of PyKX the `upsert` and `insert` methods for PyKX tables and keyed tables now support the keyword argument `inplace`, this change will deprecate usage of `replace_self` with the next major release of PyKX. ## PyKX 2.2.1 -Release Date: 2023-11-30 - - Deprecation of internally used environment variable `UNDER_PYTHON` which has been replaced by `PYKX_UNDER_PYTHON` to align with other internally used environment variables. - Addition of deprecation warning for environmental configuration option `PYKX_NO_SIGINT` which is to be replaced by `PYKX_NO_SIGNAL`. This is used when users require no signal handling logic overwrites and now covers `SIGTERM`, `SIGINT`, `SIGABRT` signals amongst others. ## PyKX 1.6.1 -Release Date: 2023-07-19 - - Added deprecation warning around the discontinuing of support for Python 3.7. ## PyKX 1.0.1 -Release Date: 2022-03-18 - - The `sync` parameter for `pykx.QConnection` and `pykx.QConnection.__call__` has been renamed to the less confusing name `wait`. The `sync` parameter remains, but its usage will result in a `DeprecationWarning` being emitted. The `sync` parameter will be removed in a future version. ## PyKX 1.0.0 -Release Date: 2022-02-14 - - The `pykdb.q.ipc` attribute has been removed. The IPC module can be accessed directly instead at `pykx.ipc`, but generally one will only need to access the `QConnection` class, which can be accessed at the top-level: `pykx.QConnection`. - The `pykdb.q.K` attribute has been removed. Instead, `K` types can be used as constructors for that type by leveraging the `toq` module. For example, instead of `pykdb.q.K(x)` one should write `pykx.K(x)`. Instead of `pykx.q.K(x, k_type=pykx.k.SymbolAtom)` one should write `pykx.SymbolAtom(x)` or `pykx.toq(x, ktype=pykx.SymbolAtom)`. - Most `KdbError`/`QError` subclasses have been removed, as identifying them is error prone, and we are unable to provide helpful error messages for most of them. diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index e5b3e51..144b45c 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -1,28 +1,38 @@ # PyKX under q Changelog -This changelog provides updates from PyKX 2.0.0 and above, for information relating to versions of PyKX prior to this version see the changelog linked below. +This changelog covers PyKX under q from version 2.0.0 onwards. !!! Note - The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). + For changes to PyKX within a Python environment, refer to the [Python changelog](./changelog.md). + +## PyKX 3.2.0 + +**Release Date** + +2026-06-24 + +**Fixes and Improvements** + +- Fixed a memory leak when using `.pykx.getattr` on foreign object in `q`. ## PyKX 3.1.5 -#### Release Date +**Release Date** 2025-10-21 -### Fixes and Improvements +**Fixes and Improvements** - Added `.pykx.noalloc` which can be used to wrap a foreign object. When used with `.pykx.toq` the conversion will not use the `PYKX_ALLOCATOR`. See [here](../help/issues.md#known-issues) for details. ## PyKX 3.1.4 -#### Release Date +**Release Date** 2025-07-17 -### Fixes and Improvements +**Fixes and Improvements** - Resolved `object has no attribute 't'` error for certain conversions @@ -49,11 +59,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.1.3 -#### Release Date +**Release Date** 2025-06-12 -### Fixes and Improvements +**Fixes and Improvements** - Dashboards integration now lists missing libraries by name. @@ -78,17 +88,17 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat - Resolved `double free or corruption (out)` error when `pykx.q` was loaded in `QINIT` or `q.q`. - `pykx.q` load time has been roughly halved. -### Deprecations & Removals +**Deprecations & Removals** - `.pykx.console[]` has been removed on Windows due to incompatibility. Will now error with `'.pykx.console is not available on Windows` if called. ## PyKX 3.1.1 -#### Release Date +**Release Date** 2025-02-14 -### Fixes and Improvements +**Fixes and Improvements** - Fix throwing of errors for `.pykx.safeReimport`, now throws an error instead of returning a value. @@ -116,11 +126,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.1.0 -#### Release Date +**Release Date** 2025-02-11 -### Additions +**Additions** - Addition of `.pykx.typepy` which returns an objects datatype as a `CharVector` after being passed to python. @@ -133,7 +143,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat "" ``` -### Fixes and Improvements +**Fixes and Improvements** - Using `.pykx.toq`/`.pykx.toq0` now return the q representation of an object when passed a wrapped type conversion object @@ -189,7 +199,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat - When attempting to load PyKX after embedPy has already been loaded, an error will be thrown and PyKX will not continue to load. -### Beta Features +**Beta Features** - Added ability for users to convert between PyKX numeric vectors or N-Dimensional Lists and PyTorch Tensor objects using the `.pykx.topt` function. @@ -200,11 +210,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.0.1 -#### Release Date +**Release Date** 2024-12-04 -### Additions +**Additions** - Added a function `.pykx.loadPy` which can be used to execute/load a Python file following expected Python syntax, this removes limitations which exist due to use of q parsing syntax when loading a Python file using `system"l /path/to/file.p"` for example @@ -222,16 +232,16 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.0.0 -#### Release Date +**Release Date** 2024-11-12 -### Additions +**Additions** - Added `cloud_libraries` kwarg to `install_into_QHOME` allowing installation of the kdb Insights cloud libraries to QHOME. - Addition of support for new environment variable `PYKX_USE_FIND_LIBPYTHON` which will use the Python package [`find_libpython`](https://pypi.org/project/find-libpython/) to specify the location from which `libpython.[so|dll]` will be taken. -### Fixes and Improvements +**Fixes and Improvements** - Addition of function `.pykx.toq0` to support conversion of Python strings to q strings rather than q symbols as is the default behavior @@ -293,21 +303,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.5.4 -#### Release Date +**Release Date** 2024-10-22 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.util.loadfile` now loads a file using it's full path unless it contains a space. This is to avoid issues loading scripts which are sensitive to their working directory. ## PyKX 2.5.3 -#### Release Date +**Release Date** 2024-08-20 -### Fixes and Improvements +**Fixes and Improvements** - Previously PyKX conversions of generic lists (type 0h) would convert this data to it's `raw` representation rather than it's `python` representation as documented. This had the effect of restricting the usability of some types within PyKX under q in non-trivial use-cases. With the `2.5.2` changes to more accurately represent `raw` data at depth this became more obvious as an issue. @@ -329,21 +339,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.5.0 -#### Release Date +**Release Date** 2024-05-15 -### Fixes and Improvements +**Fixes and Improvements** - When loading PyKX under from a source file path containing a space initialisation would fail with an `nyi` error message, this has now been resolved. ## PyKX 2.4.1 -#### Release Date +**Release Date** 2024-03-27 -### Fixes and Improvements +**Fixes and Improvements** - When loading PyKX under q users who had previously loaded [embedPy](https://github.com/KxSystems/embedPy) into their process would cause a segfault of unspecified origin. With this release we have added a warning prior to loading of PyKX which specifies that if a value of `.p.e` has been specified which does not match that expected of PyKX a user should consider installing PyKX under q fully: @@ -359,11 +369,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.3.1 -#### Release Date +**Release Date** 2024-02-07 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.eval` is now variadic allowing an optional second parameter to be passed to define return type. Previously would error with `rank`. @@ -424,11 +434,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.3.0 -#### Release Date +**Release Date** 2024-01-22 -### Fixes and Improvements +**Fixes and Improvements** - A bug was fixed when using `.pykx.console`, it is now possible to access python variables set using the console with `.pykx.(eval|pyexec|pyeval)` functions. @@ -455,21 +465,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.2.2 -#### Release Date +**Release Date** 2023-12-07 -### Fixes and Improvements +**Fixes and Improvements** - When loaded in a q process loading `pykx.q` would not allow `Ctrl+C` (SIGINT) interruption. ## PyKX 2.2.1 -#### Release Date +**Release Date** 2023-11-30 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.print` was using `repr` representation for some objects. Now consistently calls `print`. - `.pykx.safeReimport` now resets environment variables correctly before throwing any error raised by the function supplied to it. @@ -519,15 +529,15 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.2.0 -#### Release Date +**Release Date** 2023-11-09 -### Additions +**Additions** - Addition of `PYKX_EXECUTABLE` environment/configuration variable to allow control of which Python executable is used under q. -### Fixes and Improvements +**Fixes and Improvements** - Failure to access and load PyKX resulting in an `os` error now returns Python backtrace outlining the underlying Python error allowing for easier debugging @@ -578,11 +588,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.1.0 -#### Release Date +**Release Date** 2023-10-09 -### Fixes and Improvements +**Fixes and Improvements** - Update to default conversion logic for q objects passed to PyKX functions to more closely match embedPy based conversion expectations.For version <=2.0 conversions of KX lists would produce N Dimensional Numpy arrays of singular type. This results in issues when applying to many analytic libraries which rely on lists of lists rather than singular N Dimensional arrays. Additionally q tables and keyed tables would be converted to Numpy recarrays, these are now converted to Pandas DataFrames. To maintain previous behavior please set the following environment variable `PYKX_DEFAULT_CONVERSION="np"`. @@ -606,11 +616,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.0.0 -#### Release Date +**Release Date** 2023-09-18 -### Additions +**Additions** - Addition of `.pykx.qcallable` and `.pykx.pycallable` functions which allow wrapping of a foreign Python callable function returning the result as q or Python foreign respectively. - Addition of `.pykx.version` allowing users to programmatically access their version from a q process. @@ -628,7 +638,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat 4 0.691953 0.375638 ``` -### Fixes and Improvements +**Fixes and Improvements** - Application of object setting on a Python list returns generic null rather than wrapped foreign object. - Use of environment variables relating to `PyKX under q` must use `"true"` as accepted value, previously any value set for such environment variables would be supported. diff --git a/docs/scripts/strip-q-prompt-copy.js b/docs/scripts/strip-q-prompt-copy.js new file mode 100644 index 0000000..42e9b9a --- /dev/null +++ b/docs/scripts/strip-q-prompt-copy.js @@ -0,0 +1,150 @@ +// When a user clicks a code copy button, copy the code but remove +// leading REPL prompts (q), >>>, and Python continuation ...). +// +// Add this file to mkdocs extra_javascript as javascript/strip-repl-prompts.js +// and ensure it loads after the Material copy button script. + +(function () { + + // Robust selectors covering Material for MkDocs variations + // (theme versions use slightly different class names) + const COPY_BUTTON_SELECTORS = [ + '.md-code .md-clipboard', // older / generic + 'button.md-code__copy', // theme internal + 'button.md-clipboard', // fallback + '.md-code__copy-button' // some custom builds + ].join(','); + + // Regex to remove leading REPL prompts (allow optional leading whitespace) + // Matches: + // q) + // >>> + // ... + const REPL_PROMPT_RE = /^\s*(q\)|>>>|\.\.\.)\s?/; + + // Helper: find nearest ancestor matching a selector + function closest(el, selector) { + while (el) { + if (el.matches && el.matches(selector)) return el; + el = el.parentElement; + } + return null; + } + + // Clean the code text by removing REPL prompts at the start of each line + function stripReplPrompts(text) { + return text + .split('\n') + .map(line => line.replace(REPL_PROMPT_RE, '')) + .join('\n'); + } + + // Retrieve visible code text from a code block container + function getCodeTextFromBlock(container) { + if (!container) return ''; + + // Typical structure: + //
+ // 
...
+ //
+ const codeElem = + container.querySelector('pre > code') || + container.querySelector('code'); + + if (!codeElem) return ''; + + // Use textContent to preserve visible text only (no markup) + return codeElem.textContent || ''; + } + + // Clipboard write with modern API and legacy fallback + function writeToClipboard(text) { + if (navigator.clipboard && navigator.clipboard.writeText) { + return navigator.clipboard.writeText(text); + } + + // Fallback for older browsers + const textarea = document.createElement('textarea'); + textarea.value = text; + textarea.style.position = 'fixed'; + textarea.style.left = '-9999px'; + document.body.appendChild(textarea); + textarea.select(); + + try { + document.execCommand('copy'); + document.body.removeChild(textarea); + return Promise.resolve(); + } catch (err) { + document.body.removeChild(textarea); + return Promise.reject(err); + } + } + + // Attach copy interception to all copy buttons + function wireCopyButtons() { + document.querySelectorAll(COPY_BUTTON_SELECTORS).forEach(btn => { + + // Prevent duplicate binding if MutationObserver re-runs + if (btn.__replPromptBound) return; + btn.__replPromptBound = true; + + btn.addEventListener('click', function (ev) { + try { + // Locate the containing code block + const codeContainer = + closest(btn, '.md-code') || + closest(btn, 'pre') || + closest(btn, 'code') || + btn.parentElement; + + const raw = getCodeTextFromBlock(codeContainer); + + if (!raw) { + // If no code found, allow default behavior + return; + } + + // Remove REPL prompts before copying + const cleaned = stripReplPrompts(raw); + + // Override default copy behavior + ev.preventDefault(); + ev.stopPropagation(); + + writeToClipboard(cleaned).then(() => { + // Optional: emit event for future UI feedback hooks + btn.dispatchEvent( + new CustomEvent('repl-copy-success', { bubbles: true }) + ); + }).catch(() => { + // On failure, allow theme fallback behavior + }); + + } catch (e) { + // Fail silently and allow default copy behavior + return; + } + + }, { capture: false }); + + }); + } + + // Initial binding once DOM is ready + document.addEventListener('DOMContentLoaded', () => { + wireCopyButtons(); + + // Re-bind if new code blocks are dynamically added + const observer = new MutationObserver(() => { + wireCopyButtons(); + }); + + observer.observe(document.body, { + childList: true, + subtree: true + }); + }); + + })(); + \ No newline at end of file diff --git a/docs/stylesheets/2021-edited.css b/docs/stylesheets/2026-edited.css similarity index 75% rename from docs/stylesheets/2021-edited.css rename to docs/stylesheets/2026-edited.css index 2f7c1ea..cf82b53 100644 --- a/docs/stylesheets/2021-edited.css +++ b/docs/stylesheets/2026-edited.css @@ -145,9 +145,9 @@ /* RHS TOC TITLE & NAV TREE TITLE - Euclid Square Bold UPPERCASE 9/12pt */ .md-nav--lifted>.md-nav__list>.md-nav__item--active>.md-nav__link, .md-nav__title { - font-weight: 300; + font-weight: 600; letter-spacing: 0.2em; - opacity: 0.5; + opacity: 1; padding-bottom: .8em; text-transform: uppercase; } @@ -254,3 +254,111 @@ background-color: transparent; margin-top: 0; } + + [data-md-color-scheme="kx-light"] .md-header { + background-color: #194387; + color: white; + } + + [data-md-color-scheme="kx-light"] .md-tabs { + background-color: #dae6f9; + } + + [data-md-color-scheme="kx-light"] .md-tabs__link { + color: #2b2f38; + } + + [data-md-color-scheme="kx-light"] .md-tabs__link--active { + color: black; + border-bottom: none; + } + [data-md-color-scheme="kx-light"] .md-nav--primary .md-nav__title { + background-color: #194387; + color: white; + } + +/* Tiles */ + +.md-typeset div.large-tile-grid { + display: grid !important; + grid-template-columns: 1fr 1fr !important; + gap: 3rem !important; +} + +@media screen and (max-width: 640px) { + .md-typeset div.large-tile-grid { + grid-template-columns: 1fr; + } +} +.md-typeset div.large-tile-grid > div { + display: flex; + flex-direction: column; + padding: 1.25rem; + border: 1px solid var(--kx-black); + border-radius: 10px; +} +.md-typeset div.large-tile-grid > div h2 { + margin: 0; + font-weight: 500; +} +.md-typeset div.large-tile-grid > div h2 span { + display: block; + margin-top: 0.3rem; + font-size: 0.9rem; + font-weight: normal; +} +.md-typeset div.large-tile-grid > div ul { + flex-grow: 1; + list-style: none; + padding: 0; + margin: 0; +} +.md-typeset div.large-tile-grid > div ul li { + margin: 0.75rem 0; + font-size: 0.8rem; +} +.md-typeset div.large-tile-grid > div a.cta { + display: inline-block; + align-self: flex-start; + margin-top: 0.3rem; + padding: 0.5rem 1rem; + border-radius: 50px; + font-weight: bold; + background-color: var(--kx-blue); + color: var(--kx-off-white); +} +.md-typeset div.large-tile-grid > div a.cta svg { + position: relative; + top: 3px; + display: inline-block; + margin-left: 0.3rem; + transition: all 200ms ease-in-out; + fill: var(--kx-off-white); +} +.md-typeset div.large-tile-grid > div a.cta:hover svg { + transform: translateX(4px); +} + +.md-typeset div.large-tile-grid > div { + display: flex; + flex-direction: column; + padding: 1.25rem; + border: 1px solid #0b0d0e; + border-radius: 10px; +} + +[data-md-color-scheme="slate"] .md-typeset div.large-tile-grid > div { + border-color: #FFC300; + background-color: #0f1114; +} + +[data-md-color-scheme="slate"] { + --md-default-bg-color: #181c1f; +} +.md-nav__link { + user-select: none; +} + +.md-tabs__link { + user-select: none; +} \ No newline at end of file diff --git a/docs/stylesheets/mkdocstrings.css b/docs/stylesheets/mkdocstrings.css index a101a33..3d5a8e9 100644 --- a/docs/stylesheets/mkdocstrings.css +++ b/docs/stylesheets/mkdocstrings.css @@ -23,3 +23,23 @@ h2.doc > code { h3.doc > code { background: var(--md-code-bg-color); } +.md-grid { + max-width: 72rem; + } + + @media screen and (min-width: 76.25em) { + [dir="ltr"] .md-sidebar--primary:not([hidden])~.md-content>.md-content__inner { + margin-left: 0.4rem; + } + + [dir="ltr"] .md-sidebar--secondary:not([hidden])~.md-content>.md-content__inner { + margin-right: 0.4rem; + } + + [dir="rtl"] .md-sidebar--primary:not([hidden])~.md-content>.md-content__inner { + margin-right: 0.4rem; + } + + [dir="rtl"] .md-sidebar--secondary:not([hidden])~.md-content>.md-content__inner { + margin-left: 0.4rem; + } \ No newline at end of file diff --git a/docs/stylesheets/pykx.css b/docs/stylesheets/pykx.css index a726c37..3bc2ec0 100644 --- a/docs/stylesheets/pykx.css +++ b/docs/stylesheets/pykx.css @@ -8,3 +8,4 @@ div.doc-contents:not(.first) { border-left: .05rem solid var(--md-default-fg-color--lightest); margin-bottom: 80px; } + diff --git a/docs/user-guide/advanced/Pandas_API.ipynb b/docs/user-guide/advanced/Pandas_API.ipynb index 7cb4833..9e54ad2 100644 --- a/docs/user-guide/advanced/Pandas_API.ipynb +++ b/docs/user-guide/advanced/Pandas_API.ipynb @@ -2171,7 +2171,7 @@ "| right_on | str | The column name in the right table to join on. | None |\n", "| left_index | bool | Use the index of the left Table. | False |\n", "| right_index | bool | Use the index of the right Table. | False |\n", - "| by | str | Not yet implemented. | None |\n", + "| by | Union[str/list] | Match on these columns before performing merge operation. | None |\n", "| left_by | str | Field names to match on in the left table. | None |\n", "| right_by | str | Field names to match on in the right table. | None |\n", "| suffixes | Tuple(str, str) | The number of rows to return. | ('\\_x', '\\_y') |\n", @@ -2303,6 +2303,16 @@ "trades.merge_asof(quotes, on=\"time\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc094351", + "metadata": {}, + "outputs": [], + "source": [ + "trades.merge_asof(quotes, on=\"time\", by=\"ticker\")" + ] + }, { "cell_type": "markdown", "id": "e6280a9a", diff --git a/docs/user-guide/advanced/license.md b/docs/user-guide/advanced/license.md index 4c5d77c..9c6e503 100644 --- a/docs/user-guide/advanced/license.md +++ b/docs/user-guide/advanced/license.md @@ -17,7 +17,7 @@ In the [install page](../../getting-started/installing.md) you can follow along ## When will your license expire? -For personal edition licenses your license will expire one year after initial download, for enterprise/commercial licenses the expiry date will vary. To provide some forewarning of when your license will expire PyKX provides the following: +To provide some forewarning of when your license will expire PyKX provides the following: - For the 10 days before expiry PyKX will print a message at start up indicating the license will expire in N days - The utility function `#!python kx.license.expires` provides a programmatic method of finding days to expiry. @@ -46,7 +46,7 @@ When your license is expired/expiring you will need to update it to ensure you c There are three methods by which updating your license is possible with PyKX. - You have allowed your license to expire and on restart of PyKX you will be presented with an new license install walkthrough similar to [installing](../../getting-started/installing.md). -- You pre-emptively install a newly downloaded license using `#!python kx.license.install`. +- You preemptively install a newly downloaded license using `#!python kx.license.install`. === "After Expiry" @@ -90,9 +90,9 @@ There are three methods by which updating your license is possible with PyKX. ... force=True) ``` -## Upgrade to a commercial license +## Upgrade to a `k4.lic` from a `kc.lic` license -If you are currently using a `kc.lic` personal license and need to upgrade to a `k4.lic` license the following steps allow you to ensure this can be done effectively. +If you are currently using a `kc.lic` license and need to change to a `k4.lic` license the following steps allow you to ensure this can be done effectively. 1. Delete your existing `kc.lic` license. 2. Install your new license following the [license installation](../../getting-started/installing.md) diff --git a/docs/user-guide/advanced/modes.md b/docs/user-guide/advanced/modes.md index fafa356..b1a5b14 100644 --- a/docs/user-guide/advanced/modes.md +++ b/docs/user-guide/advanced/modes.md @@ -8,21 +8,21 @@ tags: PyKX, q language, Python, licensed, unlincensed, # Modes of operation -_This page explains how to operate PyKX in Python and q, with or without a KDB Insights license._ +_This page explains how to operate PyKX in Python and q, with or without a license._ PyKX can operate in different modes. Each mode has its limitations and requirements, so we're breaking them down into the following: 1. PyKX within Python - 1.a Unlicensed - 1.b Licensed -2. PyKX within q with a kdb Insights license +2. PyKX within q - licensed ## 1. PyKX within Python The purpose of operating PyKX within a Python session is to replace [qPython](https://github.com/exxeleron/qPython) and [PyQ](https://github.com/kxsystems/pyq). Within Python, PyKX has two modes of operation: -- `#!python Licensed` (this means you have a kdb Insights license with PyKX enabled) -- `#!python Unlicensed` (this means you don't have a kdb Insights license or a license in which PyKX is not enabled) +- `#!python Licensed` (this means you have a license with PyKX enabled) +- `#!python Unlicensed` (this means you don't have a license or a license in which PyKX is not enabled) The main difference between the two is that the `#!python Unlicensed` mode is for IPC-based communication. This mean that it allows to create IPC connections and convert data from Pythonic representations to PyKX objects. @@ -55,9 +55,9 @@ This mode cannot run q embedded within it. Also, it lacks the ability to run q c ### 1.b Running in Licensed mode -Licensed mode is the standard way to operate PyKX, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#2-install-a-kdb-insights-license). This modality aims to replace PyQ as the Python-first library for KX. All PyKX features are available in this mode. +Licensed mode is the standard way to operate PyKX, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#2-install-a-license). This modality aims to replace PyQ as the Python-first library for KX. All PyKX features are available in this mode. -The differences provided through operating with a valid kdb Insights license are: +The differences provided through operating with a valid license are: 1. You can execute PyKX/q functionalities directly within a Python session. 2. PyKX objects can be represented in a human readable format rather than as a memory address, namely: diff --git a/docs/user-guide/advanced/remote-functions.md b/docs/user-guide/advanced/remote-functions.md index 4fe83dc..e2c300e 100644 --- a/docs/user-guide/advanced/remote-functions.md +++ b/docs/user-guide/advanced/remote-functions.md @@ -35,11 +35,11 @@ This walkthrough demonstrates the following steps: 1. Import PyKX and generate a remote session object which denotes the process against which the Python functions will be executed. 1. Define a number of Python functions which will be executed on the remote q/kdb+ server. -### Initializea q/kdb+ server with PyKX under q +### Initialize a q/kdb+ server with PyKX under q This step ensures you have a q process running with PyKX under q, as well as having a kdb+ table available to query. If you have this already, proceed to the next step. -Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/), retrieving your license following the instructions provided [here](https://kx.com/kdb-insights-sdk-personal-edition-download). +Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/). Install PyKX under q using the following command. diff --git a/docs/user-guide/advanced/threading.md b/docs/user-guide/advanced/threading.md index 2d53e04..3201f02 100644 --- a/docs/user-guide/advanced/threading.md +++ b/docs/user-guide/advanced/threading.md @@ -27,6 +27,7 @@ All calls to q from other threads are run on this background thread, created usi Before globally enabling this functionality, consider the following: +- **Platform Support**: Multithreading mode is only supported on Linux and macOS; it is not available on Windows. - **Concurrency Cost**: While the overhead for offloading calls onto a secondary thread is low, there will always be a cost in forcing a thread context switch. As such single-threaded performance is faster at the cost of concurrency. - **Memory-Safe Use**: While using `#!python PYKX_THREADING` it's not possible nor memory safe to have `#!python q` call back into Python; this could result in memory corruption or side-effects which may not be immediately obvious. - **Shutdown**: When using `#!python PYKX_THREADING`, it creates a background thread for running queries to `#!python q`. Make sure to call `#!python kx.shutdown_thread()` at the end of your script to properly close this thread. If you don’t, the thread will remain running in the background after the script finishes. To avoid this, it’s best to start your `#!python main` function within a `#!python try` - `#!python finally` block. diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 567d6cd..dfcd9e8 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -65,7 +65,7 @@ To add values to your configuration file you can modify the file directly or alt >>> import pykx as kx >>> kx.util.add_to_config({'PYKX_GC': 'True', 'PYKX_BETA_FEATURES': 'True'}) -Configuration updated at: /Users/conormccarthy/.pykx-config. +Configuration updated at: /Users/user/.pykx-config. Profile updated: default. Successfully added: - PYKX_GC = True diff --git a/docs/user-guide/fundamentals/query/perf.md b/docs/user-guide/fundamentals/query/perf.md index 688a0e6..16ad99e 100644 --- a/docs/user-guide/fundamentals/query/perf.md +++ b/docs/user-guide/fundamentals/query/perf.md @@ -81,7 +81,7 @@ The following shows the scaling of queries based on the number of rows on an in- ## SQL -To optimize frequently called SQL queries you can make use of the [prepare](../../../api/query.md#pykx.query.SQL.prepare) and [execute](../../../api/query.md#pykx.query.SQL.execute) functionality to separate SQL parsing from query execution as detailed [here](https://code.kx.com/insights/1.10/core/sql.html#prepare-and-execute). +To optimize frequently called SQL queries you can make use of the [prepare](../../../api/query.md#pykx.query.SQL.prepare) and [execute](../../../api/query.md#pykx.query.SQL.execute) functionality to separate SQL parsing from query execution as detailed [here](https://code.kx.com/insights/core/sql.html#prepare-and-execute). ## Next Steps diff --git a/docs/user-guide/fundamentals/query/sql.md b/docs/user-guide/fundamentals/query/sql.md index 020da72..5386d6a 100644 --- a/docs/user-guide/fundamentals/query/sql.md +++ b/docs/user-guide/fundamentals/query/sql.md @@ -121,7 +121,7 @@ GOOG 2022.01.02 423.6121 Now that you have learnt the fundamentals of how to query your data using the SQL API you may be interested in: -- To optimize frequently called SQL queries the [prepare](../../../api/query.md#pykx.query.SQL.prepare) and [execute](../../../api/query.md#pykx.query.SQL.execute) methods can be used to separate SQL parsing from query execution as detailed [here](https://code.kx.com/insights/1.10/core/sql.html#prepare-and-execute). +- To optimize frequently called SQL queries the [prepare](../../../api/query.md#pykx.query.SQL.prepare) and [execute](../../../api/query.md#pykx.query.SQL.execute) methods can be used to separate SQL parsing from query execution as detailed [here](https://code.kx.com/insights/core/sql.html#prepare-and-execute). - If you want to query your data in a more Python-first way follow the guide [here](./pyquery.md). - If you want to query your data in q follow the guide [here](./qquery.md). - To learn how to make your queries more performant following the tips and tricks [here](./perf.md). diff --git a/licenses/QSO.txt b/licenses/QSO.txt index ee31923..9e57305 100644 --- a/licenses/QSO.txt +++ b/licenses/QSO.txt @@ -1,7 +1,7 @@ Software License for q.so -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -13,7 +13,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/mkdocs.yml b/mkdocs.yml index 293a4d2..15c1577 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,7 @@ site_name: 'PyKX' site_author: 'KX' site_description: 'PyKX Documentation' site_url: 'https://code.kx.com/pykx' -copyright: '© 2025 Kx Systems, Inc. KX and kdb+ are registered trademarks of Kx Systems, Inc., a subsidiary of FD Technologies plc.' +copyright: '© 2026 KX Systems, Inc. KX, KDB-X, and kdb+ are registered trademarks of KX Systems, Inc., a subsidiary of KX Software Limited.' site_dir: 'public' dev_addr: 'localhost:8080' @@ -28,7 +28,7 @@ extra_css: - https://code.kx.com/dashboards/stylesheets/dashboards.css - stylesheets/swagger.css # More accessible colors in the edited versions - - stylesheets/2021-edited.css + - stylesheets/2026-edited.css - stylesheets/prism-edited.css extra_javascript: @@ -40,6 +40,7 @@ extra_javascript: - 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-MML-AM_CHTML' - 'https://code.jquery.com/jquery-3.6.0.min.js' - 'https://use.fontawesome.com/3dbf29ed12.js' + - scripts/strip-q-prompt-copy.js # this relative path is needed to source from code.kx.com!!! - "../../pykx/scripts/versions.js" # Uncomment only for local test @@ -104,7 +105,7 @@ plugins: rendering: docstring_section_style: 'table' heading_level: 2 - line_length: 80 + line_length: 100 members_order: 'source' merge_init_into_class: true separate_signature: true @@ -146,7 +147,56 @@ plugins: - examples/db-management.ipynb - examples/streaming/Evolving System.ipynb - examples/charting.ipynb - + - redirects: + redirect_maps: + 'api/console.md': 'api/pykx-execution/console.md' + 'api/ctx.md': 'api/pykx-execution/ctx.md' + 'api/embedded_q.md': 'api/pykx-execution/embedded_q.md' + 'api/pykx-save-load/read.md': 'api/pykx-save-load/fileio.md' + 'api/pykx-save-load/write.md': 'api/pykx-save-load/fileio.md' + 'api/pykx_under_q.md': 'api/pykx-execution/ctx.md' + 'api/q/q.md': 'api/pykx-execution/q.md' + 'api/read.md': 'api/pykx-save-load/fileio.md' + 'api/toq.md': 'api/pykx-q-data/toq.md' + 'api/type_conversions.md': 'api/pykx-q-data/type_conversions.md' + 'api/wrappers.md': 'api/pykx-q-data/wrappers.md' + 'api/write.md': 'api/pykx-save-load/fileio.md' + 'beta-features/compress-encypt.md': 'user-guide/advanced/compress-encrypt.md' + 'beta-features/db-management.md': 'user-guide/advanced/database/index.md' + 'beta-features/remote-functions.md': 'user-guide/advanced/remote-functions.md' + 'beta-features/streamlit.md': 'user-guide/advanced/streamlit.md' + 'beta-features/threading.md': 'user-guide/advanced/threading.md' + 'changelog.md': 'release-notes/changelog.md' + 'comparisons.md': 'getting-started/what_is_pykx.md' + 'evaluating.md': 'user-guide/fundamentals/evaluating.md' + 'examples/threaded_execution/index.md': 'examples/threaded_execution/threading.md' + 'examples/threaded_execution/threaded_execution.md': 'examples/threaded_execution/threading.md' + 'extras/comparisons.md': 'getting-started/what_is_pykx.md' + 'extras/faq.md': 'help/faq.md' + 'extras/known_issues.md': 'help/issues.md' + 'faq.md': 'help/faq.md' + 'getting-started/interface_overview.md': 'getting-started/what_is_pykx.md' + 'getting-started/q_magic_command.md': 'examples/jupyter-integration.ipynb' + 'indexing.md': 'user-guide/fundamentals/indexing.md' + 'installing.md': 'getting-started/installing.md' + 'interface_overview.md': 'getting-started/what_is_pykx.md' + 'known_issues.md': 'help/issues.md' + 'licenses.md': 'license.md' + 'limitations.md': 'help/issues.md' + 'modes.md': 'user-guide/advanced/modes.md' + 'nulls_and_infinities.md': 'user-guide/fundamentals/nulls_and_infinities.md' + 'parallelization.md': 'user-guide/advanced/performance.md' + 'performance.md': 'user-guide/advanced/performance.md' + 'pykx-under-q/known_issues.md': 'help/issues.md' + 'troubleshooting.md': 'help/troubleshooting.md' + 'user-guide/advanced/environment_variables.md': 'user-guide/configuration.md' + 'user-guide/advanced/limitations.md': 'help/issues.md' + 'user-guide/advanced/running_under_q.md': 'pykx-under-q/intro.md' + 'user-guide/fundamentals/querying.md': 'user-guide/fundamentals/query/index.md' + 'user-guide/index.md': 'index.md' + '1.0/licenses.md': 'license.md' + '1.5/api/type_conversion.md': 'api/pykx-q-data/type_conversions.md' + '3.0/pykx-under-q/type_conversions.md': 'api/pykx-q-data/type_conversions.md' theme: name: material @@ -181,125 +231,118 @@ theme: docs_dir: docs nav: - - Home: 'https://code.kx.com/insights/' - - kdb+ and q: 'https://code.kx.com/q' - - kdb Insights SDK: "https://code.kx.com/insights/core" - - kdb Insights Enterprise: "https://code.kx.com/insights/platform/" - - KDB.AI: "https://code.kx.com/kdbai/" - - PyKX: - - index.md + - Home: + - Overview: index.md - Get Started: - Install: getting-started/installing.md - Quickstart guide: getting-started/quickstart.md - - Learn: - - What is PyKX?: getting-started/what_is_pykx.md - - Objects and attributes: learn/objects.md - - Databases: user-guide/advanced/database/index.md - - Modes of operation: user-guide/advanced/modes.md - - Performance tips: user-guide/advanced/performance.md - - Why upgrade from embedPy: pykx-under-q/upgrade.md - - Glossary: extras/glossary.md - - Blogs, articles, videos: blogs.md - - How to: - - Configure PyKX: user-guide/configuration.md - - Interact with data: - - Create objects: user-guide/fundamentals/creating.md - - Use objects: user-guide/fundamentals/evaluating.md - - Index objects: user-guide/fundamentals/indexing.md - - Convert data: user-guide/fundamentals/conversion_considerations.md - - Convert text: user-guide/fundamentals/text.md - - Convert nulls and infinities: user-guide/fundamentals/nulls_and_infinities.md - - Convert temporal data: user-guide/fundamentals/temporal.md - - Interact with databases: - - Generate databases: user-guide/advanced/database/db_gen.md - - Load databases: user-guide/advanced/database/db_loading.md - - Manage databases: user-guide/advanced/database/db_mgmt.md - - Query data: - - Introduction: user-guide/fundamentals/query/index.md - - Query with Python: user-guide/fundamentals/query/pyquery.md - - Query with SQL: user-guide/fundamentals/query/sql.md - - Query with q: user-guide/fundamentals/query/qquery.md - - Performance considerations: user-guide/fundamentals/query/perf.md - - Communicate via IPC: user-guide/advanced/ipc.md - - Capture real-time data: - - Introduction: user-guide/advanced/streaming/index.md - - Capture and Store: user-guide/advanced/streaming/basic.md - - Publish data: user-guide/advanced/streaming/publish.md - - Subscribe to data: user-guide/advanced/streaming/subscribe.md - - Analyze streaming data: user-guide/advanced/streaming/rta.md - - Build custom APIs: user-guide/advanced/streaming/custom_apis.md - - Manage query routing: user-guide/advanced/streaming/gateways.md - - Complex infrastructure: user-guide/advanced/streaming/complex.md - - Apply attributes: user-guide/advanced/attributes.md - - Compress and encrypt data: user-guide/advanced/compress-encrypt.md - - Import existing q functions: user-guide/advanced/context_interface.md - - Run remote functions: user-guide/advanced/remote-functions.md - - Serialize data: user-guide/advanced/serialization.md - - Use PyKX in subprocesses: user-guide/advanced/subprocess.md - - Run q code multithreaded: user-guide/advanced/threading.md - - Manage your license: user-guide/advanced/license.md - - Use Python in a q process: pykx-under-q/intro.md - - Reference: - - API references: - - q functions and operators: api/pykx-execution/q.md - - Execution classes: api/pykx-execution/embedded_q.md - - q contexts: api/pykx-execution/ctx.md - - Emulated q console: api/pykx-execution/console.md - - Data generation: api/random.md - - Schema generation: api/schema.md - - Query data: api/query.md - - Query classes: api/columns.md - - Registering custom operations: api/pykx-q-data/register.md - - Compression and encryption: api/compress.md - - Database management: api/db.md - - File I/O: api/pykx-save-load/fileio.md - - q IPC interface: api/ipc.md - - Streamlit integration: api/streamlit.md - - Serialize/deserialize: api/serialize.md - - Remote Python execution: api/remote.md - - Real-time capture: api/tick.md - - System command wrappers: api/system.md - - Utility functions: api/util.md - - Reimporter module: api/reimporting.md - - License management: api/license.md - - PyKX under q: pykx-under-q/api.md - - Data types and conversions: - - Python to PyKX: api/pykx-q-data/toq.md - - Wrappers: api/pykx-q-data/wrappers.md - - PyKX to Python: api/pykx-q-data/type_conversions.md - - Pandas API: user-guide/advanced/Pandas_API.ipynb - - Exceptions: api/exceptions.md - - Integrations: - - NumPy: user-guide/advanced/numpy.md - - Streamlit: user-guide/advanced/streamlit.md - - Python charting libraries: examples/charting.ipynb - - Jupyter Integration: examples/jupyter-integration.ipynb - - Examples: - - PyKX Introduction Notebook: examples/interface-overview.ipynb - - Subscriber: examples/subscriber/readme.md - - Compress and encrypt: examples/compress_and_encrypt/readme.md - - Database creation and management: examples/db-management.ipynb - - IPC: examples/ipc/README.md - - PyKX as a server: examples/server/server.md - - Real-time streaming: examples/streaming/index.md - - Multithreaded execution: examples/threaded_execution/threading.md - - Asynchronous Queries: examples/AsynchronousQueries/async_querying.md - - Releases: - - Release notes: - - PyKX: release-notes/changelog.md - - PyKX under q: release-notes/underq-changelog.md - - Deprecations: release-notes/deprecations.md - - 2.x -> 3.x Upgrade : upgrades/2030.md - - Beta features: - - Introduction: beta-features/index.md - - PyTorch Conversions: beta-features/torch.md - - Help and Support: - - Troubleshooting: help/troubleshooting.md - - FAQ: help/faq.md - - PyKX support: help/support.md - - Issues and limitations: help/issues.md - - PyKX license: license.md - - Contributors: contributors.md - - APIs: 'https://code.kx.com/insights/api/index.html' - - Licensing: 'https://code.kx.com/insights/licensing/licensing/' - - Help: 'https://code.kx.com/home/support.html' + - Learn: + - What is PyKX?: getting-started/what_is_pykx.md + - Objects and attributes: learn/objects.md + - Databases: user-guide/advanced/database/index.md + - Modes of operation: user-guide/advanced/modes.md + - Performance tips: user-guide/advanced/performance.md + - Why upgrade from embedPy: pykx-under-q/upgrade.md + - Glossary: extras/glossary.md + - Blogs, articles, videos: blogs.md + - How to: + - Configure PyKX: user-guide/configuration.md + - Interact with data: + - Create objects: user-guide/fundamentals/creating.md + - Use objects: user-guide/fundamentals/evaluating.md + - Index objects: user-guide/fundamentals/indexing.md + - Convert data: user-guide/fundamentals/conversion_considerations.md + - Convert text: user-guide/fundamentals/text.md + - Convert nulls and infinities: user-guide/fundamentals/nulls_and_infinities.md + - Convert temporal data: user-guide/fundamentals/temporal.md + - Interact with databases: + - Generate databases: user-guide/advanced/database/db_gen.md + - Load databases: user-guide/advanced/database/db_loading.md + - Manage databases: user-guide/advanced/database/db_mgmt.md + - Query data: + - Introduction: user-guide/fundamentals/query/index.md + - Query with Python: user-guide/fundamentals/query/pyquery.md + - Query with SQL: user-guide/fundamentals/query/sql.md + - Query with q: user-guide/fundamentals/query/qquery.md + - Performance considerations: user-guide/fundamentals/query/perf.md + - Communicate via IPC: user-guide/advanced/ipc.md + - Capture real-time data: + - Introduction: user-guide/advanced/streaming/index.md + - Capture and Store: user-guide/advanced/streaming/basic.md + - Publish data: user-guide/advanced/streaming/publish.md + - Subscribe to data: user-guide/advanced/streaming/subscribe.md + - Analyze streaming data: user-guide/advanced/streaming/rta.md + - Build custom APIs: user-guide/advanced/streaming/custom_apis.md + - Manage query routing: user-guide/advanced/streaming/gateways.md + - Complex infrastructure: user-guide/advanced/streaming/complex.md + - Apply attributes: user-guide/advanced/attributes.md + - Compress and encrypt data: user-guide/advanced/compress-encrypt.md + - Import existing q functions: user-guide/advanced/context_interface.md + - Run remote functions: user-guide/advanced/remote-functions.md + - Serialize data: user-guide/advanced/serialization.md + - Use PyKX in subprocesses: user-guide/advanced/subprocess.md + - Run q code multithreaded: user-guide/advanced/threading.md + - Manage your license: user-guide/advanced/license.md + - Use Python in a q process: pykx-under-q/intro.md + - Reference: + - API references: + - q functions and operators: api/pykx-execution/q.md + - Execution classes: api/pykx-execution/embedded_q.md + - q contexts: api/pykx-execution/ctx.md + - Emulated q console: api/pykx-execution/console.md + - Data generation: api/random.md + - Schema generation: api/schema.md + - Query data: api/query.md + - Query classes: api/columns.md + - Registering custom operations: api/pykx-q-data/register.md + - Compression and encryption: api/compress.md + - Database management: api/db.md + - File I/O: api/pykx-save-load/fileio.md + - q IPC interface: api/ipc.md + - Streamlit integration: api/streamlit.md + - Serialize/deserialize: api/serialize.md + - Remote Python execution: api/remote.md + - Real-time capture: api/tick.md + - System command wrappers: api/system.md + - Utility functions: api/util.md + - Reimporter module: api/reimporting.md + - License management: api/license.md + - PyKX under q: pykx-under-q/api.md + - Data types and conversions: + - Python to PyKX: api/pykx-q-data/toq.md + - Wrappers: api/pykx-q-data/wrappers.md + - PyKX to Python: api/pykx-q-data/type_conversions.md + - Pandas API: user-guide/advanced/Pandas_API.ipynb + - Exceptions: api/exceptions.md + - Integrations: + - NumPy: user-guide/advanced/numpy.md + - Streamlit: user-guide/advanced/streamlit.md + - Python charting libraries: examples/charting.ipynb + - Jupyter Integration: examples/jupyter-integration.ipynb + - Examples: + - PyKX Introduction Notebook: examples/interface-overview.ipynb + - Subscriber: examples/subscriber/readme.md + - Compress and encrypt: examples/compress_and_encrypt/readme.md + - Database creation and management: examples/db-management.ipynb + - IPC: examples/ipc/README.md + - PyKX as a server: examples/server/server.md + - Real-time streaming: examples/streaming/index.md + - Multithreaded execution: examples/threaded_execution/threading.md + - Asynchronous Queries: examples/AsynchronousQueries/async_querying.md + - Releases: + - Release notes: + - PyKX: release-notes/changelog.md + - PyKX under q: release-notes/underq-changelog.md + - Deprecations: release-notes/deprecations.md + - 2.x -> 3.x Upgrade : upgrades/2030.md + - Beta features: + - Introduction: beta-features/index.md + - PyTorch Conversions: beta-features/torch.md + - Help and Support: + - Troubleshooting: help/troubleshooting.md + - FAQ: help/faq.md + - PyKX support: help/support.md + - Issues and limitations: help/issues.md + - PyKX license: license.md + - Contributors: contributors.md + diff --git a/pyproject.toml b/pyproject.toml index ade9e55..4e9c097 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "pykx" description = "An interface between Python and q" readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.9" license = { file = "LICENSE.txt" } authors = [ { name = "KX", email = "pykx@kx.com" }, @@ -31,11 +31,12 @@ classifiers = [ "Programming Language :: Cython", "Programming Language :: Other", "Programming Language :: Other Scripting Engines", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Implementation :: CPython", # "Programming Language :: Python :: Implementation :: PyPy", @@ -48,15 +49,14 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "numpy~=1.20; python_version=='3.7'", - "numpy~=1.22; python_version=='3.8'", "numpy~=1.22; python_version=='3.9'", "numpy>=1.22; python_version=='3.10'", - "numpy>=1.23, < 2.3.0; python_version=='3.11'", - "numpy>=1.26, < 2.3.0; python_version=='3.12'", - "numpy>=1.26, < 2.3.0; python_version=='3.13'", - "pandas>=1.2, < 2.0; python_version=='3.8'", - "pandas>=1.2, <= 2.3.0; python_version>'3.8'", + "numpy>=1.23; python_version=='3.11'", + "numpy>=1.26; python_version=='3.12'", + "numpy>=1.26; python_version=='3.13'", + "numpy>=2.3.0; python_version=='3.14'", + "pandas>=1.2, <3.0; python_version>'3.8' and python_version<'3.14'", + "pandas>= 2.3.3, <3.0; python_version=='3.14'", "pytz>=2022.1", "toml~=0.10.2", "dill>=0.2.0", @@ -78,6 +78,7 @@ doc = [ "mkdocs-exclude-search==0.6.6", "mkdocs-jupyter~=0.24", "mkdocs-material~=9.4.5", + "mkdocs-redirects>=1.2.0", "mkdocs-render-swagger-plugin==0.0.3", "mkdocs-spellcheck==0.2.0", "mkdocstrings[python]==0.18.0", @@ -101,7 +102,7 @@ lint = [ "pyproject-flake8==0.0.1a2", ] pyarrow = [ - "pyarrow>=3.0.0, <19.0.0", + "pyarrow>=3.0.0", ] streaming = [ "psutil>=5.0.0" @@ -110,7 +111,7 @@ dashboards = [ "ast2json~=0.3", ] streamlit = [ - "streamlit~=1.28; python_version>'3.7'" + "streamlit~=1.28" ] torch = [ "torch>2.1; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version <= '3.12'" @@ -118,7 +119,8 @@ torch = [ test = [ "coverage[toml]==6.3.2", "Cython~=3.0.0", - "pytest==7.1.2", + "pytest==7.1.2; python_version<'3.14'", + "pytest==8.4.0; python_version=='3.14'", "pytest-asyncio==0.18.3", "pytest-benchmark==3.4.1", "pytest-cov==3.0.0", @@ -130,10 +132,6 @@ test = [ "pytest-timeout>=2.0.0", "IPython", ] -help = [ - "beautifulsoup4==4.10.0", - "markdown2==2.5.0", -] [project.urls] @@ -146,17 +144,17 @@ changelog = "https://code.kx.com/pykx/release-notes/changelog.html" [build-system] requires = [ "Cython~=3.0.0", - "numpy~=1.20.0; python_version=='3.7'", # Use numpy version 1.20.x for building the python 3.7 wheel - "numpy~=1.22.0; python_version=='3.8'", # Use the highest patch version of numpy 1.22.x, this will still support a user using numpy version 1.22.0 "numpy~=2.0; python_version=='3.9'", "numpy~=2.0; python_version=='3.10'", - "numpy~=2.0, < 2.3.0; python_version=='3.11'", - "numpy~=2.0, < 2.3.0; python_version=='3.12'", - "numpy~=2.0, < 2.3.0; python_version=='3.13'", - "setuptools~=68.0.0; python_version=='3.7'", - "setuptools~=69.0.2; python_version!='3.7'", - "setuptools-scm[toml]~=7.1.0; python_version=='3.7'", - "setuptools-scm[toml]~=8.0; python_version!='3.7'", + "numpy~=2.0; python_version=='3.11'", + "numpy~=2.0; python_version=='3.12'", + "numpy~=2.0; python_version=='3.13'", + "numpy>=2.3.0; python_version=='3.14'", + "numpy~=2.0; python_version=='3.11'", + "numpy~=2.0; python_version=='3.12'", + "numpy~=2.0; python_version=='3.13'", + "setuptools~=69.0.2", + "setuptools-scm[toml]~=8.0", "tomli>=2.0.1", "wheel>=0.42.0", ] diff --git a/setup.py b/setup.py index ffb1742..db30d9a 100755 --- a/setup.py +++ b/setup.py @@ -256,8 +256,7 @@ def ext(name: str, ]), ] - if py_minor_version >= 8: # python 3.8 or higher is required for NEP-49 - exts.append(ext('_numpy', numpy=True, cython=False, libraries=['dl', *windows_libraries])) + exts.append(ext('_numpy', numpy=True, cython=False, libraries=['dl', *windows_libraries])) exts.append(ext('numpy_conversions', numpy=True, cython=False, diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index 1d8c1bd..79839d1 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -362,11 +362,9 @@ def merge_asof(left, *args, **kwargs): return left.merge_asof(*args, **kwargs) -if sys.version_info[1] < 8: +if sys.version_info[1] < 9: warn( - 'Python 3.7 has reach its end of life period and is no longer supported.' - 'Please consider upgrading to Python 3.8, as PyKX will no longer support issues for this ' - 'Python version.', + 'Python 3.9 is the minimum support version. Please upgrade.', exceptions.PyKXWarning ) diff --git a/src/pykx/_wrappers.pyx b/src/pykx/_wrappers.pyx index 80bfa99..8510220 100644 --- a/src/pykx/_wrappers.pyx +++ b/src/pykx/_wrappers.pyx @@ -60,6 +60,14 @@ NPY_TYPE_ITEMSIZE = { } +def memview_to_array(view) -> np.ndarray: + cdef np.npy_intp n = (view._ptr).n + cdef np.ndarray arr = np.PyArray_SimpleNewFromData(1, &n, 2, (view._ptr).G0) + Py_INCREF(view) + PyArray_SetBaseObject(arr, view) + return arr + + def k_vec_to_array(k_vec, np_type: int) -> np.ndarray: """Creates a Numpy array that references the K object until the array's data is deallocated.""" cdef np.npy_intp n = (k_vec._addr).n @@ -118,7 +126,11 @@ def k_str(self): cpdef inline deserialize(x): - cdef core.K buff = core.kpn(x, len(x)) + l = len(x) + if isinstance(x, np.ndarray): + x = x.data.tobytes() + cdef core.K buff = core.kpn(x, l) + if 0 == core.okx(buff): core.r0(buff) raise QError('Failed to deserialize supplied non PyKX IPC serialized format object') @@ -546,11 +558,11 @@ cdef inline factory(uintptr_t addr, bint incref, str name='', bint err_preamble= err_string = str((addr).s, 'utf-8') if err_string == 'nosocket': err_string = 'nosocket: Cannot open or use a socket on a thread other than main.\n'\ - 'Read https://code.kx.com/user-guide/advanced/threading.html for more information' + 'Read https://code.kx.com/pykx/user-guide/advanced/threading.html for more information' elif err_string == 'noupdate': err_string = 'noupdate: Cannot update a global variable while using:\n\t- Multithreaded mode'\ '\n\t- peach with secondary threads\n\t- `-b` command line argument or reval code.\n'\ - 'Read https://code.kx.com/user-guide/advanced/threading.html for more information' + 'Read https://code.kx.com/pykx/user-guide/advanced/threading.html for more information' elif err_string == 'par': err_string = 'par: Cannot execute an unsupported operation on a partitioned table or its '\ 'constituent parts' diff --git a/src/pykx/config.py b/src/pykx/config.py index b166cad..d5d6ba9 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -12,7 +12,7 @@ import toml import pandas as pd -from .exceptions import PyKXWarning, QError +from .exceptions import QError system = platform.system() @@ -219,8 +219,8 @@ def _license_check(lic_type, lic_encoding, lic_variable): # pragma: no cover def _unlicensed_config(unlicensed_message): - choice = input('\nWould you like us to remember this choice? [Y/n]: ') - if choice in ('y', 'Y', ''): + choice = input('\nWould you like us to remember this choice? [N/y]: ') + if choice in ('y', 'Y'): fpath = Path(os.path.expanduser('~')) / '.pykx-config' try: os.access(fpath, os.W_OK) @@ -271,8 +271,8 @@ def _license_install(intro=None, return_value=False, license_check=False, licens print(install_message) return True - personal_url = "https://kx.com/kdb-insights-sdk-personal-edition-download" - commercial_url = "https://kx.com/book-demo" + lic_url = 'https://developer.kx.com/products/kdb-x/install' + lic_type = 'kc.lic' unlicensed_message = '\nPyKX unlicensed mode enabled. To set this as your default behavior '\ "set the following environment variable PYKX_UNLICENSED='true'" first_user = '\nThank you for installing PyKX!\n\n'\ @@ -296,69 +296,24 @@ def _license_install(intro=None, return_value=False, license_check=False, licens if existing_license not in ('Y', 'y', 'N', 'n', ''): raise Exception('Invalid input provided please try again') if existing_license in ('N', 'n', ''): - commercial = input('\nIs the intended use of this software for:' - '\n [1] Personal use (Default)' - '\n [2] Commercial use' - '\nEnter your choice here [1/2]: ').strip().lower() - if commercial not in ('1', '2', ''): - raise Exception('User provided option was not one of [1/2]') - - personal = commercial in ('1', '') - - lic_url = personal_url if personal else commercial_url - lic_type = _kc_lic if personal else _k4_lic - - if personal: - redirect = input(f'\nTo apply for your PyKX license, navigate to {lic_url}.\n' - 'Shortly after you submit your application, you will receive a ' - 'welcome email containing your license information.\n' - 'Would you like to open this page? [Y/n]: ') - else: - redirect = input('\nTo apply for your PyKX license, contact your ' - 'KX sales representative or sales@kx.com.\n' - f'Alternately apply through {lic_url}.\n' - 'Would you like to open this page? [Y/n]: ') - + redirect = input('\n** It is recommended to upgrade to KDB-X Python pykx>=4.0 **\n\n' + f'For instructions and to obtain a license visit: {lic_url}\n' + 'Would you like to open this page? [Y/n]: ') if redirect.lower() in ('y', ''): try: webbrowser.open(lic_url) time.sleep(2) except BaseException: raise Exception('Unable to open web browser') - - install_type = input('\nPlease select the method you wish to use to activate your' - 'license:\n [1] Download the license file provided in your ' - 'welcome email and input the file path (Default)' - '\n [2] Input the activation key (base64 encoded string) ' - 'provided in your welcome email' - '\n [3] Proceed with unlicensed mode' - '\nEnter your choice here [1/2/3]: ').strip().lower() - - if install_type not in ('1', '2', '3', ''): - raise Exception('User provided option was not one of [1/2/3]') - - if install_type in ('1', ''): - license = input('\nProvide the download location of your license ' - f'(for example, {root}{lic_type}) : ').strip() - download_location = os.path.expanduser(Path(license)) - _license_install_path(download_location, qlic) - - elif install_type == '2': - license = input('\nProvide your activation key (base64 encoded string) ' - 'provided with your welcome email : ').strip() - - _license_install_B64(license, lic_type) - - print('\nPyKX license successfully installed to: {qlic / lic_type}\n') # noqa: E501 - elif install_type == '3': - _unlicensed_config(unlicensed_message) - if return_value: - return False + print('\nPyKX will proceed in unlicensed mode') + _unlicensed_config(unlicensed_message) + if return_value: + return False else: install_type = input( '\nPlease select the method you wish to use to activate your license:\n' - ' [1] Provide the location of your license\n' - ' [2] Input the activation key\n' + ' [1] Provide the location of your license file\n' + ' [2] Paste the license key\n' 'Enter your choice here [1/2]: ') if install_type not in ('1', '2', ''): @@ -370,16 +325,15 @@ def _license_install(intro=None, return_value=False, license_check=False, licens _license_install_path(download_location, qlic) else: - commercial = input('\nPlease confirm the license type:\n' - f' [1] Personal use ({_kc_lic})\n' - f' [2] Commercial use ({_k4_lic})\n' - 'Enter your choice here [1/2]: ') - if commercial not in ('1', '2', ''): + lic_type_choice = input('\nPlease confirm the license type:\n' + f' [1] {_kc_lic} - The default\n' + f' [2] {_k4_lic} - Used in some scenarios\n' + 'Enter your choice here [1/2]: ') + if lic_type_choice not in ('1', '2', ''): raise Exception('User provided option was not one of [1/2]') - personal = commercial in ('1', '') - lic_type = _kc_lic if personal else _k4_lic - license = input('\nProvide your activation key (base64 encoded string) : ').strip() + lic_type = _k4_lic if lic_type_choice == '2' else _kc_lic + license = input('f\nProvide your {lic_type} license key (base64 encoded string) : ').strip() # noqa: E501 _license_install_B64(license, lic_type) @@ -420,15 +374,7 @@ def _license_install(intro=None, return_value=False, license_check=False, licens keep_local_times = _is_enabled('PYKX_KEEP_LOCAL_TIMES') max_error_length = int(_get_config_value('PYKX_MAX_ERROR_LENGTH', 256)) -allocator = _is_enabled('PYKX_ALLOCATOR', '--pykxalloc') -if allocator: - if sys.version_info[1] <= 7: - raise PyKXWarning('A python version of at least 3.8 is required to use the PyKX allocators') # noqa nocov - k_allocator = False # nocov - else: - k_allocator = True -else: - k_allocator = False +k_allocator = _is_enabled('PYKX_ALLOCATOR', '--pykxalloc') k_gc = _is_enabled('PYKX_GC', '--pykxgc') release_gil = _is_enabled('PYKX_RELEASE_GIL', '--release-gil') diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx index cfede9e..f35bbe7 100644 --- a/src/pykx/core.pyx +++ b/src/pykx/core.pyx @@ -312,20 +312,19 @@ if not pykx_threading: 'Would you like to renew your license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' _license_message = _license_install(_exp_license, True, True, 'exp') elif re.compile('licen[cs]e error: embedq').search(_capout_msg): - _ce_license = 'You appear to be using a non kdb Insights license.\n\n'\ + _ce_license = 'You appear to be using a non PyKX enabled license.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ - 'Running PyKX in the absence of a kdb Insights license '\ + 'Running PyKX in the absence of a license '\ 'has reduced functionality.\nWould you like to install '\ - 'a kdb Insights personal license? [Y/n]: ' + 'a license? [Y/n]: ' _license_message = _license_install(_ce_license, True) elif re.compile('licen[cs]e error: upd').search(_capout_msg): _upd_license = 'Your installed license is out of date for this version'\ ' of PyKX and must be updated.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ - 'Would you like to install an updated kdb '\ - 'Insights personal license? [Y/n]: ' + 'Would you like to install an updated license? [Y/n]: ' _license_message = _license_install(_upd_license, True) elif re.compile('licen[cs]e error: k[xc4].lic').search(_capout_msg) or re.compile('licen[cs]e error: badmsg').search(_capout_msg): _k_license = '\nThe PyKX license found is corrupt or incompatible with'\ diff --git a/src/pykx/db.py b/src/pykx/db.py index 2458a50..a021920 100644 --- a/src/pykx/db.py +++ b/src/pykx/db.py @@ -146,7 +146,13 @@ def __init__(self, >>> db.tables ['quote', 'trade'] ``` + !!! note "Loading root-level objects" + When using `pykx.DB`, only partitioned tables are exposed as attributes of the + `DB` object. Root-level splayed objects (for example, dictionaries or lists stored + alongside partitions) are **not** attached to the `DB` instance, but are still loaded + into the q process and can be accessed directly using `kx.q`. + For example, you can retrieve a variable `mySplayTab` by running `kx.q['mySplayTab']`. Define the path to be used for a database which does not initially exist ```python @@ -323,7 +329,10 @@ def create(self, qfunc = q(_func_mapping[func_name]) try: if format == 'splayed': - table = q.Q.en(save_dir, table) + if sym_enum is None: + table = q.Q.en(save_dir, table) + else: + table = q.Q.ens(save_dir, table, sym_enum) q('{.Q.dd[x;`] set y}', save_dir/table_name, table) else: if isinstance(partition, str): diff --git a/src/pykx/help.py b/src/pykx/help.py index e77b0b0..75ef532 100644 --- a/src/pykx/help.py +++ b/src/pykx/help.py @@ -1,9 +1,10 @@ -import importlib.util -import warnings import os +import re __all__ = ["help"] -_filepath = os.path.join(os.path.dirname(__file__), 'docs', 'api', 'pykx-execution', 'q.md') +_filepath = os.path.join( + os.path.dirname(__file__), 'docs', 'api', 'pykx-execution', 'q.md' +) def __dir__(): @@ -16,100 +17,121 @@ def _init(_q): def _load_markdown_section(keyword, file_path=_filepath): - with open(file_path, 'r') as f: - lines = f.readlines() + """Load and parse markdown section for a keyword.""" + try: + with open(file_path, 'r') as f: + lines = f.readlines() + + in_section = False + section_lines = [] + for line in lines: + if line.startswith(f'### [{keyword}]'): + in_section = True + section_lines.append(line) + elif line.startswith('###') and in_section: + break + elif in_section: + section_lines.append(line) - # Extract the specific section - in_section = False - section_lines = [] - for line in lines: - if line.startswith(f'### [{keyword}]'): - in_section = True - section_lines.append(line) - elif line.startswith('###') and in_section: - break - elif in_section: - section_lines.append(line) - - if not section_lines: - return "No description available for this function." - - # Convert the section to HTML - section_markdown = ''.join(section_lines) - - md2_spec = importlib.util.find_spec("markdown2") - if md2_spec is None: - warnings.warn("The 'markdown2' package is required to use the 'qhelp' function.") - return "" - else: - import markdown2 - section_html = markdown2.markdown(section_markdown, extras=["fenced-code-blocks"]) + if not section_lines: + return None - return section_html + return section_lines + except IOError: + return None def qhelp(keyword): """Parse and format the documentation for a given keyword.""" - bs4_spec = importlib.util.find_spec("bs4") - if bs4_spec is None: - warnings.warn("The 'bs4' package is required to use the 'qhelp' function.") - return "" - else: - from bs4 import BeautifulSoup + markdown_lines = _load_markdown_section(keyword) + if markdown_lines: + text = f" • {keyword}\n\n" + + in_code_block = False + table_lines = [] + in_table = False + + for i, line in enumerate(markdown_lines): + if i == 0: + continue + + if line.strip().startswith('```'): + if in_code_block: + in_code_block = False + text += "\n" + else: + in_code_block = True + text += "\n" + continue + + if in_code_block: + text += " " + line.rstrip() + "\n" + continue + + if '|' in line and not line.strip().startswith('#'): + if not in_table: + in_table = True + table_lines = [] + table_lines.append(line) + continue + elif in_table: + if table_lines: + text += _parse_markdown_table(table_lines) + in_table = False + table_lines = [] + + if line.strip() == '': + if text.endswith('\n\n'): + continue + text += "\n" + continue - html_content = _load_markdown_section(keyword) - if html_content: - soup = BeautifulSoup(html_content, "html.parser") - tag = soup.find("h3", string=keyword) + if line.strip().startswith('- ') or line.strip().startswith('* '): + list_text = line.strip()[2:] + list_text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', list_text) + text += " - " + list_text + "\n" + continue - if tag is None: - return None + if not line.startswith('#'): + line_text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', line.strip()) + line_text = re.sub(r'`([^`]+)`', r'\1', line_text) + if line_text: + text += line_text + "\n" - text = " • " + tag.text + "\n\n" - for x in tag.findAllNext(): - if x.name == "h2": - break + if in_table and table_lines: + text += _parse_markdown_table(table_lines) - elif x.name == "h3": - break + return text - elif x.name == "p": - only_link = True - for w in x: - if w.name != "a": - only_link = False - if not only_link: - text += x.text + "\n" + # If both local and web sources fail + return (f" • {keyword}\n\n" + f"Documentation not available locally. Visit https://code.kx.com for documentation.") - elif x.name == "pre": - text += "\n" - for line in x.text.split("\n"): - text += " " + line + "\n" - - elif x.name == "li": - text += " - " + x.text + "\n" - - elif x.name == "thead": - table = [] - tmp = [] - for cell in x.text.split("\n"): - if cell != "": - tmp += [cell] - table += [tmp] - - elif x.name == "tbody": - for row in x.text.split("\n\n\n"): - tmp = [] - for cell in row.split("\n"): - if cell != "": - tmp += [cell] - table += [tmp] - text += __ascii_table(table[1:], border=True, header=table[0]) + "\n" - return text - else: - print("html error") - return None +def _parse_markdown_table(lines): + """Parse markdown table into ASCII table format.""" + if not lines: + return "" + + rows = [] + header = None + + for line in lines: + if re.match(r'^\s*\|[\s\-:|]+\|\s*$', line): + continue + + cells = [cell.strip() for cell in line.split('|')] + cells = [c for c in cells if c] + + if cells: + if header is None: + header = cells + else: + rows.append(cells) + + if header and rows: + return __ascii_table(rows, border=True, header=header) + "\n" + return "" def __ascii_table(table: list, header: list = None, align='left', border=False): diff --git a/src/pykx/ipc.py b/src/pykx/ipc.py index fb60e69..fca6b8f 100644 --- a/src/pykx/ipc.py +++ b/src/pykx/ipc.py @@ -458,7 +458,9 @@ def __init__(self, no_ctx: bool = False, reconnection_attempts: int = -1, reconnection_delay: float = 0.5, - reconnection_function: callable = reconnection_function + reconnection_function: callable = reconnection_function, + connection_timeout: Optional[float] = None, + ): """Interface with a q process using the q IPC protocol. @@ -498,6 +500,9 @@ def __init__(self, reconnection_delay: A `#!python float` for the initial delay between reconnect attempts (in seconds). This is passed to the provided `#!python reconnection_function` that is executed on reconnect attempt. + connection_timeout: Timeout in seconds for connection to `q` server. If left as default + `None`, the socket will not timeout when connecting. + Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the @@ -518,6 +523,9 @@ def __init__(self, super().__init__() def _create_connection_to_server(self): + con_timeout = self._connection_info['connection_timeout']\ + if self._connection_info['connection_timeout'] is not None\ + else self._connection_info['timeout'] object.__setattr__( self, '_handle', @@ -527,7 +535,7 @@ def _create_connection_to_server(self): self._connection_info['credentials'], self._connection_info['unix'], self._connection_info['tls'], - self._connection_info['timeout'], + con_timeout, self._connection_info['large_messages'] ) ) @@ -564,7 +572,8 @@ def _init(self, conn_gc_time: float = 0.0, reconnection_attempts: int = -1, reconnection_delay: float = 0.5, - reconnection_function: callable = reconnection_function + reconnection_function: callable = reconnection_function, + connection_timeout: Optional[float] = None, ): credentials = f'{normalize_to_str(username, "Username")}:' \ f'{normalize_to_str(password, "Password")}' @@ -585,7 +594,8 @@ def _init(self, 'conn_gc_time': conn_gc_time, 'reconnection_attempts': reconnection_attempts, 'reconnection_delay': reconnection_delay, - 'reconnection_function': reconnection_function + 'reconnection_function': reconnection_function, + 'connection_timeout': connection_timeout, }) if system == 'Windows' and unix: # nocov raise TypeError('Unix domain sockets cannot be used on Windows') @@ -600,13 +610,14 @@ def _init(self, object.__setattr__(self, '_handle', server_sock.fileno()) object.__setattr__(self, '_finalizer', lambda: server_sock.close()) else: + con_timeout = connection_timeout if connection_timeout is not None else timeout try: handle = _ipc.init_handle(host, port, credentials, unix, tls, - timeout, + con_timeout, large_messages) except BaseException as e: if isinstance(e, QError): @@ -1140,7 +1151,8 @@ def __init__(self, no_ctx: bool = False, reconnection_attempts: int = -1, reconnection_delay: float = 0.5, - reconnection_function: callable = reconnection_function + reconnection_function: callable = reconnection_function, + connection_timeout: Optional[float] = None, ): """Interface with a q process using the q IPC protocol. @@ -1181,6 +1193,9 @@ def __init__(self, `#!python reconnection_delay` on successive attempts to reconnect to the server. By default this is an exponential backoff where the `#!python reconnection_delay` is multiplied by two on each invocation. + connection_timeout: Timeout in seconds for connection to `q` server. If left as default + `None`, the socket will not timeout when connecting. + Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the @@ -1247,7 +1262,9 @@ def __init__(self, no_ctx=no_ctx, reconnection_attempts=reconnection_attempts, reconnection_delay=reconnection_delay, - reconnection_function=reconnection_function + reconnection_function=reconnection_function, + connection_timeout=connection_timeout, + ) super().__init__() @@ -1453,7 +1470,9 @@ def __init__(self, no_ctx: bool = False, reconnection_attempts: int = -1, reconnection_delay: float = 0.5, - reconnection_function: callable = reconnection_function + reconnection_function: callable = reconnection_function, + connection_timeout: Optional[float] = None, + ): """Interface with a q process using the q IPC protocol. @@ -1499,6 +1518,9 @@ def __init__(self, `#!python reconnection_delay` on successive attempts to reconnect to the server. By default this is an exponential backoff where the `#!python reconnection_delay` is multiplied by two on each invocation + connection_timeout: Timeout in seconds for connection to `q` server. If left as default + `None`, the socket will not timeout when connecting. + Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if @@ -1593,6 +1615,7 @@ async def main(): 'reconnection_attempts':reconnection_attempts, 'reconnection_delay': reconnection_delay, 'reconnection_function': reconnection_function, + 'connection_timeout': connection_timeout, }) object.__setattr__(self, '_initialized', False) @@ -1613,6 +1636,7 @@ async def _async_init(self, reconnection_attempts: int = -1, reconnection_delay: float = 0.5, reconnection_function: callable = reconnection_function, + connection_timeout: float = 0.0, ): object.__setattr__(self, '_call_stack', []) self._init(host, @@ -1630,6 +1654,7 @@ async def _async_init(self, reconnection_attempts=reconnection_attempts, reconnection_delay=reconnection_delay, reconnection_function=reconnection_function, + connection_timeout=connection_timeout, ) object.__setattr__(self, '_loop', event_loop) con_info = object.__getattribute__(self, '_connection_info') @@ -1657,6 +1682,7 @@ async def _initobj(self): # nocov reconnection_attempts=self._stored_args['reconnection_attempts'], reconnection_delay=self._stored_args['reconnection_delay'], reconnection_function=self._stored_args['reconnection_function'], + connection_timeout=self._stored_args['connection_timeout'], ) return self @@ -2055,6 +2081,7 @@ def __init__(self, no_ctx: bool = False, as_server: bool = False, conn_gc_time: float = 0.0, + connection_timeout: Optional[float] = None, ): """Interface with a q process using the q IPC protocol. @@ -2093,6 +2120,9 @@ def __init__(self, going through the list of opened connections and closing any that the clients have closed. If not set the default of 0.0 will cause any old connections to never be closed unless `#!python self.clean_open_connections()` is manually called. + connection_timeout: Timeout in seconds for connection to `q` server. If left as default + `None`, the socket will not timeout when connecting. + Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if the q @@ -2137,7 +2167,7 @@ def __init__(self, ``` """ if timeout > 0.0: - warnings.warn('Timeout is not supported when using AsyncQConnection objects.') + warnings.warn('Timeout is not supported when using RawQConnection objects.') # TODO: Remove this once TLS support is fixed if tls: raise PyKXException('TLS is currently only supported for SyncQConnections') @@ -2164,6 +2194,7 @@ def __init__(self, 'no_ctx': True if as_server else no_ctx, 'as_server': as_server, 'conn_gc_time': conn_gc_time, + 'connection_timeout': connection_timeout, }) object.__setattr__(self, '_initialized', False) @@ -2182,6 +2213,7 @@ async def _async_init(self, no_ctx: bool = False, as_server: bool = False, conn_gc_time: float = 0.0, + connection_timeout: Optional[float] = None, ): object.__setattr__(self, '_call_stack', []) object.__setattr__(self, '_send_stack', []) @@ -2198,6 +2230,7 @@ async def _async_init(self, no_ctx=no_ctx, as_server=as_server, conn_gc_time=conn_gc_time, + connection_timeout=connection_timeout, ) object.__setattr__(self, '_loop', event_loop) con_info = object.__getattribute__(self, '_connection_info') @@ -2222,6 +2255,7 @@ async def _initobj(self): # nocov no_ctx=self._stored_args['no_ctx'], as_server=self._stored_args['as_server'], conn_gc_time=self._stored_args['conn_gc_time'], + connection_timeout=self._stored_args['connection_timeout'], ) return self @@ -2788,6 +2822,7 @@ def __init__(self, reconnection_attempts: int = -1, reconnection_delay: float = 0.5, reconnection_function: callable = reconnection_function, + connection_timeout: Optional[float] = None, ): """Interface with a q process using the q IPC protocol. @@ -2828,6 +2863,9 @@ def __init__(self, `#!python reconnection_delay` on successive attempts to reconnect to the server. By default this is an exponential backoff where the `#!python reconnection_delay` is multiplied by two on each invocation + connection_timeout: Timeout in seconds for connection to `q` server. If left as default + `None`, the socket will not timeout when connecting. + Note: The `#!python username` and `#!python password` parameters are not required. The `#!python username` and `#!python password` parameters are only required if @@ -2869,6 +2907,7 @@ def __init__(self, reconnection_attempts=reconnection_attempts, reconnection_delay=reconnection_delay, reconnection_function=reconnection_function, + connection_timeout=connection_timeout, ) super().__init__() diff --git a/src/pykx/lib/4-1-libs/q.k b/src/pykx/lib/4-1-libs/q.k index 68d1d3e..9826538 100644 --- a/src/pykx/lib/4-1-libs/q.k +++ b/src/pykx/lib/4-1-libs/q.k @@ -98,7 +98,7 @@ j12:36/:nA?;x12:nA@0x24\: /base36 J from char12(cusip) btoa:-32!;atob:-44!;sha1:-33!;prf0:+`name`file`line`col`text`pos!*-37! objp:{x like"[mgs][s3]://*"}; lo:{if[$[1>@d:!f:-1!x;1;`.d~*d];:.[$[qt d;*|`\:f;99=@d;`.;'`type];();:;d:. f]];d@:&~d like"*$";p:(d=`par.txt)|d like"[0-9]*"; - if[y;if[objp x;'"no cd to object storage"];."\\cd ",$x;f:`:.];{.q.set[*|`\:x]$[0h>@!x;. x;x`]}'`/:'f,'d@&~(d=`html)|(d like"*#")|p|s:d like"*.?";if[|/p;L[d@&p;f;~y|objp x]];if[z&~`.=x;(."\\l ",$:)'`/:'f,'d@&s&~p];} + if[y;if[objp x;'"no cd to object storage"];."\\cd ",$x;f:`:.];{.q.set[*|`\:x]$[0h>@!x;. x;x`]}'`/:'f,'d@&~(d=`html)|(d like"*#")|p|s:|/d like/:("*.[kq]_";"*.?");if[|/p;L[d@&p;f;~y|objp x]];if[z&~`.=x;(."\\l ",$:)'`/:'f,'d@&s&~p];} l:lo[;1;1] sw:{.q.sublist[_.5**|."\\C"]x};tab:{" "/:'(x*|/#:''y)$/:y};t0:{$[x>#y;((#*y)#'" -"),y;y]} @@ -111,17 +111,17 @@ hap:{@[(0;h;(h:1+i 1)^(1+*&"@"=x)&e;e:(#x)^e)_x,(^e:(i:&"/"=x:(":"=*x)_x)2)#"/"; hmb:{x:$[10=@x;x;1_$x];p:{$[#y;y;x]}/'getenv@+`$_:\("HTTP";"NO"),\:"_PROXY";u:hap@x;t:~(~#*p)||/(*":"\:u 2)like/:{(("."=*x)#"*"),x}'","\:p 1;a:$[t;p:hap@*p;u]1;r:(`$":",,/($[t;p;u]0 2))($y)," ",$[t;x;u 3]," HTTP/1.1",s,(s/:(~![-35;::])_("Accept-Encoding: gzip";"Connection: close";"Host: ",u 2),((0<#a)#,$[t;"Proxy-";""],"Authorization: Basic ",btoa a),$[#z;("Content-type: ",z 0;"Content-length: ",$#z 1);()]),(d:s,s:"\r\n"),$[#z;z 1;""];(0;n)_$[#ss[(n:4+*r ss d)#r;"content-encoding: gzip\r\n"];-35!r;r]} hg:{hmb[x;`GET;()]1};hp:{hmb[x;`POST;(y;z)]1} -a0:(#:;*:;last;sum;prd;min;max;.q.all;.q.any;?:),a1:(avg;wsum;wavg;var;dev;cov;cor),`.q `svar`sdev`scov`med +a0:(#:;*:;last;+/[0;];sum;prd;min;max;.q.all;.q.any;?:),a1:(avg;wsum;wavg;var;dev;cov;cor),`.q `svar`sdev`scov`med IN:{$[99h<@x;x in y;0]};qa:{$[qb x;0;IN[*x;a0];1;|/qa'1_x]};qb:{(2>#x)|(@x)&~11=@x} / CAN EXIT HERE FOR SMALL Q / pt(tables) pf(date/month/year/int) pd(dirs) pv(values) pn(count) pt::0#pf::` vt:(,`)!,()!(); -bvfp:{g:$[(::)~x;max;min];x:.Q.d;d:{$[y~pv;`/:'x,'d@&(d:!x)in`$$y;u@&{11h=@!x}'u:`/:'x,'`$$y]}[;y@:&y in pv]'P:$[`par.txt in!x;jp[x]'`$0:`/:x,`par.txt;,x];if[.Q.vt~(,`)!,()!();.Q.vt:P!(#P)#.:.Q.vt]; +bvfp:{g:$[(::)~x;max;min];x:.Q.d;d:$[y~z;{`/:'x,'d@&(d:!x)in`$$y}[;z];{u@&{11h=@!x}'u:`/:'x,'`$$y}[;y@:&y in pv]]'P:$[`par.txt in!x;jp[x]'`$0:`/:x,`par.txt;,x];if[.Q.vt~(,`)!,()!();.Q.vt:P!(#P)#.:.Q.vt]; t:?,/!:'vt:{(&#:'x)(=,/. x)}'{({("DMJJ"`date`month`year`int?pf)$$last@`\:x}'x)!!:':x}'d; - if[#nt:(exc:.q.except)[t;!vp];d:{`/:'x[(. y)[;0]],'(`$$(. y)[;1]),'!y}[P]@{i:y@&:x=y x:@[x;&x~\:();:;*0#pv];(i;x i)}[;g]'+:nt#/:g''nt#/:vt;.Q.vp,:nt!{(+(,pf)!,0#pv),'+(-2!'.+x)#'+|0#x:?[x;();0b;()]}':d;.Q.vt:.Q.vt,\:nt!(#nt)#,exc[?pv;y]];.Q.vt:(.Q.vt,''P!exc[y]''vt)exc''vt:(nt,!vp)#/:vt;{.[x;();:;+.q.except[!+vp x;pf]!x]}'exc[!vp;pt];pt::.q.asc@!vp;vpv::?pv;} -bvi:{$[(`vpv in !.Q);bvfp[x;?.q.except[pv;vpv]];bv x]} -bv:{vt::(,`)!,()!();vp::(0#`)!();vpv::0#pv;bvfp[x;?pv]} + if[#nt:(exc:.q.except)[t;!vp];d:{`/:'x[(. y)[;0]],'(`$$(. y)[;1]),'!y}[P]@{i:y@&:x=y x:@[x;&x~\:();:;*0#pv];(i;x i)}[;g]'+:nt#/:g''nt#/:vt;.Q.vp,:nt!{(+(,pf)!,0#pv),'+(-2!'.+x)#'+|0#x:?[x;();0b;()]}':d;.Q.vt:.Q.vt,\:nt!(#nt)#,exc[z;y]];.Q.vt:(.Q.vt,''P!exc[y]''vt)exc''vt:(nt,!vp)#/:vt;{.[x;();:;+.q.except[!+vp x;pf]!x]}'exc[!vp;pt];pt::.q.asc@!vp;vpv::z;} +bvi:{$[(`vpv in !.Q);bvfp[x;?.q.except[pv;vpv];?pv];bv x]} +bv:{vt::(,`)!,()!();vp::(0#`)!();vpv::0#pv;dpv:?pv;bvfp[x;dpv;dpv]} sp:{$[0>."\\s";x'y;x':y]} pt:pm:();MAP:{{$[0>@a:.+0!. x;.q.set[x]@.`$-1_$a;]}'a@&~(a:."\\a")in pt;pm::();if[#pt;pm::pt!sp[{(`u#pd,'pv)!sp[p2[(x;();0b;())]/;+(pd;pv)]}]pt]} @@ -142,7 +142,7 @@ fp:{+((,*x)!,(#z)#$[-7h=@y;y;(*|x)$y]),+z} foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/r(::;i)1=#i:&0<#:'r:$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} / select{u's|a's[by[date,]u's]}from t where[date..],[sym{=|in}..],.. -a2:({(%;(sum;("f"$;x));(sum;(~^:;x)))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} +a2:({(%;(sum;("f"$;x));(-;(#:;x);(+/[0;];(^:;x))))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} {(-;(avg;(*;("f"$;x);y));(*;(avg;x);(avg;y)))};{(%;(cov;x;y);(*;(dev;x);(dev;y)))};{(.q.scov;x;x)};{(sqrt;(.q.svar;x))};{(*;(%;(#:;`i);(+;-1;(#:;`i)));(cov;x;y))};{'`part}) / x0 translate;x1 aggrs;x2 translate ?terminal (subselect/exec within partition) unknown assumed uniform? diff --git a/src/pykx/license.py b/src/pykx/license.py index 9a532d1..c639a21 100644 --- a/src/pykx/license.py +++ b/src/pykx/license.py @@ -39,8 +39,7 @@ def check(license: str, format: Is the license check being completed using a downloaded file or base64 encoded string. Accepted inputs are "FILE"(default) or "STRING". license_type: The license file type/name which is to be checked, by default this - is 'kc.lic' which is the version provided with personal and commercial - evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + is 'kc.lic' but can be changed to 'k4.lic' or 'kx.lic' if appropriate Returns: A boolean indicating if the license is correct or not and a printed message describing @@ -166,8 +165,7 @@ def install(license: str, format: Is the license check being completed using a downloaded file or base64 encoded string. Accepted inputs are "FILE"(default) or "STRING". license_type: The license file type/name which is to be checked, by default this - is 'kc.lic' which is the version provided with personal and commercial - evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + is 'kc.lic' but can be changed to 'k4.lic' or 'kx.lic' if appropriate force: Enforce overwrite without opt-in message for overwrite Returns: diff --git a/src/pykx/pandas_api/pandas_conversions.py b/src/pykx/pandas_api/pandas_conversions.py index e26b6eb..3dc5e3a 100644 --- a/src/pykx/pandas_api/pandas_conversions.py +++ b/src/pykx/pandas_api/pandas_conversions.py @@ -145,20 +145,20 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c1:()!(); if[any b1; dCols1:dictCols where b1; - f1:{(`$';x)}; c1:dCols1!(f1 each dCols1)]; + f1:{({(0b;`$'x)};x)}; c1:dCols1!(f1 each dCols1)]; // Check casting to symbol, run `$string col // (also covers any symbol -> symbol cases) b2:(dictColTypes=11h) & not (b1 or tabColTypes=0h); c2:()!(); if[any b2; dCols2:dictCols where b2; - f2:{(`$string; x)}; c2:dCols2!(f2 each dCols2)]; + f2:{({(0b;`$string x)}; x)}; c2:dCols2!(f2 each dCols2)]; // Casting to string covering all cases except mixed lists b3: (dictColTypes=10h) & not tabColTypes=0h; c3:()!(); if[any b3; dCols3:dictCols where b3; - f3:{(string; x)}; c3:dCols3!(f3 each dCols3)]; + f3:{({(0b;string x)}; x)}; c3:dCols3!(f3 each dCols3)]; // Check mixed lists // if string column then allow cast to symbol // Check at beginning of method @@ -168,31 +168,32 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c4:()!(); if[any b4; dCols4:dictCols where b4; - f4:{(`$; x)}; c4:dCols4!(f4 each dCols4)]; + f4:{({(0b;`$x)}; x)}; c4:dCols4!(f4 each dCols4)]; // Any matches that meet the vanilla case // and don't have additional needs --> not any (bools) b5:not any (b1;b2;b3;b4); - .pykx.i.errorList:(); if[any b5; dCols5:dictCols where b5; dictColTypes5:dictColTypes where b5; f5:{[c;t;tvd] ({[cl;t;tvd] - @[t$;cl; - {[cl;ty;tvd;err].pykx.i.errorList,:enlist + @[{(0b;x$y)}t;cl; + {[cl;ty;tvd;err] (1b; "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], - " with q error: ", err;}[cl;t;tvd;]]}[;t;tvd]; + " with q error: ", err)}[cl;t;tvd;]]}[;t;tvd]; c)}; f5:f5[;;typeValDict]; c5:dCols5!(f5\'[dCols5;dictColTypes5])]; // Grab all cols c:c1,c2,c3,c4,c5; - tableOutput:tabColsOrig xcols ![tab;();0b;c]; - $[count .pykx.i.errorList; - .pykx.i.errorList; - tableOutput] + res:?[tab;();();c]; + $[any value[res][;0]; + {x[;1] where x[;0]} value[res]; + tabColsOrig xcols flip res[;1], + $[count cls:cols[tab] except key c; + ?[tab;();();{x!x} cls];()]] }''', self, dict_grab, type_number_to_pykx_k_type) else: @@ -234,21 +235,21 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c1:()!(); if[any b1; tCols1:tabCols where b1; - f1:{(`$';x)}; c1:tCols1!(f1 each tCols1) + f1:{({(0b;`$'x)};x)}; c1:tCols1!(f1 each tCols1) ]; // Support casting to symbol b2:(dtype=11h) & not (b1 or tabColTypes=0h); c2:()!(); if[any b2; tCols2:tabCols where b2; - f2:{(`$string@; x)}; c2:tCols2!(f2 each tCols2) + f2:{({(0b;`$string x)}; x)}; c2:tCols2!(f2 each tCols2) ]; // Support casting to string except for mixed lists b3:(dtype=10h) & not tabColTypes=0h; c3:()!(); if[any b3; tCols3:tabCols where b3; - f3:{(string; x)}; c3:tCols3!(f3 each tCols3) + f3:{({(0b;string x)}; x)}; c3:tCols3!(f3 each tCols3) ]; // For mixed lists support casting strings to symbols b4:(dtype=11h) & (tabColTypes=0h) @@ -256,30 +257,31 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c4:()!(); if[any b4; tCols4:tabCols where b4; - f4:{(`$; x)}; c4:tCols4!(f4 each tCols4) + f4:{({(0b;`$x)}; x)}; c4:tCols4!(f4 each tCols4) ]; // Any other combination not matching b1-4 b5:not any (b1;b2;b3;b4); - .pykx.i.errorList:(); c5:()!(); if[any b5; tCols5: tabCols where b5; f5:{[c;t;tvd] ({[cl;t;tvd] - @[t$;cl; - {[cl;ty;tvd;err].pykx.i.errorList,:enlist + @[{(0b;x$y)}t;cl; + {[cl;ty;tvd;err] (1b; "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], " with q error: ", - err;}[cl;t;tvd;]]}[;t;tvd]; + err)}[cl;t;tvd;]]}[;t;tvd]; c)}; c5:tCols5!(f5[;dtype;typeValDict] each tCols5) ]; c:c1,c2,c3,c4,c5; - tableOutput:tabCols xcols ![tab;();0b;c]; - $[count .pykx.i.errorList; - .pykx.i.errorList; - tableOutput] + res:?[tab;();();c]; + $[any value[res][;0]; + {x[;1] where x[;0]} value[res]; + tabCols xcols flip res[;1], + $[count cls:cols[tab] except key c; + ?[tab;();();{x!x} cls];()]] }''', self, dtype_val, type_number_to_pykx_k_type) diff --git a/src/pykx/pandas_api/pandas_merge.py b/src/pykx/pandas_api/pandas_merge.py index 56e5cbe..c893291 100644 --- a/src/pykx/pandas_api/pandas_merge.py +++ b/src/pykx/pandas_api/pandas_merge.py @@ -383,7 +383,6 @@ def merge_asof( direction != 'backward' or not allow_exact_matches or tolerance is not None - or by is not None or left_by is not None or right_by is not None ): @@ -395,7 +394,9 @@ def merge_asof( ) else: raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() only implemented for direction='backward', \ - allow_exact_matches=True, tolerance=None, by=None, left_by=None, right_by=None.") # noqa: E501 + allow_exact_matches=True, tolerance=None, left_by=None, right_by=None.") # noqa: E501 + if by is not None: + on = ([by] if isinstance(by, str) else by) + [on] (left, right, on, added_idx) = _parse_input( self, right, diff --git a/src/pykx/pandas_api/pandas_meta.py b/src/pykx/pandas_api/pandas_meta.py index 92f2427..e3bad0c 100644 --- a/src/pykx/pandas_api/pandas_meta.py +++ b/src/pykx/pandas_api/pandas_meta.py @@ -437,6 +437,14 @@ def count(self, axis=0, numeric_only=False): def isna(self): return q.null(self) + @api_return + def fills(self): + return q.fills(self) + + @api_return + def ungroup(self): + return q.ungroup(self) + @api_return def isnull(self): return self.isna() diff --git a/src/pykx/pykx.c b/src/pykx/pykx.c index 36ffdbd..9ae3b49 100644 --- a/src/pykx/pykx.c +++ b/src/pykx/pykx.c @@ -550,6 +550,8 @@ EXPORT K get_attr(K f, K attr) { } K res = create_foreign(pres); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(pres); PyGILState_Release(gstate); return res; } diff --git a/src/pykx/pykxq.c b/src/pykx/pykxq.c index 10fad12..50c8d0c 100644 --- a/src/pykx/pykxq.c +++ b/src/pykx/pykxq.c @@ -506,8 +506,11 @@ EXPORT K get_attr(K f, K attr) { int gstate = PyGILState_Ensure(); P p = get_py_ptr(f); P _attr = Py_BuildValue("s", attr->s); - K res = create_foreign(PyObject_GetAttr(p, _attr)); + P __attr = PyObject_GetAttr(p, _attr); + K res = create_foreign(__attr); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(__attr); if ((k = k_py_error())) { PyGILState_Release(gstate); return k; diff --git a/src/pykx/pykxq_m.c b/src/pykx/pykxq_m.c index f3515e0..7f2d038 100644 --- a/src/pykx/pykxq_m.c +++ b/src/pykx/pykxq_m.c @@ -157,7 +157,7 @@ EXPORT K k_pykx_init(K k_q_lib_pat, K _pykx_threading) { return (K)0; } -void* thread_init(); +void* thread_init(void* args); EXPORT K k_init_python(K x, K y, K z) { pthread_mutex_init(&head_mutex, NULL); pthread_mutex_init(&cond_mutex, NULL); @@ -664,8 +664,11 @@ K _get_attr(K f, K attr) { } P p = get_py_ptr(f); P _attr = Py_BuildValue("s", attr->s); - K res = create_foreign(PyObject_GetAttr(p, _attr)); + P __attr = PyObject_GetAttr(p, _attr); + K res = create_foreign(__attr); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(__attr); if ((k = k_py_error())) { return k; } diff --git a/src/pykx/q.so/libs/4-1/l64/libq.so b/src/pykx/q.so/libs/4-1/l64/libq.so index 9402e6a..21f2573 100755 Binary files a/src/pykx/q.so/libs/4-1/l64/libq.so and b/src/pykx/q.so/libs/4-1/l64/libq.so differ diff --git a/src/pykx/q.so/libs/4-1/l64arm/libq.so b/src/pykx/q.so/libs/4-1/l64arm/libq.so index 0fa82bc..c5c519d 100755 Binary files a/src/pykx/q.so/libs/4-1/l64arm/libq.so and b/src/pykx/q.so/libs/4-1/l64arm/libq.so differ diff --git a/src/pykx/q.so/libs/4-1/m64/libq.dylib b/src/pykx/q.so/libs/4-1/m64/libq.dylib index 548949f..84566b3 100755 Binary files a/src/pykx/q.so/libs/4-1/m64/libq.dylib and b/src/pykx/q.so/libs/4-1/m64/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib b/src/pykx/q.so/libs/4-1/m64arm/libq.dylib index 4088297..bda211c 100755 Binary files a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib and b/src/pykx/q.so/libs/4-1/m64arm/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-1/w64/q.dll b/src/pykx/q.so/libs/4-1/w64/q.dll index c679126..ee5593d 100644 Binary files a/src/pykx/q.so/libs/4-1/w64/q.dll and b/src/pykx/q.so/libs/4-1/w64/q.dll differ diff --git a/src/pykx/query.py b/src/pykx/query.py index 5a6e23e..8ed61f6 100644 --- a/src/pykx/query.py +++ b/src/pykx/query.py @@ -38,7 +38,7 @@ class QSQL: ```python qtable = pykx.q('([]1 2 3;4 5 6)') - qtable.update(pykx.Column('x', value=[10, 20, 30])) + qtable.update(pykx.Column('x', data=[10, 20, 30])) ``` 3. Development and maintenance of this interface is easier with regard to the different @@ -246,13 +246,13 @@ def update(self, ```python qtab.update({'eye': '`blue`brown`green'}) - qtab.update(pykx.Column('age', value=[25, 30, 31])) + qtab.update(pykx.Column('age', data=[25, 30, 31])) ``` Update the content of a column restricting scope using a where clause ```python - qtab.update(pykx.Column('age', value=['blue']), where=pykx.Column('hair')=='fair') + qtab.update(pykx.Column('age', data=['blue']), where=pykx.Column('hair')=='fair') ``` Define a q table suitable for by clause example @@ -424,17 +424,17 @@ def _generate_clause_columns_by(self, clause_value, clause_name, query_type): elif isinstance(clause_value, k.Column): if query_type == 'exec': if clause_value._is_tree: - return [b'.[;enlist 0;eval]', clause_value._value] + return [b'.[;enlist 0;eval]', clause_value._data] else: - return k.ParseTree(clause_value._value) + return k.ParseTree(clause_value._data) elif query_type == 'delete' and clause_name == 'columns': return [b'enlist', clause_value._name] else: if clause_value._is_tree: return [b'{enlist[x]!enlist .[y;enlist 0;eval]}', - clause_value._name, clause_value._value] + clause_value._name, clause_value._data] else: - return {clause_value._name: clause_value._value} + return {clause_value._name: clause_value._data} elif isinstance(clause_value, k.Variable): if query_type == 'exec': return k.ParseTree(clause_value._name) @@ -459,7 +459,7 @@ def _generate_clause_columns_by(self, clause_value, clause_name, query_type): for x in clause_value: if isinstance(x, k.Column): kys.append(x._name) - vls.append(x._value) + vls.append(x._data) else: kys.append(x) vls.append(x) @@ -480,9 +480,9 @@ def _generate_clause_columns_by_dict(self, clause_value): clause_dict[key] = [1, k.CharVector(val)] elif isinstance(val, k.Column): if val._is_tree: - clause_dict[key] = [2, val._value] + clause_dict[key] = [2, val._data] else: - clause_dict[key] = [0, val._value] + clause_dict[key] = [0, val._data] else: clause_dict[key] = [0, val] return [b'''{ diff --git a/src/pykx/serialize.py b/src/pykx/serialize.py index 183f1c5..45e3c58 100644 --- a/src/pykx/serialize.py +++ b/src/pykx/serialize.py @@ -3,7 +3,9 @@ """ from typing import Any, Union -from ._wrappers import _to_bytes, decref, deserialize as _deserialize +import numpy as np + +from ._wrappers import _to_bytes, decref, deserialize as _deserialize, memview_to_array from .wrappers import K @@ -70,6 +72,28 @@ def __getitem__(self, key): def __del__(self): decref(self._ptr) + def np(self): + """Returns a copy of the bytes making up the serialized object as a NumPy array. + + Examples: + + Serializing a `K` object and then copying the serialized data to a new variable. + + ```python + >>> k_obj = kx.q('til 10') + >>> ser = kx.serialize(k_obj) + >>> k_obj_array = ser.np() + >>> k_obj_array + array([ 1, 0, 0, 0, 94, 0, 0, 0, 7, 0, 10, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, + 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, + 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, + 0, 9, 0, 0, 0, 0, 0, 0, 0], dtype=uint8) + ``` + """ + return memview_to_array(self) + def copy(self): """Returns a copy of the bytes making up the serialized object. @@ -120,4 +144,6 @@ def deserialize(data: Union[bytes, serialize, memoryview]): return _deserialize(data.data.tobytes()) elif isinstance(data, memoryview): return _deserialize(data.tobytes()) + elif isinstance(data, np.ndarray): + return _deserialize(data) return _deserialize(data) diff --git a/src/pykx/toq.pyx b/src/pykx/toq.pyx index a9ef043..ef606cf 100644 --- a/src/pykx/toq.pyx +++ b/src/pykx/toq.pyx @@ -2013,7 +2013,10 @@ def from_arrow_py(x, """ if pa is None: raise PyArrowUnavailable - return toq(x.as_py(), ktype=ktype, cast=cast, handle_nulls=handle_nulls) + val = x.as_py() + if isinstance(x, pa.lib.HalfFloatScalar) and val is not None: + val = np.float16(val) + return toq(val, ktype=ktype, cast=cast, handle_nulls=handle_nulls) def from_datetime_date(x: Any, @@ -2994,7 +2997,7 @@ class ToqModule(ModuleType): elif isinstance(x, k.GroupbyTable): return self(x.tab, ktype=ktype, cast=cast, handle_nulls=handle_nulls) elif isinstance(x, k.Column): - return self(x._value) + return self(x._data) elif isinstance(x, k.QueryPhrase): return self(x._phrase) elif isinstance(x, k.Variable): diff --git a/src/pykx/util.py b/src/pykx/util.py index 8d4560b..54451ec 100644 --- a/src/pykx/util.py +++ b/src/pykx/util.py @@ -21,7 +21,7 @@ from .config import ( _executable, _get_qexecutable, _get_qhome, _pykx_config_location, _pykx_profile_content, - allocator, beta_features, ignore_qhome, jupyterq, k_gc, keep_local_times, + beta_features, ignore_qhome, jupyterq, k_allocator, k_gc, keep_local_times, load_pyarrow_unsafe, max_error_length, no_pykx_signal, no_qce, pykx_4_1, pykx_config_location, pykx_config_profile, pykx_debug_insights, pykx_dir, pykx_lib_dir, pykx_qdebug, pykx_threading, q_executable, qargs, qhome, qlic, release_gil, @@ -467,7 +467,7 @@ def env_information(): env_info = '\n**** PyKX Configuration Variables ****\n' global_config = {'PYKX_IGNORE_QHOME': ignore_qhome, 'PYKX_KEEP_LOCAL_TIMES': keep_local_times, - 'PYKX_ALLOCATOR': allocator, 'PYKX_GC': k_gc, + 'PYKX_ALLOCATOR': k_allocator, 'PYKX_GC': k_gc, 'PYKX_LOAD_PYARROW_UNSAFE': load_pyarrow_unsafe, 'PYKX_MAX_ERROR_LENGTH': max_error_length, 'PYKX_NOQCE': no_qce, 'PYKX_RELEASE_GIL': release_gil, 'PYKX_Q_LIB_LOCATION': pykx_lib_dir, diff --git a/src/pykx/wrappers.py b/src/pykx/wrappers.py index 1d44d04..65f4f0e 100644 --- a/src/pykx/wrappers.py +++ b/src/pykx/wrappers.py @@ -1924,10 +1924,18 @@ def pa(self, *, raw: bool = False, has_nulls: Optional[bool] = None): np_array = self.np(raw=raw, has_nulls=has_nulls) if not raw and isinstance(self, List): null_inds = [] + atom_types = set() for i in range(len(self)): - if isinstance(self._unlicensed_getitem(i), IntegralNumericAtom)\ - and self._unlicensed_getitem(i).is_null: + item = self._unlicensed_getitem(i) + if not isinstance(item, Function): + atom_types.add(type(item)) + if isinstance(item, IntegralNumericAtom) and item.is_null: null_inds.append(i) + if len(atom_types) > 1: + raise QError( + 'Unable to convert pykx.List with non conforming types ' + 'to PyArrow,\n failed with error: ' + 'cannot mix types in a pykx.List for PyArrow conversion') if 0 != len(null_inds): np_array[null_inds] = None return pa.array(np_array) @@ -1981,6 +1989,9 @@ def mean(self): def median(self): return q.med(self) + def fills(self): + return q.fills(self) + def mode(self): return q('{where max[c]=c:count each d:group x}', self) @@ -2620,6 +2631,11 @@ def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: boo def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): return _wrappers.guid_vector_np(self, raw, has_nulls) + def __arrow_array__(self, type=None): + if pa is None: + raise PyArrowUnavailable # nocov + return pa.array(self.np(raw=True).view('S16'), pa.binary(16)) + def pa(self, *, raw: bool = False, has_nulls: Optional[bool] = None): if pa is None: raise PyArrowUnavailable # nocov @@ -4307,7 +4323,7 @@ def __getitem__(self, key): def __setitem__(self, key, val): if isinstance(key, tuple): raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() is not implemented, .pykx.Dictionary objects do not support tuple key assignment") # noqa: E501 - self.__dict__.update(q('{x,((),y)!((),z)}', self, key, val).__dict__) + self.__dict__.update(q('{ye:(),y;x,ye!$[y~ye;z;enlist z]}', self, key, val).__dict__) def _repr_html_(self): if not licensed: @@ -5184,12 +5200,6 @@ def each_left(self): vs = each_left -bs4_spec = importlib.util.find_spec("bs4") -md2_spec = importlib.util.find_spec("markdown2") -if bs4_spec is not None and md2_spec is not None: - Function.scan.__doc__ = help.qhelp('scan') - - class Lambda(Function): """Wrapper for q lambda functions. @@ -5759,7 +5769,7 @@ def __init__(self, tree): if isinstance(tree, str): tree = q.parse(CharVector(tree)) elif isinstance(tree, Column): - tree = tree._value + tree = tree._data elif isinstance(tree, QueryPhrase): tree = tree._phrase elif isinstance(tree, Variable): @@ -5869,7 +5879,7 @@ def exists(self): class Column: """Helper class creating queries for the Query API""" - def __init__(self, column=None, name=None, value=None, is_tree=False): + def __init__(self, column=None, name=None, data=None, is_tree=False, value=None): if not licensed: raise LicenseException("use kx.Column objects") if name is not None: @@ -5879,21 +5889,73 @@ def __init__(self, column=None, name=None, value=None, is_tree=False): self._name = column self._renamed = False if value is not None: - self._value = value + self._data = value + self._renamed = True + warn("The kx.Column 'value' keyword is deprecated, please use 'data' instead", + DeprecationWarning) + elif data is not None: + if isinstance(data, (SymbolAtom, SymbolVector, str)): + data = [data] + elif isinstance(data, list) and all(isinstance(x, str) for x in data): + data = [data] + self._data = data + self._renamed = True else: - self._value = column + self._data = column self._is_tree = is_tree + if self._name is None: + raise TypeError("Column 'name' value cannot be None.") + elif not isinstance(self._name, (str, SymbolAtom)): + raise TypeError("Column 'name' can only be of type Str and pykx.SymbolAtom") + def __repr__(self): preamble = f'pykx.{type(self).__name__}' - return f"{preamble}(name='{self._name}', value={type(self._value)})" + return f"{preamble}(name='{self._name}', data={type(self._data)})" + + def __getitem__(self, key): + res = copy.deepcopy(self) + if isinstance(key, (SymbolAtom, SymbolVector, str)): + key = [key] + elif isinstance(key, list) and all(isinstance(x, str) for x in key): + key = [key] + res._data = [res._data, key] + return res - """Function for building up a function call off a Column""" def call(self, op, *other, iterator=None, col_arg_ind=0, project_args=None): + """ + Used to makes function calls off a Column object. + + Parameters: + op: The function to apply. + *other: The other column(s) or variable(s) (Python/q) to be used + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Examples: + + Find the maximum values in nested lists. + + ```python + >>> t = kx.q('([] n:(1 2;4 3;10 10))') + >>> t.select(kx.Column('n').call('max', iterator='each')) + pykx.Table(pykx.q(' + n + -- + 2 + 4 + 10 + ')) + ``` + """ params = [] for param in other: if isinstance(param, Column): - param = param._value + param = param._data elif isinstance(param, Variable): param = param._name else: @@ -5904,7 +5966,7 @@ def call(self, op, *other, iterator=None, col_arg_ind=0, project_args=None): ): param = [param] params.append(param) - params.insert(col_arg_ind, self._value) + params.insert(col_arg_ind, self._data) if isinstance(op, (str, bytes, CharVector)): op = op.encode() if isinstance(op, str) else op cmd = [q(op)] @@ -5947,7 +6009,7 @@ def call(self, op, *other, iterator=None, col_arg_ind=0, project_args=None): i.extend(cmd) cmd = i cpy = copy.deepcopy(self) - cpy._value = cmd + cpy._data = cmd return cpy def __add__(self, other, iterator=None, col_arg_ind=0, project_args=None): @@ -6048,13 +6110,13 @@ def __and__(self, other): def __or__(self, other): if isinstance(other, Column): - other = other._value + other = other._data elif isinstance(other, QueryPhrase): raise TypeError("Cannot | off a Column with a QueryPhrase") elif isinstance(other, ParseTree): other = other._tree cpy = copy.deepcopy(self) - cpy._value = [q(b'or'), self._value, other] + cpy._data = [q(b'or'), self._data, other] return cpy def abs(self, iterator=None): @@ -6143,7 +6205,7 @@ def _and(self, other, iterator=None, col_arg_ind=0, project_args=None): Use `&` or `[]` in general to build queries. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -6467,7 +6529,7 @@ def cor(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -6611,7 +6673,7 @@ def cov(self, other, sample=False, iterator=None, col_arg_ind=0, project_args=No - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. sample: Should calculations of covariance return the sample covariance (set True) covariance (set False {default}) iterator: What iterator to use when operating on the column @@ -6673,7 +6735,7 @@ def cross(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -6997,7 +7059,7 @@ def div(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7111,7 +7173,7 @@ def fby(int_or_class, by, aggregate, data, by_table=False, data_table=False): # name = by[0] elif isinstance(by, Column): name = by._name - by = by._value + by = by._data elif isinstance(by, QueryPhrase): name = by._names[0] by = by.to_dict() @@ -7120,7 +7182,7 @@ def fby(int_or_class, by, aggregate, data, by_table=False, data_table=False): # if isinstance(data, QueryPhrase): data = data.to_dict() pt = ParseTree.fby(by, aggregate, data, by_table, data_table) - return Column(name=name, value=pt) + return Column(name=name, data=pt) def fills(self, iterator=None): """ @@ -7446,7 +7508,7 @@ def inter(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7495,7 +7557,7 @@ def isin(self, other, iterator=None, col_arg_ind=0, project_args=None): Most commonly this function is used in where clauses to filter data Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7598,7 +7660,7 @@ def like(self, other, iterator=None, col_arg_ind=0, project_args=None): clauses to filter data. Parameters: - other: A string/byte array defining a regex pattern to be used for query + other: A string/byte array defining a regex pattern to be used for query. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7746,7 +7808,7 @@ def mavg(self, other, iterator=None, col_arg_ind=1, project_args=None): The results are returned as a floating point. Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving average iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -7884,7 +7946,7 @@ def mcount(self, other, iterator=None, col_arg_ind=1, project_args=None): so far, thereafter the result is the moving average Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving count iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -7957,7 +8019,7 @@ def mdev(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving standard deviation Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving standard deviation iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8131,7 +8193,7 @@ def mmax(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving maximum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving maximum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8164,7 +8226,7 @@ def mmin(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving minimum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving minimum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8195,11 +8257,11 @@ def mod(self, other, iterator=None, col_arg_ind=0, project_args=None): Calculate the modulus of items in a column for a given value. Parameters: - other: An integer denoting the divisor to be used when calculating the modulus + other: An integer denoting the divisor to be used when calculating the modulus. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 1. + where the column parameter will be used. Default 0. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -8227,7 +8289,7 @@ def msum(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving sum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving sum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8320,7 +8382,7 @@ def _or(self, other, iterator=None, col_arg_ind=0, project_args=None): Return the larger of the underlying boolean values between two columns: Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -8596,7 +8658,7 @@ def rotate(self, other, iterator=None, col_arg_ind=1, project_args=None): by the parameter other. Parameters: - other: An integer denoting the number of elements left(positive) or right(negative) + other: An integer denoting the number of elements left(positive) or right(negative). which the column list will be shifted iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -9168,7 +9230,7 @@ def union(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9286,7 +9348,7 @@ def var(self, iterator=None, sample=False): fn = 'svar' if sample else 'var' return self.call(fn, iterator=iterator) - def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): + def wavg(self, weight, iterator=None, col_arg_ind=0, project_args=None): """ Return the weighted average between a column and: @@ -9295,7 +9357,7 @@ def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + weight: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9330,7 +9392,7 @@ def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): pykx.FloatAtom(pykx.q('2.431111')) ``` """ - return self.call('wavg', other, iterator=iterator, col_arg_ind=col_arg_ind, + return self.call('wavg', weight, iterator=iterator, col_arg_ind=col_arg_ind, project_args=project_args) def within(self, lower, upper, iterator=None, col_arg_ind=0, project_args=None): @@ -9396,7 +9458,7 @@ def wsum(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9440,7 +9502,7 @@ def xbar(self, other, iterator=None, col_arg_ind=1, project_args=None): parameter other. Parameters: - other: An integer denoting the multiple to which all values will be rounded + other: An integer denoting the multiple to which all values will be rounded. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9498,7 +9560,7 @@ def xexp(self, other, iterator=None, col_arg_ind=1, project_args=None): the parameter other. Parameters: - other: An integer denoting the power to which all values will be raised + other: An integer denoting the power to which all values will be raised. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9538,7 +9600,7 @@ def xlog(self, other, iterator=None, col_arg_ind=1, project_args=None): by the parameter other. Parameters: - other: An integer denoting the logarithmic base to which all values will be set + other: An integer denoting the logarithmic base to which all values will be set. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9577,7 +9639,7 @@ def xprev(self, other, iterator=None, col_arg_ind=1, project_args=None): before it. Where N is specified by the parameter other. Parameters: - other: An integer denoting the number of indices before elements in the list + other: An integer denoting the number of indices before elements in the list. to retrieve the value of iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -9612,6 +9674,39 @@ def xprev(self, other, iterator=None, col_arg_ind=1, project_args=None): return self.call('xprev', other, iterator=iterator, col_arg_ind=col_arg_ind, project_args=project_args) + @property + def time(self): + """ + Retrieve the time information from a temporal column + + + Examples: + + Retrieve time information from a column + + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={'time': kx.random.random(3, kx.TimestampAtom.inf)}) + >>> tab + pykx.Table(pykx.q(' + time + ----------------------------- + 2118.11.10D05:05:45.270444032 + 2052.01.18D19:21:15.973214208 + 2088.03.13D18:56:07.388233728 + ')) + >>> tab.select(kx.Column('time').time) + pykx.Table(pykx.q(' + time + ------------ + 05:05:45.270 + 19:21:15.973 + 18:56:07.388 + ')) + ``` + """ + return self.call('`time$') + @property def hour(self): """ @@ -9787,7 +9882,7 @@ def add(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9886,11 +9981,48 @@ def name(self, name): ')) ``` """ + if not isinstance(name, (str, SymbolAtom)): + raise TypeError("'name' must be of type str or SymbolAtom") cpy = copy.deepcopy(self) cpy._name = name cpy._renamed = True return cpy + def data(self, data): + """ + Replace data of a column object + + Parameters: + data: The data to assign to the column + + Examples: + + Change the value of a column after selecting from table + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={ + ... 'a': [1, -1, 0.5], + ... 'b': [[-1, 2, 1], [0, 1, 2], [1, 2, 3]] + ... }) + >>> tab.update(kx.Column('a').data([1,2,3])) + pykx.Table(pykx.q(' + a b + -------- + 1 -1 2 1 + 2 0 1 2 + 3 1 2 3 + ')) + ``` + """ + cpy = copy.deepcopy(self) + if isinstance(data, (SymbolAtom, SymbolVector, str)): + data = [data] + elif isinstance(data, list) and all(isinstance(x, str) for x in data): + data = [data] + cpy._data = data + cpy._renamed=True + return cpy + def average(self, iterator=None): """ Calculate the average value for a column or items in a column @@ -9942,13 +10074,13 @@ def cast(self, other, iterator=None, col_arg_ind=1, project_args=None): Convert the content of a column to another PyKX type Parameters: - other: The name of the type to which your column should be cast + other: The name of the type to which your column should be cast. or the lower case letter used to define it in q, for more information see [here](https://code.kx.com/q/ref/cast/). iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -9984,6 +10116,8 @@ def cast(self, other, iterator=None, col_arg_ind=1, project_args=None): """ if not isinstance(other, str): raise QError('Supplied value other must be a str') + if other == 'symbol': + other = '' if 1 == len(other): other = other.encode('UTF-8') return self.call('$', other, iterator=iterator, col_arg_ind=col_arg_ind, @@ -9998,7 +10132,7 @@ def correlation(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10058,7 +10192,7 @@ def covariance(self, other, sample=False, iterator=None, col_arg_ind=0, project_ - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. sample: Should calculations of covariance return the sample covariance (set True) covariance (set False {default}) iterator: What iterator to use when operating on the column @@ -10129,7 +10263,7 @@ def divide(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10209,11 +10343,11 @@ def drop(self, other, iterator=None, col_arg_ind=1, project_args=None): an iterator. Where N is specified by the other parameter. Parameters: - other: An integer defining the number of elements to drop + other: An integer defining the number of elements to drop. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10247,11 +10381,11 @@ def fill(self, other, iterator=None, col_arg_ind=1, project_args=None): Replace all null values in a column with a specified 'other' parameter Parameters: - other: The value which should replace nulls within a column + other: The value which should replace nulls within a column. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10340,7 +10474,7 @@ def join(self, other, iterator=None, col_arg_ind=0, project_args=None): - A variable in q memory Parameters: - other: The Column, list, item or variable to be joined to the + other: The Column, list, item or variable to be joined to the. original column iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -10427,7 +10561,7 @@ def modulus(self, other, iterator=None, col_arg_ind=1, project_args=None): Calculate the modulus of items in a column for a given value. Parameters: - other: An integer denoting the divisor to be used when calculating the modulus + other: An integer denoting the divisor to be used when calculating the modulus. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10470,7 +10604,7 @@ def multiply(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10777,7 +10911,7 @@ def subtract(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10857,11 +10991,11 @@ def take(self, other, iterator=None, col_arg_ind=1, project_args=None): from a column using an iterator. Where N is specified by the other parameter. Parameters: - other: An integer defining the number of elements to retrieve + other: An integer defining the number of elements to retrieve. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10960,6 +11094,441 @@ def variance(self, sample=False, iterator=None): fn = 'svar' if sample else 'var' return self.call(fn, iterator=iterator) + def bin(self, other, iterator=None, col_arg_ind=1, project_args=None): + """ + Perform a binary search on the Column. + + Parameters: + other: A sorted list of same type as column. + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 1. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.select(kx.Column('n').bin([1,3,5])) + pykx.Table(pykx.q(' + n + - + 0 + 1 + ')) + ``` + """ + return self.call('bin', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def enlist(self, iterator=None): + """ + Return column values as a list. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.exec(kx.Column('n').enlist(iterator='each')) + pykx.List(pykx.q(' + 2 + 4 + ')) + ``` + """ + return self.call('enlist', iterator=iterator) + + def all(self, iterator=None): + """ + Returns a boolean atom 1b if all items are nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 5#1b)') + >>> t.exec(kx.Column('n').all()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + """ + return self.call('all', iterator=iterator) + + def any(self, iterator=None): + """ + Returns a boolean atom 1b if at least one item is nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 0010b)') + >>> t.exec(kx.Column('n').any()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + """ + return self.call('any', iterator=iterator) + + def _in(self, other, iterator=None, col_arg_ind=0, project_args=None): + """ + Returns a boolean atom 1b if at least one item is nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 4 5)') + >>> t.select(kx.Column('n')._in([0,2,4], iterator='\\:')) + pykx.Table(pykx.q(' + n + - + 0 + 1 + 0 + 1 + 0 + ')) + ``` + """ + return self.call('in', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def _except(self, other, iterator=None, col_arg_ind=0, project_args=None): + """ + Returns a list of all items of column that are not items of `other`. + + Parameters: + other: A list or atom to be excepted from the column. + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: til 5)') + >>> t.exec(kx.Column('n')._except(3)) + pykx.LongVector(pykx.q('0 1 2 4')) + ``` + """ + return self.call('except', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def hsym(self, iterator=None): + """ + Returns a list of all items of column prefixed with a colon. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: `a`b`c)') + >>> t.exec(kx.Column('n').hsym()) + pykx.SymbolVector(pykx.q('`:a`:b`:c')) + ``` + """ + return self.call('hsym', iterator=iterator) + + def raze(self, iterator=None): + """ + Returns the collapsed items of the column. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').raze()) + pykx.List(pykx.q(' + 1 + 2 + 3 4 + 5 6 + 7 + 8 + ')) + ``` + + If you want to flatten all levels use the `over` or `\\` option for iterator. + ```python + >>> t.exec(kx.Column('n').raze(iterator='over')) + pykx.LongVector(pykx.q('1 2 3 4 5 6 7 8')) + ``` + """ + return self.call('raze', iterator=iterator) + + def type(self, iterator=None): + """ + Returns the type of the column. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.ShortVector(pykx.q('7 0 -7 -7h')) + ``` + """ + return self.call('type', iterator=iterator) + + def key(self, iterator=None): + """ + Where a column is made of dictionaries, return the keys. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> d1 = kx.q('`a`b`c ! 1 2 3') + >>> d2 = kx.q('`d`e`f ! 4 5 6') + >>> t = kx.Table(data={'n':[d1, d2]}) + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.Table(pykx.q(' + n + ----- + a b c + d e f + ')) + ``` + """ + return self.call('key', iterator=iterator) + + def get(self, iterator=None): + """ + Read value from a column of named global variables or file names. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:tb1 set ([] n: `a`b`c)') + pykx.SymbolAtom(pykx.q('`:tb1')) + >>> kx.q('`:tb2 set ([] n: 1 2 3)') + pykx.SymbolAtom(pykx.q('`:tb2')) + >>> t = kx.q('([] n: `:tb1`:tb2)') + >>> t.exec(kx.Column('n').get(iterator='each')) + pykx.List(pykx.q(' + +(,`n)!,`a`b`c + +(,`n)!,1 2 3 + ')) + ``` + """ + return self.call('get', iterator=iterator) + + def read0(self, iterator=None): + """ + Read text from a file or process handle. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read0(iterator='each')) + pykx.List(pykx.q(' + "hello" + "goodbye" + ')) + ``` + """ + return self.call('read0', iterator=iterator) + + def read1(self, iterator=None): + """ + Read bytes from a file or named pipe. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read1(iterator='each')) + pykx.List(pykx.q(' + 0x68656c6c6f0a + 0x676f6f646279650a + ')) + ``` + """ + return self.call('read1', iterator=iterator) + + def next(self, iterator=None): + """ + For each item in the column, return the next item. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:3 6 9 12)') + >>> t.exec(kx.Column('n').next())) + pykx.LongVector(pykx.q('6 9 12 0N')) + ``` + """ + return self.call('next', iterator=iterator) + + def prior(self, other, iterator=None, col_arg_ind=1, project_args=None): + """ + Applies passed function to value in column and the item preceding it. + + Parameters: + other: A binary function. + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 1. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 1 2 3)') + >>> t.select(kx.Column('n').prior(kx.q('+'))) + pykx.Table(pykx.q(' + n + - + 1 + 3 + 5 + ')) + ``` + """ + return self.call('prior', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def where(self, iterator=None): + """ + Return indexes of True values. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each` + + Examples: + + Find indexes where values are True + + ```python + >>> import pykx as kx + >>> tab = kx.q('([] a:101010b)') + >>> tab.exec(kx.Column('a').where()) + pykx.LongVector(pykx.q('0 2 4')) + ``` + """ + return self.call('where', iterator=iterator) + class QueryPhrase: """Special wrapper for a list which will be treated as a QueryPhrase. @@ -10973,7 +11542,7 @@ def __init__(self, phrase, names=None, are_trees=False): elif isinstance(phrase, ParseTree): self._phrase = phrase._tree elif isinstance(phrase, Column): - self._phrase = [phrase._value] + self._phrase = [phrase._data] self._names = [phrase._name] self._are_trees = [phrase._is_tree] elif isinstance(phrase, str): @@ -10998,7 +11567,7 @@ def append(self, other): self._names.append('') self._are_trees.append(False) elif isinstance(other, Column): - self._phrase.append(other._value) + self._phrase.append(other._data) self._names.append(other._name) self._are_trees.append(other._is_tree) elif isinstance(other, QueryPhrase): @@ -11016,7 +11585,7 @@ def extend(self, other): self._names.extend('') self._are_trees.extend(False) elif isinstance(other, Column): - self._phrase.extend(other._value) + self._phrase.extend(other._data) self._names.extend(other._name) self._are_trees.extend(other._is_tree) elif isinstance(other, QueryPhrase): diff --git a/tests/qcumber_tests/memory.quke b/tests/qcumber_tests/memory.quke index 0abf799..21b7274 100644 --- a/tests/qcumber_tests/memory.quke +++ b/tests/qcumber_tests/memory.quke @@ -23,3 +23,11 @@ feature Memory Allocation tests initmem:pymem[]; do[100000;.pykx.eval"bbb(10, 100)"]; 10000 > abs initmem - pymem[] + + expect repeated getting of an objects attribute to not increase memory + .pykx.pyexec"class Foo:\n def __init__(self):\n self.a = 1\n self.b = list(range(10000))"; + f: .pykx.eval"Foo()"; + initmem:pymem[]; + do[10000;f[`:a]]; + do[10000;f[`:b]]; + 10000 > abs initmem-pymem[] diff --git a/tests/test_db.py b/tests/test_db.py index 0c3b3ac..d18014e 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -294,6 +294,7 @@ def test_compress(kx): assert compress_info['zipLevel'].py() == 8 +@pytest.mark.order(20) def test_enumerate(kx): tab = kx.Table(data={ 'date': kx.q('2015.01.01 2015.01.01 2015.01.02 2015.01.02'), @@ -313,6 +314,7 @@ def test_enumerate(kx): assert type(kx.q.value(entab1['sym'])) == kx.SymbolVector # noqa: E721 +@pytest.mark.order(21) def test_partition_count(kx): db = kx.DB(path='db') fullview = db.partition_count() @@ -325,6 +327,7 @@ def test_partition_count(kx): assert all(cache == kx.q.Q.pv) +@pytest.mark.order(22) def test_subview(kx): db = kx.DB(path='db') db.subview([kx.q('2015.01.01')]) @@ -407,6 +410,74 @@ def test_load_failure(kx): os.remove('db/test.q') +@pytest.mark.order(23) +def test_sym_enum_create_1(kx): + shutil.rmtree('db') + db = kx.DB(path='db') + N = 333 + qtab = kx.Table(data={ + 'sym': kx.random.random(N, ['AAPL', 'GOOG', 'MSFT']), + 'price': kx.random.random(N, 10.0), + 'size': kx.random.random(N, 100) + }) + db.create(qtab, 'stocks', kx.q('2020.04m'), by_field='sym', sym_enum='junk') + test_path = db.path/"junk" + assert test_path.is_file() + + +@pytest.mark.order(24) +def test_sym_enum_load_1(kx): + db2 = kx.DB() + db2.load("db") + assert db2.tables == ['stocks'] + assert db2.stocks.size == kx.toq(1332) + + +@pytest.mark.order(25) +def test_sym_enum_create_2(kx): + db = kx.DB(path='db') + N = 333 + qtab = kx.Table(data={ + 'sym': kx.random.random(N, ['AAPL', 'GOOG', 'MSFT']), + 'price': kx.random.random(N, 10.0), + 'size': kx.random.random(N, 100) + }) + db.create(qtab, 'stocks', kx.q('2020.04m'), by_field='sym') + test_path = db.path/"sym" + assert test_path.is_file() + + +@pytest.mark.order(26) +def test_sym_enum_load_2(kx): + db2 = kx.DB() + db2.load('db') + assert db2.tables == ['stocks'] + assert db2.stocks.size == kx.toq(1332) + + +@pytest.mark.order(27) +def test_sym_enum_create_3(kx): + db = kx.DB(path='db') + N = 333 + qtab = kx.Table(data={ + 'sym': kx.random.random(N, ['AAPL', 'GOOG', 'MSFT']), + 'price': kx.random.random(N, 10.0), + 'size': kx.random.random(N, 100) + }) + db.create(qtab, 'stocks', kx.q('2020.04m'), by_field='sym', format="splayed", + sym_enum="test_junk") + test_path = db.path/"test_junk" + assert test_path.is_file() + + +@pytest.mark.order(28) +def test_sym_enum_load_3(kx): + db2 = kx.DB() + db2.load('db') + assert db2.tables == ['stocks'] + assert db2.stocks.size == kx.toq(1332) + + @pytest.mark.order(-1) def test_cleanup(kx): shutil.rmtree('db') diff --git a/tests/test_ipc.py b/tests/test_ipc.py index acf7574..c61c8ef 100644 --- a/tests/test_ipc.py +++ b/tests/test_ipc.py @@ -945,6 +945,7 @@ async def test_debug_kwarg_async_global(q_port): q('.Q.sbt:.pykx_test.cache_sbt') +@pytest.mark.asyncio @pytest.mark.embedded async def test_debug_kwarg_embedded(kx, q): assert q('til 10', debug=True).py() == list(range(10)) @@ -1266,3 +1267,110 @@ async def test_async_reponse(kx, q_port): q('{t: .z.p;while[.z.p < t+00:00:02; neg[.z.w]99]}[]', reuse=False, async_response=True) with pytest.warns(UserWarning, match='Cannot use async_response=True without reuse=False.'): q('{t: .z.p;while[.z.p < t+00:00:02; neg[.z.w]99]}[]', wait=False, async_response=True) + + +def test_secure_connection_timeout(kx, q_port): + conn = kx.SecureQConnection(port=q_port, timeout=2.0, connection_timeout=1.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0) + and (info['connection_timeout']==1.0)) + + conn = conn= kx.SecureQConnection(port=q_port) + info = conn._connection_info + assert (info['port'] == q_port) + + conn = kx.SecureQConnection(port=q_port, timeout=2.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0)) + + conn = kx.SecureQConnection(port=q_port, connection_timeout=1.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['connection_timeout']==1.0)) + + conn = kx.SecureQConnection(port=q_port, timeout=2.0, connection_timeout=0.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0) + and (info['connection_timeout']==0.0)) + + conn = kx.SecureQConnection(port=q_port, timeout=2.0, connection_timeout=None) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0) + and (info['connection_timeout'] is None)) + + conn = kx.SecureQConnection(port=q_port, timeout=0.0, connection_timeout=None) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 0.0) + and (info['connection_timeout'] is None)) + + conn = kx.SecureQConnection(port=q_port, timeout=0.0, connection_timeout=0.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 0.0) + and (info['connection_timeout']==0.0)) + + +def test_connection_timeout(kx, q_port): + conn = kx.QConnection(port=q_port, timeout=2.0, connection_timeout=1.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0) + and (info['connection_timeout']==1.0)) + + conn = conn= kx.QConnection(port=q_port) + info = conn._connection_info + assert (info['port'] == q_port) + + conn = kx.QConnection(port=q_port, timeout=2.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0)) + + conn = kx.QConnection(port=q_port, connection_timeout=1.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['connection_timeout']==1.0)) + + conn = kx.QConnection(port=q_port, timeout=2.0, connection_timeout=0.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0) + and (info['connection_timeout']==0.0)) + + conn = kx.QConnection(port=q_port, timeout=2.0, connection_timeout=None) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 2.0) + and (info['connection_timeout'] is None)) + + conn = kx.QConnection(port=q_port, timeout=0.0, connection_timeout=None) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 0.0) + and (info['connection_timeout'] is None)) + + conn = kx.QConnection(port=q_port, timeout=0.0, connection_timeout=0.0) + info = conn._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 0.0) + and (info['connection_timeout']==0.0)) + + +@pytest.mark.asyncio +async def test_async_connection_timeout(kx, q_port): + async with kx.AsyncQConnection(port=q_port, connection_timeout=1.0) as q: + info = q._connection_info + assert ((info['port'] == q_port) and (info['connection_timeout'] == 1.0)) + + async with kx.AsyncQConnection(port=q_port) as q: + info = q._connection_info + assert (info['port'] == q_port) + + async with kx.AsyncQConnection(port=q_port, connection_timeout=0.0) as q: + info = q._connection_info + assert ((info['port'] == q_port) and (info['connection_timeout']==0.0)) + + async with kx.AsyncQConnection(port=q_port, connection_timeout=None) as q: + info = q._connection_info + assert ((info['port'] == q_port) and (info['connection_timeout'] is None)) + + async with kx.AsyncQConnection(port=q_port, timeout=0.0, connection_timeout=None) as q: + info = q._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 0.0) + and (info['connection_timeout'] is None)) + + async with kx.AsyncQConnection(port=q_port, timeout=0.0, connection_timeout=0.0) as q: + info = q._connection_info + assert ((info['port'] == q_port) and (info['timeout'] == 0.0) + and (info['connection_timeout']==0.0)) diff --git a/tests/test_pandas_api.py b/tests/test_pandas_api.py index 1d8c3a7..f8f43ea 100644 --- a/tests/test_pandas_api.py +++ b/tests/test_pandas_api.py @@ -935,25 +935,25 @@ def test_table_outer_merge(kx, q): assert df_res.equals(res) -def test_merge_sort(kx, q): - d = kx.q.z.D - a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) - b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) - res = a.merge(b, how="outer", on="date", sort=True) - assert all(res[0]['date'] == kx.q('enlist', d-3)) - - -def test_merge_sort_single_key(kx, q): - d = kx.q.z.D - a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) - b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) - a = kx.q.xkey('date', a) - res = a.merge(b, how="outer", on="date", sort=True) - assert all(res.keys().columns == ['date']) - res = kx.q('0!', res) - assert all(res[0]['date'] == kx.q('enlist', d-3)) - - +def test_merge_sort(kx, q): + d = kx.q.z.D + a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) + b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) + res = a.merge(b, how="outer", on="date", sort=True) + assert all(res[0]['date'] == kx.q('enlist', d-3)) + + +def test_merge_sort_single_key(kx, q): + d = kx.q.z.D + a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) + b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) + a = kx.q.xkey('date', a) + res = a.merge(b, how="outer", on="date", sort=True) + assert all(res.keys().columns == ['date']) + res = kx.q('0!', res) + assert all(res[0]['date'] == kx.q('enlist', d-3)) + + def test_cross_merge(kx, q): df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) @@ -1978,13 +1978,16 @@ def compare_q_to_pd(tab, df): def test_table_merge_asof(kx, q): - left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]}) - right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]}) + left = pd.DataFrame({"a": [1, 5, 10], "b": ["a", "a", "b"], "left_val": ["a", "b", "c"]}) + right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "b": ["a", "a", "b", "b", "a"], + "right_val": [1, 2, 3, 6, 7]}) qleft = kx.toq(left) qright = kx.toq(right) assert (pd.merge_asof(left, right, on='a') == kx.merge_asof(qleft, qright, on='a').pd()).all().all() + assert (pd.merge_asof(left, right, on='a', by="b") + == kx.merge_asof(qleft, qright, on='a', by="b").pd()).all().all() assert (pd.merge_asof(left, right, on='a') == qleft.merge_asof(qright, on='a').pd()).all().all() assert (pd.merge_asof(left, right, on='a') @@ -2658,3 +2661,13 @@ def test_merge_qjoin_errors(kx): match=r"Right Join requires a keyed table" " for the left dataset."): assert tab1.merge(tab2, how='right', q_join=True) + + +def test_fills(kx, q): + t = q('([] n:0N 2 3 0N 0N 7 0N;b:``h`h```j`;g:0N 1 0N 2 0N 3 0N)') + assert kx.q('~', t.fills(), kx.q('fills', t)) + + +def test_ungroup(kx, q): + t = q('([] n:1 2 3;nn:(1 1;2 2 2;3 3 3))') + assert kx.q('~', t.ungroup(), kx.q('ungroup', t)) diff --git a/tests/test_pykx.py b/tests/test_pykx.py index f8bde85..5820387 100644 --- a/tests/test_pykx.py +++ b/tests/test_pykx.py @@ -292,10 +292,6 @@ def test_pykx_star(): os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) -@pytest.mark.skipif( - (sys.version_info.major == 3) and (sys.version_info.minor == 8), - reason="python3.8 subprocess behavior inconsistent with newest versions" -) def test_pykx_stdout_stderr(): output = subprocess.run( (str(Path(sys.executable).as_posix()), '-c', @@ -314,10 +310,6 @@ def test_pykx_stdout_stderr(): os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) -@pytest.mark.skipif( - (sys.version_info.major == 3) and (sys.version_info.minor == 8), - reason="python3.8 subprocess behavior inconsistent with newest versions" -) def test_pykx_stdout_stderr_under_q(): subprocess.run( (str(Path(sys.executable).as_posix()), '-c', @@ -448,31 +440,31 @@ def test_kx_versions(kx): if test_vars == ('Linux', 'x86_64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Linux', 'x86_64', '4.1'): - assert kx.q.z.k == kx.q('2025.11.25') + assert kx.q.z.k == kx.q('2026.05.01') # elif test_vars == ('Linux', 'x86_64', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Linux', 'aarch64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Linux', 'aarch64', '4.1'): - assert kx.q.z.k == kx.q('2025.11.25') + assert kx.q.z.k == kx.q('2026.05.01') # elif test_vars == ('Linux', 'aarch64', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Darwin', 'x86_64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Darwin', 'x86_64', '4.1'): - assert kx.q.z.k == kx.q('2025.11.25') + assert kx.q.z.k == kx.q('2026.05.01') # elif test_vars == 'Darwin', 'x86_64', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Darwin', 'arm64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Darwin', 'arm64', '4.1'): - assert kx.q.z.k == kx.q('2025.11.25') + assert kx.q.z.k == kx.q('2026.05.01') # elif test_vars == ('Darwin', 'arm', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Windows', 'AMD64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Windows', 'AMD64', '4.1'): - assert kx.q.z.k == kx.q('2025.11.25') + assert kx.q.z.k == kx.q('2026.05.01') # elif test_vars == ('Windows', 'AMD64', '4.2'): # assert kx.q.z.k == kx.q('?') else: diff --git a/tests/test_query.py b/tests/test_query.py index 158b9fc..d36a992 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -592,6 +592,8 @@ def test_pythonic_query(kx): == table.select(columns=kx.Column('b').call('neg'))) assert all(kx.q('{select negb:neg b from x}', table) == table.select(columns=kx.Column('b', name='negb').call('neg'))) + assert all(kx.q('{select negb:neg b from x}', table) + == table.select(columns=kx.Column(name='negb', data=[kx.q('neg'), 'b']))) assert all(kx.q('{select negb:neg b from x}', table) == table.select(columns=kx.Column(name='negb', value=[kx.q('neg'), 'b']))) assert all(kx.q('{exec neg b from x}', table) @@ -756,6 +758,8 @@ def test_pythonic_query_ipc(kx, q_port): q.qsql.select('table', columns=kx.Column('b', name='negb').call('neg'))) assert q('{y~select negb:neg b from table}', q.qsql.select('table', columns=kx.Column(name='negb', value=[q('neg'), 'b']))) + assert q('{y~select negb:neg b from table}', + q.qsql.select('table', columns=kx.Column(name='negb', data=[q('neg'), 'b']))) assert q('{y~exec neg b from table}', q.qsql.exec('table', columns=kx.Column('b').call('neg'))) assert ({'asA': 'a', 'negB': [q('neg'), 'b']} == (kx.Column('a', name='asA')& kx.Column('b', name='negB').call('neg')).to_dict()) @@ -903,28 +907,28 @@ def test_column_licensed(kx): def test_fby_instance_call(kx): table = kx.q('([] x:`a`b`c;x1:1 2 3;x2:`a`e`g;x11:0 3 3;b:011b)') - assert kx.Column.fby(['c1,c2'], 'sum', table)._name == ['c1,c2'] + assert kx.Column.fby('c1', 'sum', table)._name == 'c1' with pytest.raises(RuntimeError) as err: - kx.Column('a').fby(['c1,c2'], 'sum', table) + kx.Column('a').fby('c1', 'sum', table) assert "Column object" in str(err) def test_no_inplace(kx): a = kx.Column('a') a.max() - assert a._value == 'a' + assert a._data == 'a' a.call('{x}') - assert a._value == 'a' + assert a._data == 'a' b = kx.Column('b') a | b - assert a._value == 'a' - assert b._value == 'b' + assert a._data == 'a' + assert b._data == 'b' a.name('aa') - assert a._value == 'a' + assert a._data == 'a' a & b - assert a._value == 'a' - assert b._value == 'b' + assert a._data == 'a' + assert b._data == 'b' pt = kx.ParseTree(['a']) pt.enlist() assert pt._tree == ['a'] diff --git a/tests/test_streamlit.py b/tests/test_streamlit.py index 41970cd..03cc6d9 100644 --- a/tests/test_streamlit.py +++ b/tests/test_streamlit.py @@ -7,7 +7,6 @@ import streamlit as st -@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher") def test_streamlit(kx, q_port): conn = st.connection('pykx', type=kx.streamlit.PyKXConnection, host='localhost', port=q_port) diff --git a/tests/test_toq.py b/tests/test_toq.py index bd46197..57748ff 100644 --- a/tests/test_toq.py +++ b/tests/test_toq.py @@ -3,6 +3,7 @@ import math import os from pathlib import Path +from platform import system from sys import getrefcount from uuid import UUID, uuid4 @@ -1741,12 +1742,18 @@ def test_cast_setting(kx): def test_2d_array_from_file(kx): - df = pd.read_parquet('tests/nestedFloats.parquet') + basePath = Path(__file__).parent.parent.parent + file = 'nestedFloats.parquet' + testFilePath = (basePath/file) if system() == 'Windows' else 'tests/' + file + df = pd.read_parquet(testFilePath) kx.toq(df, no_allocator=True) def test_2d_array_from_file_no_allocator(kx): - df = pd.read_parquet('tests/nestedFloats.parquet') + basePath = Path(__file__).parent.parent.parent + file = 'nestedFloats.parquet' + testFilePath = (basePath/file) if system() == 'Windows' else 'tests/' + file + df = pd.read_parquet(testFilePath) kx.toq(df, no_allocator=True) diff --git a/tests/test_util.py b/tests/test_util.py index 16d3449..1d74be0 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -5,6 +5,7 @@ from time import sleep from uuid import uuid4 +import numpy as np import pytest import toml @@ -241,3 +242,17 @@ def test_config_add_type(kx): assert data['default']['PYKX_MAX_ERROR_LENGTH'] == 1 assert data['default']['PYKX_BETA_FEATURES'] os.remove(fpath) + + +def test_serialize_deserialize(kx, q): + vec = q('til 10') + ser = kx.serialize(vec) + assert isinstance(ser, kx.serialize) + + byte = ser.copy() + arr = ser.np() + assert isinstance(byte, bytes) + assert isinstance(arr, np.ndarray) + + assert q('{x ~ til 10}', kx.deserialize(byte)).py() + assert q('{x ~ til 10}', kx.deserialize(arr)).py() diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index 5538246..d5379cc 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -14,6 +14,7 @@ from textwrap import dedent from uuid import UUID import itertools +import sys # Do not import Pandas, PyArrow, or PyKX here - use the pd/pa/kx fixtures instead! import numpy as np @@ -1677,7 +1678,10 @@ def test_index_iter(self, q): py_err = str(e) if kx_err is not None or py_err is not None: assert "is not in" in kx_err - assert "is not in" in py_err + if sys.version_info < (3, 14): + assert "is not in" in py_err + else: + assert "not in list" in py_err else: assert kx_res == py_res @@ -2092,11 +2096,8 @@ def test_ArrowUUIDType(q, kx, pa): kx.wrappers.ArrowUUIDType) assert isinstance(kx.wrappers.PandasUUIDArray(v), kx.wrappers.PandasUUIDArray) - with pytest.raises(ValueError, match=r'(?i)Cannot convert multiple chunks'): - pa.Table.from_arrays( - [pa.chunked_array([q('enlist 0Ng'), q('enlist 0Ng')])], - names=['a'] - ).to_pandas() + ca = pa.chunked_array([q('enlist 0Ng'), q('enlist 0Ng')]) + assert len(ca.chunks) == 2 def test_deserialize(kx): @@ -3503,6 +3504,17 @@ def test_nested_keyed_dict(self, q): '(`c`d)!([] metaa: (19; 14); metab: (17; 16)));' 'data: (1; 2))').py() == double_nested + def test_setitem(self, kx): + a = kx.q('`a`b!(`;2)') + c = kx.q('([] a:enlist 2;b:enlist 6)') + a['c'] = c + assert isinstance(a['c'], kx.Table) + + a = kx.q('`a`b!(`;2)') + d = kx.q('([] a:1 2;b:3 6)') + a['d'] = d + assert isinstance(a['d'], kx.Table) + class Test_KeyedTable: kt = '([k1:100+til 3] x:til 3; y:`singly`keyed`table)' @@ -5725,3 +5737,186 @@ def test_to_vec(kx): def test_atom_ufunc(kx): assert 5 == np.add(kx.LongAtom(2), kx.LongAtom(3)).py() assert 2 == np.floor(kx.FloatAtom(2.5)).py() + + +class Test_Column: + def test_name_type(self, kx): + col = kx.Column('age', data=[25, 30, 31]) + assert col._name == 'age' + col = col.name('age_upd') + assert col._name == 'age_upd' + with pytest.raises(TypeError) as err: + col.name(123) + assert "str or SymbolAtom" in str(err) + with pytest.raises(TypeError) as err: + col = kx.Column(1, data=[25, 30, 31]) + assert "str or SymbolAtom" in str(err) + + with pytest.raises(TypeError) as err: + col = kx.Column(None, [25, 30, 31]) + assert "str or SymbolAtom" in str(err) + + col = kx.Column(kx.SymbolAtom('age'), data=[25, 30, 31]) + assert col._name == 'age' + + def test_column_positional(self, kx): + t = kx.q('([] a: 1 2 3)') + tc = kx.q('([] a: 1 2 3;c:1 2 3)') + assert kx.q('~', tc, t.update(kx.Column('a', 'c'))) + + def test_column_time(self, kx): + d1 = kx.TimestampAtom(2150, 10, 22, 20, 31, 15, 70713856) + d2 = kx.TimestampAtom(2025, 11, 25, 11, 9, 13, 20120214) + tab = kx.Table(data={'a': [d1, d2]}) + + assert all(tab.exec(kx.Column('a').time) == kx.TimeVector(kx.q('20:31:15.070 11:09:13.020'))) # noqa E:501 + + def test_column_bin(self, kx): + t = kx.q('([] n: 2 4)') + assert all(t.select(kx.Column('n').bin([1, 3, 5])) == kx.q('select 1 3 5 bin n from ([] n: 2 4)')) # noqa E:501 + + with pytest.raises(kx.QError): + t.select(kx.Column('n').bin(['a', 'b', 'c'])) + + def test_column_enlist(self, kx): + t = kx.q('([] n: 2 4)') + assert all(t.select(kx.Column('n').enlist(iterator='each')) == kx.q('select enlist each n from ([] n: 2 4)')) # noqa E:501 + t2 = kx.q('([] n:((2 3);(4 5)))') + assert (t2.exec(kx.Column('n').enlist(iterator='each')) == kx.q('exec enlist each n from ([] n:((2 3);(4 5)))')).all() # noqa E:501 + + def test_column_any(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_all(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_not(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_in(self, kx): + t = kx.q('([] n: til 5)') + assert (t.exec(kx.Column('n')._in(3)) == kx.q('exec n in 3 from ([] n: til 5)')).all() + + def test_column_except(self, kx): + t = kx.q('([] n: til 5)') + assert (t.exec(kx.Column('n')._except(3)) == kx.q('exec n except 3 from ([] n: til 5)')).all() # noqa E:501 + + def test_column_hsym(self, kx): + t = kx.q('([] n: `a`b`c)') + assert (t.exec(kx.Column('n').hsym()) == kx.q('exec hsym n from ([] n : `a`b`c)')).all() + + def test_column_raze(self, kx): + t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + assert (t.exec(kx.Column('n').raze()) == kx.q('exec raze n from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + assert (t.exec(kx.Column('n').raze(iterator='over')) == kx.q('exec raze/[n] from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + + def test_column_type(self, kx): + t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + assert (t.exec(kx.Column('n').type(iterator='each')) == kx.q('exec type each n from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + + def test_column_key(self, kx): + d1 = kx.q('`a`b`c ! 1 2 3') + kx.q['d1'] = d1 + d2 = kx.q('`d`e`f ! 4 5 6') + kx.q['d2'] = d2 + t = kx.Table(data={'n': [d1, d2]}) + assert all(t.select(kx.Column('n').key(iterator='each')) == kx.q('select key each n from ([] n:(d1;d2))')) # noqa E:501 + + def test_column_get(self, kx): + kx.q('`:tb1 set (`a`b`c)') + kx.q('`:tb2 set (1 2 3)') + t = kx.q('([] n: `:tb1`:tb2)') + assert (t.exec(kx.Column('n').get(iterator='each')) == kx.q('exec get each n from ([] n: `:tb1`:tb2)')).all() # noqa E:501 + os.remove('tb1') + os.remove('tb2') + + def test_column_read0(self, kx): + kx.q('`:t1.txt 0: enlist "hello"') + kx.q('`:t2.txt 0: enlist "goodbye"') + t = kx.q('([] n:`:t1.txt`:t2.txt)') + assert (t.exec(kx.Column('n').read0(iterator='each')) == kx.q('exec read0 each n from ([]n:`:t1.txt`:t2.txt)')).all() # noqa E:501 + os.remove('t1.txt') + os.remove('t2.txt') + + def test_column_read1(self, kx): + kx.q('`:t1.txt 0: enlist "hello"') + kx.q('`:t2.txt 0: enlist "goodbye"') + t = kx.q('([] n:`:t1.txt`:t2.txt)') + assert (t.exec(kx.Column('n').read1(iterator='each')) == kx.q('exec read1 each n from ([]n:`:t1.txt`:t2.txt)')).all() # noqa E:501 + os.remove('t1.txt') + os.remove('t2.txt') + + def test_column_next(self, kx): + t = kx.q('([] n:3 6 9 12)') + assert (t.exec(kx.Column('n').call('next')) == kx.q('exec next n from ([]n: 3 6 9 12)')).all() # noqa E:501 + + def test_column_prior(self, kx): + t = kx.q('([] n:1 2 3 3 2 1)') + assert t.exec(kx.Column('n').prior(kx.q('<')) == kx.q('000011b')).all() + + def test_column_where(self, kx): + t = kx.q('([] a:101010b)') + assert (t.exec(kx.Column('a').where()) == kx.q('0 2 4')).all() + + def test_column_types(self, kx): + a = kx.Column(name='age', data=(1, 2, 3)) + assert (a._name=='age' and a._data==(1, 2, 3)) + + with pytest.raises(TypeError) as err: + kx.Column(data=(1, 2, 3)) + assert "'name' cannot be None" in str(err) + + with pytest.raises(TypeError) as err: + kx.Column(age=5, data=(1, 2, 3)) + assert "name can only be of type Str and pykx.SymbolAtom" in str(err) + + def test_column_data_str(self, kx): + t = kx.q('([] a:1 2 3)') + r = kx.q('([] a:1 2 3;n:`t`t`t)') + kx.q['s'] = 't' + kx.q['svec'] = ['t', 't', 't'] + assert kx.q('~', r, t.update(kx.Column('n', data='t'))) + assert kx.q('~', r, t.update(kx.Column('n', data=['t', 't', 't']))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.SymbolAtom('t')))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.SymbolVector(['t', 't', 't'])))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.Variable('s')))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.Variable('svec')))) + assert kx.q('~', r, t.update(kx.Column('n').data('t'))) + assert kx.q('~', r, t.update(kx.Column('n').data(['t', 't', 't']))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.SymbolAtom('t')))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.SymbolVector(['t', 't', 't'])))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.Variable('s')))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.Variable('svec')))) + + def test_column_getitem(self, kx): + t = kx.q('([] a:5 10 15 20)') + assert kx.q('~', kx.q('([] a:10 15)'), t.select(kx.Column('a')[[1, 2]])) + kx.q['myvar'] = [1, 2] + assert kx.q('~', kx.q('([] a:10 15)'), t.select(kx.Column('a')[kx.Variable('myvar')])) + + def test_column_cast(self, kx): + t = kx.q('([] a:("this";"that"))') + assert kx.q('~', kx.q('([] a:`this`that)'), t.select(kx.Column('a').cast('symbol'))) + t = kx.q('([] a:1 2 3)') + assert kx.q('~', kx.q('([] a:1 2 3i)'), t.select(kx.Column('a').cast('int')))