diff --git a/LICENSE.txt b/LICENSE.txt index 43e0d3b..d49f2bd 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -218,8 +218,8 @@ Software License for q.so KX Confidential -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -231,7 +231,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/README.md b/README.md index 08bc1b9..89b0a00 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Then install the latest version of PyKX with the following command: pip install pykx ``` -To install a specific version of PyKX run the following command replacing with a specific released semver version of the interface +To install a specific version of PyKX run the following command replacing `` with a specific released semver version of the interface ```bash pip install pykx== @@ -55,37 +55,20 @@ pip install pykx== ### PyKX License access and enablement -Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](docs/user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a kdb+ license, this can be achieved either through use of a personal evaluation license or receipt of a commercial license. +Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](docs/user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a license. -#### Personal Evaluation License +If you do not have a license go to the [KX Developer Center](https://developer.kx.com/products/kdb-x/install) to obtain a KDB-X license. +You will need to update to KDB-X Python `pykx>=4.0.0` to use the provided license. -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX - -1. Visit https://kx.com/kdb-insights-sdk-personal-edition-download/ and fill in the attached form following the instructions provided. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` - -#### Commercial Evaluation License - -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX - -1. Contact you KX sales representative or sales@kx.com requesting a trial license for PyKX evaluation. Alternately apply through https://kx.com/book-demo. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` - -__Note:__ PyKX will not operate with a vanilla or legacy kdb+ license which does not have access to specific feature flags embedded within the license. In the absence of a license with appropriate feature flags PyKX will fail to initialise with full feature functionality. +Full instructions on license installation are covered on the [Installing page](docs/getting-started/installing.md). ### Supported Environments KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX installed from wheel files. Support for user-built installations of PyKX (e.g. built from the source distribution) is only provided on a best-effort basis. Currently, PyKX provides wheels for the following environments: -- Linux (`manylinux_2_17_x86_64`) with CPython 3.8-3.11 -- macOS (`macosx_10_10_x86_64`) with CPython 3.8-3.11 -- Windows (`win_amd64`) with CPython 3.8-3.11 +- Linux (`manylinux2014_x86_64`, `manylinux2014_aarch64`) with CPython 3.9-3.14 +- macOS (`macosx_10_15_x86_64`, `macosx_10_15_arm64`) with CPython 3.9-3.14 +- Windows (`win_amd64`) with CPython 3.9-3.14 ### Dependencies @@ -93,7 +76,6 @@ KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX PyKX depends on the following third-party Python packages: -- `pandas>=1.2, <2.0; python_version=='3.8'` - `pandas>=1.2, <3.0; python_version>'3.8'` - `numpy>=1.22; python_version<'3.11'` - `numpy>=1.23; python_version=='3.11'` @@ -169,7 +151,7 @@ cd pykx pip3 install -U '.[all]' ``` -To run PyKX in licensed mode ensure to follow the steps to receive a [Personal Evaluation License](https://code.kx.com/pykx/getting-started/installing.html#personal-evaluation-license) +To run in licensed mode see [here](docs/getting-started/installing.md). Now you can run/test PyKX: diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index b3a1c31..5f2fff5 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,5 +1,4 @@ python: - - 3.8 - 3.9 - 3.10 - 3.11 diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 71a1532..205de72 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -26,8 +26,6 @@ requirements: - numpy==2.0.* # [py==311] - numpy==2.0.* # [py==310] - numpy==2.0.* # [py==39] - - numpy==1.22.* # [py==38] - - numpy==1.20.* # [py==37] - tomli>=2.0.1 - wheel>=0.36 - sysroot_linux-64 ==2.17 # [linux and py<314] @@ -37,12 +35,10 @@ requirements: run: - python - - numpy>=1.20 # [py==37] - - numpy>=1.22 # [py>37 , py<314] + - numpy>=1.22 # [py>38 , py<314] - numpy>=2.3.0 # [py==314] - - pandas>=1.2, <3.0 # [py>38 , py<314] + - pandas>=1.2 , <3.0 # [py>38 , py<314] - pandas>=2.3.3, <3.0 # [py==314] - - pandas<2.0 # [py==38] - pyarrow>=3.0.0 - pytz>=2022.1 - toml>=0.10.2 diff --git a/custom_theme/main.html b/custom_theme/main.html index 4c73cdc..bffbb07 100644 --- a/custom_theme/main.html +++ b/custom_theme/main.html @@ -8,15 +8,4 @@ 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-PG938LS'); - - - {% endblock %} diff --git a/custom_theme/partials/header.html.tpl b/custom_theme/partials/header.html.tpl index 5ede20a..fa6afdc 100644 --- a/custom_theme/partials/header.html.tpl +++ b/custom_theme/partials/header.html.tpl @@ -25,6 +25,11 @@ var currVersion = '{{ config.curr_docs_version }}'
-
    +
@@ -85,11 +90,14 @@ {% endif %} - {% if "search" in config["plugins"] %} - - {% include "partials/search.html" %} + {% if "material/search" in config.plugins %} + {% set search = config.plugins["material/search"] | attr("config") %} + {% if search.enabled %} + + {% include "partials/search.html" %} + {% endif %} {% endif %} {% if config.repo_url %}
@@ -108,23 +116,6 @@ .md-typeset span{display:none;} } /* Button used to open the chat form - fixed at the bottom of the page */ - .open-button { - background-color: var(--md-typeset-a-color); - border: none; - border-radius: 10px; - bottom: 23px; - box-sizing: border-box; - color: var(--md-default-bg-color); - cursor: pointer; - font-size: 13px; - opacity: 0.8; - padding: 10px 10px; - position: fixed; - right: 28px; - width: 160px; - z-index: 3; - } - diff --git a/docs/examples/interface-overview.ipynb b/docs/examples/interface-overview.ipynb index bd7c79a..e75bec0 100644 --- a/docs/examples/interface-overview.ipynb +++ b/docs/examples/interface-overview.ipynb @@ -574,7 +574,7 @@ "source": [ "### 3.5 Query external processes via IPC\n", "\n", - "One of the most common usage patterns in organizations with access to data in kdb+/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://kx.com/kdb-insights-sdk-personal-edition-download/).\n", + "One of the most common usage patterns in organizations with access to data in kdb+/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://code.kx.com/q/).\n", "\n", "First, set up a q/kdb+ server. Set it on port 5050 and populate it with some data in the form of a table `tab`:" ] diff --git a/docs/getting-started/installing.md b/docs/getting-started/installing.md index 9d642de..fab23e4 100644 --- a/docs/getting-started/installing.md +++ b/docs/getting-started/installing.md @@ -21,7 +21,7 @@ _This page explains how to install PyKX on your machine._ Before you start, make sure you have: -- [**Python**](https://www.python.org/downloads/) (versions 3.8-3.14) +- [**Python**](https://www.python.org/downloads/) (versions 3.9-3.14) - [**pip**](https://pypi.org/project/pip/) Recommended: a virtual environment with packages such as [venv](https://docs.python.org/3/library/venv.html) from the standard library. @@ -30,9 +30,9 @@ Recommended: a virtual environment with packages such as [venv](https://docs.pyt KX only supports versions of PyKX built by KX (installed from wheel files) for: -- **Linux** (`manylinux_2_17_x86_64`, `linux-arm64`) with CPython 3.8-3.14 -- **macOS** (`macosx_10_15_x86_64`, `macosx_10_15_arm`) with CPython 3.8-3.14 -- **Windows** (`win_amd64`) with CPython 3.8-3.14 +- **Linux** (`manylinux2014_x86_64`, `manylinux2014_aarch64`) with CPython 3.9-3.14 +- **macOS** (`macosx_10_15_x86_64`, `macosx_10_15_arm64`) with CPython 3.9-3.14 +- **Windows** (`win_amd64`) with CPython 3.9-3.14 We provide assistance to user-built installations of PyKX only on a best-effort basis. @@ -114,20 +114,20 @@ You can install PyKX from three sources: At this point you have [partial access to PyKX](../user-guide/advanced/modes.md#1a-running-in-unlicensed-mode). To gain access to all PyKX features, follow the steps in the next section, otherwise go straight to [3. Verify PyKX Installation](#3-verify-pykx-installation). -## 2. Install a kdb Insights license +## 2. Install a license -To use all PyKX functionalities, you need to download and install a kdb Insights license. +To use all PyKX functionalities, you need to download and install a license. + +If you do not have a license, it is recommended to upgrade to KDB-X Python `pykx>=4.0` and obtain a license from the [KX Developer Center](https://developer.kx.com/products/kdb-x/install). !!! Warning "Legacy kdb+/q licenses do not support all PyKX features." -There are two types of kdb Insights licenses for PyKX: personal and commercial. For either of them, you have two installation options: - - a) from Python - - b) using environment variables +!!! Warning "Newer KDB-X Python `pykx>=4.0` licenses will not work with PyKX `pykx<4.0`. Refer to [troubleshooting](../help/troubleshooting.md) for more." -### 2.a Install license in Python +### 2.a Install an existing license in Python -Follow the steps below to install a kdb Insights license for PyKX from Python: +Follow the steps below to install a license for PyKX from Python: 1. Start your Python session: @@ -146,79 +146,45 @@ Follow the steps below to install a kdb Insights license for PyKX from Python: Would you like to install a license? [Y/n]: ``` -3. Indicate whether you have access to an existing PyKX enabled license or not, type `#!python N` or press `#!python Enter` to continue with accessing a new license: +3. Indicate whether you have access to an existing PyKX enabled license or not, type `#!python Y` if you have a license: ```python Do you have access to an existing license for PyKX that you would like to use? [N/y]: ``` -4. Choose whether you wish to install a personal or commercial license, type `#!python Y` or press `#!python Enter` to choose a personal license - - ```python - Is the intended use of this software for: - [1] Personal use (Default) - [2] Commercial use - Enter your choice here [1/2]: - ``` - -5. When asked if you would like to apply for a license, type `#!python Y` or press `#!python Enter`: - - === "Personal license" - - ```bash - To apply for a PyKX license, navigate to https://kx.com/kdb-insights-sdk-personal-edition-download - Shortly after you submit your license application, you will receive a welcome email containing your license information. - Would you like to open this page? [Y/n]: - ``` - - === "Commercial license" - - ```bash - To apply for your PyKX license, contact your KX sales representative or sales@kx.com. - Alternately apply through https://kx.com/book-demo. - Would you like to open this page? [Y/n]: - ``` - -6. For personal use, complete the form to receive your welcome email. For commercial use, the license will be provided over email after the commercial evaluation process has been followed with the support of your sales representative. - -7. Choose the desired method to activate your license by typing `#!python 1`, `#!python 2`, or `#!python 3` as appropriate: +4. Choose the desired method to activate your license by typing `#!python 1` or `#!python 2` as appropriate: ```bash - Select the method you wish to use to activate your license: - [1] Download the license file provided in your welcome email and input the file path (Default) - [2] Input the activation key (base64 encoded string) provided in your welcome email - [3] Proceed with unlicensed mode - Enter your choice here [1/2/3]: + Please select the method you wish to use to activate your license: + [1] Provide the location of your license file + [2] Paste the license key ``` -8. Depending on your choice (`#!python 1`, `#!python 2`, or `#!python 3`), complete the installation by following the final step as below: +5. Depending on your choice (`#!python 1` or `#!python 2`), complete the installation by following the final steps as below: === "1" - === "Personal license" - - ```bash - Provide the download location of your license (for example, ~/path/to/kc.lic): - ``` - - === "Commercial license" - - ```bash - Provide the download location of your license (for example, ~/path/to/k4.lic): - ``` + ```bash + Provide the download location of your license (for example, ~/path/to/kc.lic): + ``` === "2" + Specify the license type: + ```bash - Provide your activation key (base64 encoded string) provided with your welcome email: + Please confirm the license type: + [1] kc.lic - The default + [2] k4.lic - Used in some scenarios ``` - === "3" + + Then paste your license key and hit enter: ```bash - No further actions needed. + Provide your kc.lic license key (base64 encoded string) : ``` -9. Validate the correct installation of your license: +6. Validate the correct installation of your license: ```python >>> kx.q.til(10) @@ -227,29 +193,25 @@ Follow the steps below to install a kdb Insights license for PyKX from Python: ### 2.b Install license with environment variables -For environment-specific flexibility, there are two ways to install your license: by using a file or by copying text. Both are sourced in your welcome email. Click on the tabs below, read the instructions, and choose the method you wish to follow: +For environment-specific flexibility, there are two ways to install your license: by using a file or by copying text. Click on the tabs below, read the instructions, and choose the method you wish to follow: !!! Note "" === "Using a file" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. - - 2. On receipt of an email from KX, download and save the license file to a secure location on your computer. + 1. Download your license file. - 3. Set an environment variable pointing to the folder with the license file. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). + 2. Set an environment variable pointing to the folder with the license file. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). * **Variable Name**: `#!bash QLIC` * **Variable Value**: `#!bash /user/path/to/folder` === "Using text" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. - - 2. On receipt of an email from KX, copy the `#!bash base64` encoded contents of your license provided in plain-text within the email. + 1. Copy the `#!bash base64` encoded contents of your license key provided. - 3. On your computer, set an environment variable `#!bash KDB_LICENSE_B64` when using a personal license or `KDB_K4LICENSE_B64` for a commercial license, pointing with the value copied in step 2. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). + 2. On your computer, set an environment variable with the contents of your `kc.lic` license key named `#!bash KDB_LICENSE_B64`. Or use `KDB_K4LICENSE_B64` if you have an older `k4.lic` license key. (Learn how to set environment variables from [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/)). * **Variable Name**: `KDB_LICENSE_B64` / `KDB_K4LICENSE_B64` - * **Variable Value**: `` + * **Variable Value**: `` To validate if you successfully installed your license with environment variables, start Python and import PyKX as follows: @@ -291,7 +253,6 @@ This command should display the installed version of PyKX. PyKX depends on the following third-party Python packages: - - `pandas>=1.2, <2.0; python_version=='3.8'` - `pandas>=1.2, <3.0; python_version>'3.8'` - `numpy>=1.22; python_version<'3.11'` - `numpy>=1.23; python_version=='3.11'` @@ -354,11 +315,11 @@ This command should display the installed version of PyKX. | Mac ARM | kdb+ 4.0 | libq.dylib | 2025.02.18 | | Mac x86 | kdb+ 4.0 | libq.dylib | 2025.02.18 | | Windows | kdb+ 4.0 | q.dll/q.lib | 2025.02.18 | -| Linux ARM | kdb+ 4.1 | libq.so | 2026.01.23 | -| Linux x86 | kdb+ 4.1 | libq.so | 2026.01.23 | -| Mac ARM | kdb+ 4.1 | libq.dylib | 2026.01.23 | -| Mac x86 | kdb+ 4.1 | libq.dylib | 2026.01.23 | -| Windows | kdb+ 4.1 | q.dll/q.lib | 2026.01.23 | +| Linux ARM | kdb+ 4.1 | libq.so | 2026.07.06 | +| Linux x86 | kdb+ 4.1 | libq.so | 2026.07.06 | +| Mac ARM | kdb+ 4.1 | libq.dylib | 2026.07.06 | +| Mac x86 | kdb+ 4.1 | libq.dylib | 2026.07.06 | +| Windows | kdb+ 4.1 | q.dll/q.lib | 2026.07.06 | | Linux ARM | Unlicensed | libe.so | 2023.11.22 | | Linux x86 | Unlicensed | libe.so | 2023.11.22 | | Mac ARM | Unlicensed | libe.so | 2023.11.22 | diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index aae05bf..16f0cdc 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -15,7 +15,7 @@ _This quickstart guide provides first time users with essential instructions for Before you start, make sure to: - [Install the PyKX library](installing.md#1-install-pykx). -- [Have a kdb Insights license](installing.md#2-install-a-kdb-insights-license). +- [Have a license](installing.md#2-install-a-license). ## 1. Import PyKX diff --git a/docs/help/troubleshooting.md b/docs/help/troubleshooting.md index 3f96ba0..c1eb53d 100644 --- a/docs/help/troubleshooting.md +++ b/docs/help/troubleshooting.md @@ -6,12 +6,11 @@ The following section outlines practical information useful when dealing with ge ### Accessing a license valid for PyKX -A number of trial and enterprise type licenses exist for q/kdb+. Not all licenses for q/kdb+ however are valid for PyKX. In particular users require access to a license which contains the feature flags **pykx** and **embedq** which provide access to the PyKX functionality. The following locations can be used for the retrieval of evaluation/personal licenses +Not all licenses for q/kdb+ are valid for PyKX. In particular users require access to a license which contains the feature flags **pykx** and **embedq** which provide access to the PyKX functionality. +Newer KDB-X licenses use shorter flags **py** and **eq**, these work with KDB-X Python `pykx>=4.0` only and not PyKX `pykx<4.0`. -- For non-commercial personal users you can access a 12 month kdb+ license with PyKX enabled [here](https://kx.com/kdb-insights-sdk-personal-edition-download). -- For commercial evaluation, contact your KX sales representative or sales@kx.com requesting a PyKX trial license. Alternately apply through https://kx.com/book-demo. - -For non-personal or non-commercial usage please contact sales@kx.com. +Go to the [KX Developer Center](https://developer.kx.com/products/kdb-x/install) to obtain a KDB-X license. +For other enquiries email sales@kx.com. Once you have access to your license you can install the license following the steps provided [here](../getting-started/installing.md) or through installation using the function `#!python kx.license.install` as follows @@ -24,6 +23,11 @@ Once you have access to your license you can install the license following the s Failure to initialize PyKX while raising an error `#!python embedq` indicates that the license you are attempting to use for PyKX in [licensed modality](../user-guide/advanced/modes.md) does not have the sufficient feature flags necessary to run PyKX. To access a license which does allow for running PyKX in this modality please following the instructions [here](#accessing-a-license-valid-for-pykx) to get a new license with appropriate feature flags. +This error can also be thrown when attempting to use a KDB-X license with PyKX. +Newer KDB-X licenses use shorter flags **py** and **eq**, these work with KDB-X Python `pykx>=4.0` only and not PyKX `pykx<4.0`. + +Check the output of `.z.l` in a q process to view the flags in your license. + ### Initialization failing with a `#!python kc.lic` error If after initially completing the installation guide for PyKX [here](../getting-started/installing.md) you receive the following error: @@ -34,6 +38,7 @@ pykx.exceptions.PyKXException: Failed to initialize embedded q. Captured output ``` It usually indicates that your license was not correctly written to disk or a license could not be found, to check that the installed license matches the license you expect. +This error can also be thrown when attempting to use a KDB-X license with PyKX. === "License file based checking" diff --git a/docs/license.md b/docs/license.md index 7421799..6114927 100644 --- a/docs/license.md +++ b/docs/license.md @@ -63,8 +63,8 @@ END OF TERMS AND CONDITIONS Software License for q.so -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -76,7 +76,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index 10f951b..bfdbb11 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -1,16 +1,439 @@ # PyKX Changelog +This changelog tracks all PyKX releases and their associated additions, fixes, and improvements. + !!! Note - The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). + For changes to PyKX within a q environment, refer to the [PyKX under q changelog](./underq-changelog.md). + +## PyKX 3.2.1 + +**Release Date** + +2026-XX-XX + +**Fixes and Improvments** + +- Updated 4.1 qlibs to 2026.07.06 for all platforms. + +## PyKX 3.2.0 + +**Release Date** + +2026-06-24 + +**Additions** + +- Added `by` keyword to `merge_asof`. + + ```python + >>> import pandas as pd + >>> left = pd.DataFrame({"a": [1, 5, 10], "b":["a", "a", "b"], "left_val": ["a", "b", "c"]}) + >>> right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "b":["a", "a", "b", "b", "a"], "right_val": [1, 2, 3, 6, 7]}) + >>> import pykx as kx + >>> qleft = kx.toq(left) + >>> qright = kx.toq(right) + >>> kx.merge_asof(qleft, qright, on='a') + pykx.Table(pykx.q(' + a b_x left_val b_y right_val + ----------------------------- + 1 a a a 1 + 5 a b b 3 + 10 b c a 7 + ')) + >>> kx.merge_asof(qleft, qright, on='a', by="b") + pykx.Table(pykx.q(' + a b left_val right_val + ----------------------- + 1 a a 1 + 5 a b 2 + 10 b c 6 + ')) + ``` + +- Added `__getitem__` to `pykx.Column` class. + + ```python + >>> t = kx.q('([] a:5 10 15 20)') + # Equivalent to qsql: select a 1 2 from t + >>> t.select(kx.Column('a')[[1,2]]) + pykx.Table(pykx.q(' + a + -- + 10 + 15 + ')) + >>> kx.q['myvar'] = [1, 2] + # Equivalent to qsql: select a myvar from t + >>> t.select(kx.Column('a')[kx.Variable('myvar')]) + pykx.Table(pykx.q(' + a + -- + 10 + 15 + ')) + >>> kx.q('mylookup:5 10 15 20!`five`ten`fifteen`twenty') + pykx.Identity(pykx.q('::')) + # Equivalent to qsql: update word:mylookup a from t + >>> t.update(kx.Column('word', data=kx.Variable('mylookup'))[kx.Column('a')]) + pykx.Table(pykx.q(' + a word + ---------- + 5 five + 10 ten + 15 fifteen + 20 twenty + ')) + ``` + +- Added `.enlist()` method to `pykx.Column` class to enlist data in a `pykx.Column` object. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.exec(kx.Column('n').enlist(iterator='each')) + pykx.List(pykx.q(' + 2 + 4 + ')) + ``` + +- Added `.all()` method to `pykx.Column` class to return `1b` when all items are nonzero. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 5#1b)') + >>> t.exec(kx.Column('n').all()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + + +- Added `.any()` method to `pykx.Column` class which returns `1b` when at least one item is nonzero. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 0010b)') + >>> t.exec(kx.Column('n').any()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + +- Added `._in()` method to `pykx.Column` class to check whether an item is in the column. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 4 5)') + >>> t.select(kx.Column('n')._in([0,2,4], iterator=r'\:')) + pykx.Table(pykx.q(' + n + - + 0 + 1 + 0 + 1 + 0 + ')) + ``` + +- Added `._except()` method to `pykx.Column` class to return column values that are not present in the supplied variable. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: til 5)') + >>> t.exec(kx.Column('n')._except(3)) + pykx.LongVector(pykx.q('0 1 2 4')) + ``` + +- Added `.hsym()` method to `pykx.Column` class which returns a column of symbol objects prefixed with a colon. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: `a`b`c)') + >>> t.exec(kx.Column('n').hsym()) + pykx.SymbolVector(pykx.q('`:a`:b`:c')) + ``` + +- Added `.raze()` method to `pykx.Column` class to collapse column values. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').raze(iterator='over')) + pykx.LongVector(pykx.q('1 2 3 4 5 6 7 8')) + ``` + +- Added `.type()` method to `pykx.Column` class to return type of values in a column. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.ShortVector(pykx.q('7 0 -7 -7h')) + ``` + +- Added `.key()` method to `pykx.Column` class to return keys from a column of dictionaries. + ```python + >>> import pykx as kx + >>> d1 = kx.q('`a`b`c ! 1 2 3') + >>> d2 = kx.q('`d`e`f ! 4 5 6') + >>> t = kx.Table(data={'n':[d1, d2]}) + >>> t.exec(kx.Column('n').key(iterator='each')) + pykx.Table(pykx.q(' + n + ----- + a b c + d e f + ')) + ``` + +- Added `.get()` method to `pykx.Column` class to read from a column of named global variables or file names. + ```python + >>> import pykx as kx + >>> kx.q('`:tb1 set ([] n: `a`b`c)') + pykx.SymbolAtom(pykx.q('`:tb1')) + >>> kx.q('`:tb2 set ([] n: 1 2 3)') + pykx.SymbolAtom(pykx.q('`:tb2')) + >>> t = kx.q('([] n: `:tb1`:tb2)') + >>> t.exec(kx.Column('n').get(iterator='each')) + pykx.List(pykx.q(' + +(,`n)!,`a`b`c + +(,`n)!,1 2 3 + ')) + ``` + +- Added `.read0()` method to `pykx.Column` class to read text from a file or process handle. + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read0(iterator='each')) + pykx.List(pykx.q(' + "hello" + "goodbye" + ')) + ``` + +- Added `.read1()` method to `pykx.Column` class to read bytes from a file or named pipe. + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read1(iterator='each')) + pykx.List(pykx.q(' + 0x68656c6c6f0a + 0x676f6f646279650a + ')) + ``` + +- Added `.next()` method to `pykx.Column` class to return the next item in a list. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:3 6 9 12)') + >>> t.exec(kx.Column('n').call('next')) + pykx.LongVector(pykx.q('6 9 12 0N')) + ``` + +- Added `.prior()` method to `pykx.Column` class to apply a function across current and previous value. + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 3 2 1)') + >>> t.update(kx.Column('n').prior(kx.q('<')).name('lessThanPrevious')) + pykx.Table(pykx.q(' + n lessThanPrevious + ------------------ + 1 0 + 2 0 + 3 0 + 3 0 + 2 1 + 1 1 + ')) + ``` + +- Added `.where()` method to `pykx.Column` class to return indexes of True values. + + ```python + >>> import pykx as kx + >>> tab = kx.q('([] a:101010b)') + >>> tab.exec(kx.Column('a').where()) + pykx.LongVector(pykx.q('0 2 4')) + ``` + +- Added `.fills()` method to `pykx.Table`, `pykx.KeyedTable`, and `pykx.Vector` classes to replace nulls with preceding non-nulls. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:0N 2 0N 3 0N;b:``h`j``;g:0N 1 0N 0N 2)') + >>> t + pykx.Table(pykx.q(' + n b g + ----- + + 2 h 1 + j + 3 + 2 + ')) + >>> t.fills() + pykx.Table(pykx.q(' + n b g + ----- + + 2 h 1 + 2 j 1 + 3 j 1 + 3 j 2 + ')) + ``` + +- Added `.ungroup()` method to `pykx.Table` and `pykx.KeyedTable` classes to ungroup nested values. + + ```python + >>> import pykx as kx + >>> t = q('([] n:1 2 3;nn:(1 1;2 2 2;3 3 3))') + >>> t + pykx.Table(pykx.q(' + n nn + ------- + 1 1 1 + 2 2 2 2 + 3 3 3 3 + ')) + >>> t.ungroup() + pykx.Table(pykx.q(' + n nn + ---- + 1 1 + 1 1 + 2 2 + 2 2 + 2 2 + 3 3 + 3 3 + 3 3 + ')) + ``` + +**Fixes and Improvements** + +- Updated 4.1 to 2026.05.01 for all platforms. +- Fixed setting the contents of `Column` objects using `data=` or `.data()` for inputs: `SymbolAtom`, `SymbolVector`, `str`, and lists of `str`. + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:1 2 3)') + >>> t.update(kx.Column('new', data='this')) + Traceback (most recent call last): + ... + pykx.exceptions.QError: this + ``` + + === "Behavior after change" + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:1 2 3)') + >>> t.update(kx.Column('new', data='this')) + pykx.Table(pykx.q(' + a new + ------ + 1 this + 2 this + 3 this + ')) + ``` + +- Fixed dictionary item assignment that could result in errors or incorrect types: + + === "Behavior prior to change" + + ```python + # Single row table saved incorrectly as dictionary + >>> a = kx.q('`a`b!(`;2)') + >>> c = kx.q('([] a:enlist 2;b:enlist 6)') + >>> a['c'] = c + >>> a['c'] + pykx.Dictionary(pykx.q(' + a| 2 + b| 6 + ')) + + # Multiple row table fails + >>> a = kx.q('`a`b!(`;2)') + >>> d = kx.q('([] a:1 2;b:3 6)') + >>> a['d'] = d + Traceback (most recent call last): + File "", line 1, in + a['d'] = d + ~^^^^^ + ... + pykx.exceptions.QError: length + ``` + + === "Behavior after change" + + ```python + >>> a = kx.q('`a`b!(`;2)') + >>> c = kx.q('([] a:enlist 2;b:enlist 6)') + >>> a['c'] = c + >>> a['c'] + pykx.Table(pykx.q(' + a b + --- + 2 6 + ')) + + >>> a = kx.q('`a`b!(`;2)') + >>> d = kx.q('([] a:1 2;b:3 6)') + >>> a['d'] = d + >>> a['d'] + pykx.Table(pykx.q(' + a b + --- + 1 3 + 2 6 + ')) + ``` + +- Removed `.pykx.i.errorList` from `astype` calls to fix `'noupdate` errors when called from a thread. +- Updated `help` on `kx.q` methods to use local documentation instead of making a web request. +- Fixed a memory leak when using `.pykx.getattr` on foreign objects in `q`. +- Improved `.cast('symbol')` on `pykx.Column` to correctly cast `Char` columns to `Symbol`. + + ```python + >>> import pykx as kx + >>> t = kx.q('([] a:("this";"that"))') + >>> t + pykx.Table(pykx.q(' + a + ------ + "this" + "that" + ')) + >>> t.select(kx.Column('a').cast('symbol')) + pykx.Table(pykx.q(' + a + ---- + this + that + ')) + ``` + +**Version Support Changes** + +- Python 3.8 is no longer supported as it is [end of life](https://devguide.python.org/versions/#unsupported-versions). ## PyKX 3.1.9 -#### Release Date +**Release Date** 2026-04-02 -### Additions +**Additions** - Added `.np()` method to `pykx.serialize` function, this returns the serialized data as a NumPy array. - Added support for calling `pykx.deserialize` on NumPy arrays of serialized `K` objects. @@ -46,32 +469,32 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.1 to 2026.04.01 for all platforms. ## PyKX 3.1.8 -#### Release Date +**Release Date** 2026-02-18 -### Additions +**Additions** - Added `connection_timeout` keyword argument to `AsyncQConnection` objects allowing the specification of a timeout for the initial connection to a `q` server. - Added the usage of `.q.Q.ens` for `sym_enum` argument in `DB` objects. ## PyKX 3.1.7 -#### Release Date +**Release Date** 2026-02-10 -### Additions +**Additions** - Added support for Python 3.14. -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.1 to 2026.01.23 for all platforms. - Added more strict type checks for naming `Column` objects, allowing only `str` and `SymbolAtom` values. @@ -121,17 +544,17 @@ ')) ``` -### Deprecations & Removals +**Deprecations & Removals** - Deprecated the `value` keyword when creating `kx.Column` objects, use `data` instead. ## PyKX 3.1.6 -#### Release Date +**Release Date** 2025-12-01 -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.1 to 2025.11.25 for all platforms. - Fixed issue where setting `sort=True` when calling `merge` on two `kx.Tables` ignored the supplied `on` parameter. @@ -170,11 +593,11 @@ ## PyKX 3.1.5 -#### Release Date +**Release Date** 2025-10-21 -### Fixes and Improvements +**Fixes and Improvements** - Availability for Intel Macs on PyPI has been restored. - Paths starting with '~' will now have it expanded to users home path in `PYKX_CONFIGURATION_LOCATION`, `Encrypt`, `DB`, and `system.load` functionality (using `os.path.expanduser`). @@ -253,28 +676,28 @@ pykx.wrappers.Dictionary ``` -### Deprecations & Removals +**Deprecations & Removals** - Creating more than one `DB` instance without specifying `overwrite=True` ## PyKX 3.1.4 -#### Release Date +**Release Date** 2025-07-17 -### Fixes and Improvements +**Fixes and Improvements** - Resolved error on import when `PYKX_THREADING` was set. - Fixed an issue where data could be corrupted in keyed columns of PyArrow backed Pandas dataframes. ## PyKX 3.1.3 -#### Release Date +**Release Date** 2025-06-12 -### Additions +**Additions** - Addition of the utility function `kx.util.delete_q_variable` which can be used to delete a named variable stored q memory within a namespace or in the global namespace @@ -391,7 +814,7 @@ - Added the current number of secondary threads being used by q to `kx.util.debug_environment()` output - Added option to pass `str` or `Path` to `pykx.install_into_QHOME` keyword `to_local_folder` -### Fixes and Improvements +**Fixes and Improvements** - Updated 4.0 to 2025.02.18 for all platforms. - Updated 4.1 to 2025.04.28 for all platforms. @@ -567,27 +990,27 @@ - PYKX_MAX_ERROR_LENGTH = 1 ``` -### Deprecations & Removals +**Deprecations & Removals** - Deprecated `kx.q.system.console_size`, use `kx.q.system.display_size` instead. ## PyKX 3.1.2 -#### Release Date +**Release Date** 2025-03-18 -### Fixes and Improvements +**Fixes and Improvements** - Fixes an issue when using `PYKX_THREADING` that could cause a segfault in some scenarios. ## PyKX 3.1.1 -#### Release Date +**Release Date** 2025-02-14 -### Fixes and Improvements +**Fixes and Improvements** - Fixed issue whereby PyKX would prompt for user inputs if a license was not found in a non-interactive session, now correctly falls back to unlicensed mode. - Fixed issue whereby PyKX could list splayed tables loaded outside of initialization of the `kx.DB` module as being supported elements of the database @@ -622,11 +1045,11 @@ ## PyKX 3.1.0 -#### Release Date +**Release Date** 2025-02-11 -### Additions +**Additions** - Added support for `Python 3.13`. - Added support for `NumPy 2.0` @@ -753,7 +1176,7 @@ pykx.LongVector(pykx.q('0 1 2 3 4')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Application of the `#!python str` function on empty PyKX objects could return unexpected results @@ -936,7 +1359,7 @@ - Error message when checking a license referenced a function `pykx.util.install_license` which is deprecated, this has now been updated to reference `pykx.license.install` -### Beta Features +**Beta Features** - Added ability for users to convert between PyKX numeric vectors or N-Dimensional Lists and PyTorch Tensor objects using the `pt` method. @@ -955,7 +1378,7 @@ ## PyKX 3.0.1 -#### Release Date +**Release Date** 2024-12-04 @@ -963,7 +1386,7 @@ PyKX 3.0.1 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. To install PyKX 3.0.1 on Mac please install from source [here](https://github.com/kxsystems/pykx). -### Additions +**Additions** - Addition of `.replace()` function to `kx.Vector` and `kx.List` objects to search for and replace items in each collection, retaining typing where appropriate. @@ -990,7 +1413,7 @@ pykx.IntVector(pykx.q('7 10 12..')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Added warning to `kx.q.system.load` and context registration when attempting to load path with a space. Can be suppressed by enabling `PYKX_SUPPRESS_WARNINGS`. - Changed `%%python` heading to `%%py` when calling Python code during `jupyter_qfirst` mode so as not to conflict with inbuilt Jupyter cell magics. @@ -1041,7 +1464,7 @@ ## PyKX 3.0.0 -#### Release Date +**Release Date** 2024-11-12 @@ -1049,7 +1472,7 @@ PyKX 3.0.0 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. To install PyKX 3.0.0 on Mac please install from source [here](https://github.com/kxsystems/pykx). -### Additions +**Additions** - Addition of functionality to allow for development of end-to-end streaming workflows consisting of data-ingestion, persistence and query. This functionality is outlined in-depth [here](../user-guide/advanced/streaming/index.md) with a worked example [here](../examples/streaming/index.md). @@ -1413,7 +1836,7 @@ - All infinity and null properties on atom classes are now usable in unlicensed mode: `null`, `inf`, `inf_pos`, `inf_neg`, `is_inf`, `is_neg_inf`, `is_pos_inf`. -### Fixes and Improvements +**Fixes and Improvements** - Previously built-in `#!python help()` python function, when called on q keywords such as `#!python kx.q.rand` or `#!python kx.q.max` was displaying the generic `wrappers` object help message. As noted in [the installation docs](../getting-started/installing.md#dependencies), this functionality requires the installation of optional libraries available through `pip install pykx[help]`. @@ -2101,24 +2524,24 @@ /usr/python/site-packages/pykx/config.py: RuntimeWarning: '-t' argument unsupported in QARGS.. ``` -### Upgrade considerations +**Upgrade considerations** - PyKX 3.0.0 is a major version update which includes changes requiring review before upgrading from 2.5.*. A page details these changes in full [here](../upgrades/2030.md) ## PyKX 2.5.5 -#### Release Date +**Release Date** 2024-11-28 -### Fixes and Improvements +**Fixes and Improvements** - PyKX Pandas dependency has been raised to allow `<=2.2.3` for Python>3.8 - PyKX Pandas dependency for Python 3.8 has been clamped to `<2.0` due to support being dropped for it by Pandas after 2.0.3. ## PyKX 2.5.4 -#### Release Date +**Release Date** 2024-10-22 @@ -2126,7 +2549,7 @@ PyKX 2.5.4 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. -### Fixes and Improvements +**Fixes and Improvements** - Resolved context interface failing to load files on Windows. @@ -2168,18 +2591,18 @@ ``` -### Version Support Changes +**Version Support Changes** - Version 2.5.4 marks the removal of support for releases to PyPI/Anaconda of Python 3.7 supported versions of PyKX ## PyKX 2.5.2 -#### Release Date +**Release Date** 2024-07-05 -### Fixes and Improvements +**Fixes and Improvements** - Converting PyKX generic lists using the keyword parameter `raw=True` would previously return incorrect results, the values received being the memory address of the individual elements of the list, this has now been resolved @@ -2287,11 +2710,11 @@ ## PyKX 2.5.1 -#### Release Date +**Release Date** 2024-06-11 -### Additions +**Additions** - Addition of methods to `pykx.Table` and `pykx.KeyedTable` objects for the application of `sql`, `select`, `exec`, `delete` and `update` queries @@ -2376,7 +2799,7 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Fix to regression in PyKX 2.5.0 where PyKX initialisation on Windows would result in a segmentation fault when using an `k4.lic` license type. - Previously users could not make direct use of `kx.SymbolicFunction` type objects against a remote process, this has been rectified @@ -2461,11 +2884,11 @@ ## PyKX 2.5.0 -#### Release Date +**Release Date** 2024-05-15 -### Additions +**Additions** - Addition of a method for `pykx.Table` objects to apply `xbar` calculations on specified columns names @@ -2621,7 +3044,7 @@ - Added `as_arrow` keyword to the `.pd()` method on PyKX Wrapped objects, using `as_arrow=True` will use PyArrow backed data types instead of the default NumPy backed data types. -### Fixes and Improvements +**Fixes and Improvements** - When importing PyKX from a source file path containing a space initialisation would fail with an `nyi` error message, this has now been resolved @@ -2924,7 +3347,7 @@ dtype: int64 ``` -### Upgrade considerations +**Upgrade considerations** - Since 2.1.0 when using Pandas >= 2.0 dataframe columns of type `datetime64[s]` converted to `DateVector` under `toq`. Now correctly converts to `TimestampVector`. See [conversion condsideratons](../user-guide/fundamentals/conversion_considerations.md#temporal-data-types) for further details. @@ -2961,27 +3384,27 @@ - Configuration option `PYKX_DISABLE_PANDAS_WARNING` has been removed. - Deprecated `.pd(raw_guids)` keyword. -### Beta Features +**Beta Features** - Addition of [streamlit](https://streamlit.io/) connection class `pykx.streamlit.Connection` to allow querying of q processes when building a streamlit application. For an example of this functionality and an introduction to it's usage see [here](../user-guide/advanced/streamlit.md). ## PyKX 2.4.2 -#### Release Date +**Release Date** 2024-04-03 -### Fixes and Improvements +**Fixes and Improvements** - Updated `libq` to 2024.03.28 for all supported OS's. ## PyKX 2.4.1 -#### Release Date +**Release Date** 2024-03-27 -### Fixes and Improvements +**Fixes and Improvements** - Previously calls to `qsql.select`, `qsql.exec`, `qsql.update` and `qsql.delete` would require multiple calls to parse the content of `where`, `columns` and `by` clauses. These have now been removed with all parsing now completed within the functional query when called via IPC or local to the Python process. - Linux x86 and Mac x86/ARM unlicensed mode `e.o` library updated to 2023.11.22. Fixes subnormals issue: @@ -3014,11 +3437,11 @@ ## PyKX 2.4.0 -#### Release Date +**Release Date** 2024-03-20 -### Additions +**Additions** - Support for q/kdb+ `4.1` documentation [here](https://code.kx.com/q/releases/ChangesIn4.1/) added as an opt-in capability, this functionality is enabled through setting `PYKX_4_1_ENABLED` environment variable. @@ -3107,7 +3530,7 @@ pykx.exceptions.QError: type ``` -### Fixes and Improvements +**Fixes and Improvements** - Added instructions for script to install Windows dependencies. - Resolved segfaults on Windows when PyKX calls Python functions under q. @@ -3172,9 +3595,9 @@ >>> conn('enlist 0Np').py() Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py + File "/home/user/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py converted_vector[i]=q('0Np') - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ raise LicenseException("run q code via 'pykx.q'") pykx.exceptions.LicenseException: A valid q license must be in a known location (e.g. `$QLIC`) to run q code via 'pykx.q'. ``` @@ -3283,7 +3706,7 @@ >>> tab1.merge(tab2_keyed, how='left', q_join=True) ``` -### Beta Features +**Beta Features** - Addition of `Compress` and `Encrypt` classes to allow users to set global configuration and for usage within Database partition persistence. @@ -3326,22 +3749,22 @@ ## PyKX 2.3.2 -#### Release Date +**Release Date** 2024-02-12 -### Fixes and Improvements +**Fixes and Improvements** - Update of PyKX 4.0 linux shared object to version 2024.02.09, this update is to facilitate deployments on more secure linux/linux-arm environments. - Update `Table.rename()` to skip over columns not in table instead of throwing error to match `pandas`. ## PyKX 2.3.1 -#### Release Date +**Release Date** 2024-02-07 -### Fixes and Improvements +**Fixes and Improvements** - Python functions saved to q would error if passed `''` or `'.'`. These now pass without issue. @@ -3355,7 +3778,7 @@ >>> kx.q('func', '') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -3363,7 +3786,7 @@ >>> kx.q('func', '.') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -3432,7 +3855,7 @@ - `df.select_dtypes()` updated to now accept `kx.*Atom` values for `include`/`exclude` params. Use of `kx.CharVector` will return error. - To align with other areas of PyKX the `upsert` and `insert` methods for PyKX tables and keyed tables now support the keyword argument `inplace`, this change will deprecate usage of `replace_self` with the next major release of PyKX. -### Beta Features +**Beta Features** - Addition of the concept of `Remote Function` execution to PyKX, this allows users, from a Python session to define Python functions which will be executed on a remote q/kdb+ server running PyKX under q. The intention with this feature is to allow onboarding of Python first operations within existing or q/kdb+ first infrastructures @@ -3450,11 +3873,11 @@ ## PyKX 2.3.0 -#### Release Date +**Release Date** 2024-01-22 -### Additions +**Additions** - PyKX now supports the use of `KDB_LICENSE_B64` or `KDB_K4LICENSE_B64` configuration values/environment variables to define the content of a `kc.lic` or `k4.lic` license respectively if no license is found on initial usage of PyKX. - Shortcut provided for access to current date, time and timestamp information using `'today'` and `'now'`. @@ -3513,7 +3936,7 @@ - `%%q` Jupyter Notebook magic adds `--debug` option which prints the q backtrace if the cell execution fails. - Release 2.3.0 adds to PyKX the concept of Beta features, these features are available to users through setting the configuration/environment variable `PYKX_BETA_FEATURES`. For more information on Beta features see further documentation [here](../beta-features/index.md) -### Fixes and Improvements +**Fixes and Improvements** - `%%q` Jupyter Notebook magic now returns all outputs up to and including an error when thrown. Previously only the error was returned. - `%%q` Jupyter Notebook magic ignores accidental whitespace in execution options. Below example no longer fails with `Received unknown argument` error: @@ -3545,7 +3968,7 @@ - Improved error message for conversion failures. - Fixes an issue where a user would receive a length error when attempting to apply `min`, `max`, `prod` and `sum` functions on `pykx.KeyedTable` objects. -### Beta Features +**Beta Features** - Database Management functionality has been added for the creation, loading and maintenance of PyKX Partitioned Databases. A full worked example of this functionality can be found [here](../examples/db-management.ipynb) along with full API documentation which includes examples of each function [here](../api/db.md). The API includes but is not limited to the following: @@ -3563,11 +3986,11 @@ ## PyKX 2.2.3 -#### Release Date +**Release Date** 2024-01-11 -### Fixes and Improvements +**Fixes and Improvements** - PyKX now raises an error appropriately when failing to locate `msvcr100.dll` when loading on Windows. - Config values now default to `False` when not set rather than `None`. @@ -3585,11 +4008,11 @@ Please skip this release and use 2.2.3 or newer. This is due to potential segfaults when reading compressed files. -#### Release Date +**Release Date** 2023-12-12 -### Fixes and Improvements +**Fixes and Improvements** - Conversions between `UUID` and `pykx.GUID` types could produce invalid results under various conditions in both licensed and unlicensed mode. - A regression in 2.2.1 resulted in `SIGINT` signals being incorrectly treated as `SIGTERM` style signals, PyKX now resets all signals overwritten by PyKX to their values prior to import. @@ -3601,11 +4024,11 @@ Please skip this release and use 2.2.3 or newer. This is due to potential segfaults when reading compressed files. -#### Release Date +**Release Date** 2023-11-30 -### Fixes and Improvements +**Fixes and Improvements** - Some messages to `stdout` were not being captured when redirecting. Now all are captured. - Deprecation of internally used environment variable `UNDER_PYTHON` which has been replaced by `PYKX_UNDER_PYTHON` to align with other internally used environment variables. @@ -3742,17 +4165,17 @@ ``` -### Upgrade considerations +**Upgrade considerations** - If dependent on the environment variable `UNDER_PYTHON` please upgrade your code to use `PYKX_UNDER_PYTHON` ## PyKX 2.2.0 -#### Release Date +**Release Date** 2023-11-09 -### Additions +**Additions** - Addition of `agg` method for application of aggregation functions on `pykx.Table` and `pykx.GroupbyTable` objects @@ -3968,7 +4391,7 @@ ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Fix to allow users to use Python functions when operating on a `pykx.GroupbyTable` with an `apply` function @@ -4062,11 +4485,11 @@ ## PyKX 2.1.2 -#### Release Date +**Release Date** 2023-10-24 -### Fixes and Improvements +**Fixes and Improvements** - Fix to issue where functions retrieved using the Context Interface with names `update/delete/select/exec` would result in an `AttributeError` @@ -4096,11 +4519,11 @@ ## PyKX 2.1.1 -#### Release Date +**Release Date** 2023-10-10 -### Fixes and Improvements +**Fixes and Improvements** - Fix to regression in PyKX 2.1.0 where execution of `from pykx import *` would result in the following behavior @@ -4112,11 +4535,11 @@ ## PyKX 2.1.0 -#### Release Date +**Release Date** 2023-10-09 -### Additions +**Additions** - Added functionality to the CSV Reader to allow for the input of data structures while defining column types. For example, the following reads a CSV file and specifies the types of the three columns named `x1`, `x2` and `x3` to be of type `Integer`, `GUID` and `Timestamp`. @@ -4173,7 +4596,7 @@ the following reads a CSV file and specifies the types of the three columns name the expected type a warning will be raised and the `DateVector` type will be used as a default. -### Fixes and Improvements +**Fixes and Improvements** - Empty PyKX keyed tables can now be converted to Pandas DataFrames, previously this would raise a `ValueError` @@ -4244,17 +4667,17 @@ the following reads a CSV file and specifies the types of the three columns name - Users with [`QINIT`](https://code.kx.com/q/basics/by-topic/#environment) environment variable would previously load twice on initialization within PyKX - Users installing PyKX under q on Windows had been missing installation of required files using `pykx.install_into_QHOME()` -### Dependency Updates +**Dependency Updates** - The version of `Cython` used to build `PyKX` was updated to the full `3.0.x` release version. ## PyKX 2.0.1 -#### Release Date +**Release Date** 2023-09-21 -### Fixes and Improvements +**Fixes and Improvements** - User input based license initialization introduced in 2.0.0 no longer expects user input when operating in a non-interactive modality, use of PyKX in this mode will revert to previous behavior - Use of the environment variables `QARGS='--unlicensed'` or `QARGS='--licensed'` operate correctly following regression in 2.0.0 @@ -4262,7 +4685,7 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 2.0.0 -#### Release Date +**Release Date** 2023-09-18 @@ -4270,7 +4693,7 @@ the following reads a CSV file and specifies the types of the three columns name - Pandas API functionality is enabled permanently which will modify data indexing and retrieval of `pykx.Table` objects. Users should ensure to review and test their codebase before upgrading. - EmbedPy replacement functionality for PyKX under q is now non-beta for Linux and MacOS installations, see [here](underq-changelog.md) for full information on 2.0.0 changelog. -### Additions +**Additions** - [Pandas API](../user-guide/advanced/Pandas_API.ipynb) is enabled by default allowing users to treat PyKX Tables similarly to Pandas Dataframes for a limited subset of Pandas like functionality. As a result of this change the environment variable `PYKX_ENABLE_PANDAS_API` is no longer required. - Addition of file based configuration setting allowing users to define profiles for various PyKX modalities through definition of the file `.pykx.config` see [here](../user-guide/configuration.md) for more information. @@ -4337,7 +4760,7 @@ the following reads a CSV file and specifies the types of the three columns name ')) ``` -### Fixes and Improvements +**Fixes and Improvements** - Update to environment variable definitions in all cases to be prefixed with `PYKX_*` - Return of Pandas API functions `dtypes`, `columns`, `empty`, `ndim`, `size` and `shape` return `kx` objects rather than Pythonic objects @@ -4408,21 +4831,21 @@ the following reads a CSV file and specifies the types of the three columns name pykx.q('1b') ``` -### Breaking Changes +**Breaking Changes** - Pandas API functionality is enabled permanently which will modify data indexing and retrieval. Users should ensure to review and test their codebase before upgrading. ## PyKX 1.6.3 -#### Release Date +**Release Date** 2023-08-18 -### Additions +**Additions** - Addition of argument `return_info` to `pykx.util.debug_environment` allowing user to optionally return the result as a `str` rather than to stdout -### Fixes and Improvements +**Fixes and Improvements** - Fixed Pandas API use of `ndim` functionality which should return `2` when interacting with tables following the expected Pandas behavior. - Fixed an error when using the Pandas API to update a column with a `Symbols`, `Characters`, and `Generic Lists`. @@ -4431,17 +4854,17 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.2 -#### Release Date +**Release Date** 2023-08-15 -### Additions +**Additions** - Added `to_local_folder` kwarg to `install_into_QHOME` to enable use of `pykx.q` without write access to `QHOME`. - Added [an example](../examples/threaded_execution/threading.md) that shows how to use `EmbeddedQ` in a multithreaded context where the threads need to modify global state. - Added [PYKX_NO_SIGINT](../user-guide/configuration.md#environment-variables) environment variable. -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue causing a crash when closing `QConnection` instances on Windows. - Updated q 4.0 libraries to 2023.08.11. Note: Mac ARM release remains on 2022.09.30. @@ -4451,18 +4874,18 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.1 -#### Release Date +**Release Date** 2023-07-19 -### Additions +**Additions** - Added `sorted`, `grouped`, `parted`, and `unique`. As methods off of `Tables` and `Vectors`. - Added `PyKXReimport` class to allow subprocesses to reimport `PyKX` safely. - Also includes `.pykx.safeReimport` in `pykx.q` to allows this behavior when running under q as well. - Added environment variables to specify a path to `libpython` in the case `pykx.q` cannot find it. -### Fixes and Improvements +**Fixes and Improvements** - Fixed memory leaks within the various `QConnection` subclasses. - Added deprecation warning around the discontinuing of support for Python 3.7. @@ -4476,11 +4899,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.6.0 -#### Release Date +**Release Date** 2023-06-16 -### Additions +**Additions** - Added `merge_asof` to the Pandas like API. - See [here](../user-guide/advanced/Pandas_API.ipynb#tablemerge_asof) for details of supported keyword arguments and limitations. @@ -4495,7 +4918,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added the ability to use `kx.RawQConnection` as a Python based `q` server using `kx.RawQConnection(port=x, as_server=True)`. - More documentation around using this functionality can be found [here](../examples/server/server.md). -### Fixes and Improvements +**Fixes and Improvements** - Improved error on Windows if `msvcr100.dll` is not found - Updated q libraries to 2023.04.17 @@ -4528,50 +4951,50 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.5.3 -#### Release Date +**Release Date** 2023-05-18 -### Additions +**Additions** - Added support for Pandas `Float64Index`. - Wheels for ARM64 based Macs are now available for download. ## PyKX 1.5.2 -#### Release Date +**Release Date** 2023-04-30 -### Additions +**Additions** - Added support for ARM 64 Linux. ## PyKX 1.5.1 -#### Release Date +**Release Date** 2023-04-28 -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue with `pykx.q` that caused errors to not be raised properly under q. - Fixed an issue when using `.pykx.get` and `.pykx.getattr` that caused multiple calls to be made. ## PyKX 1.5.0 -#### Release Date +**Release Date** 2023-04-17 -### Additions +**Additions** - Added wrappers around various `q` [system commands](https://code.kx.com/q/basics/syscmds/). - Added `merge` method to tables when using the `Pandas API`. - Added `mean`/`median`/`mode` functions to tables when using the `Pandas API`. - Added various functions around type conversions on tables when using the `Pandas API`. -### Fixes and Improvements +**Fixes and Improvements** - Fix to allow GUIDs to be sent over IPC. - Fix an issue related to IPC connection using compression. @@ -4585,21 +5008,21 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.4.2 -#### Release Date +**Release Date** 2023-03-08 -### Fixes and Improvements +**Fixes and Improvements** - Fixed an issue that would cause `EmbeddedQ` to fail to load. ## PyKX 1.4.1 -#### Release Date +**Release Date** 2023-03-06 -### Fixes and Improvements +**Fixes and Improvements** - Added constructors for `Table` and `KeyedTable` objects to allow creation of these objects from dictionaries and list like objects. - Fixed a memory leak around calling wrapped `Foreign` objects in `pykx.q`. @@ -4607,11 +5030,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.4.0 -#### Release Date +**Release Date** 2023-01-23 -### Additions +**Additions** - Addition of a utility function `kx.ssl_info()` to retrieve the SSL configuration when running in unlicensed mode (returns the same info as kx.q('-26!0') with a license). - Addition of a utility function `kx.schema.builder` to allow for the generation of `pykx.Table` and `pykx.KeyedTable` types with a defined schema and zero rows, this provides an alternative to writing q code to create an empty table. @@ -4624,7 +5047,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added environment variable `PYKX_Q_LOCK=1` to enable a Mutex Lock around calls into q, setting this environment variable to a number greater than 0 will set the max length in time to block before raising an error, a value of '-1' will block indefinitely and will not error, any other value will cause an error to be raised immediately if the lock cannot be acquired. - Added `insert` and `upsert` methods to `Table` and `KeyedTable` objects. -### Fixes and Improvements +**Fixes and Improvements** - Fixed `has_nulls` and `has_infs` properties for subclasses of `k.Collection`. - Improved error output of `kx.QConnection` objects when an error is raised within the context interface. @@ -4634,28 +5057,28 @@ the following reads a CSV file and specifies the types of the three columns name - Fixed conversion of null `GUIDAtom`'s to and from Numpy types. - Improved performance of converting `q` enums to pandas Categoricals. -### Beta Features +**Beta Features** - Added support for a Pandas like API around `Table` and `KeyedTable` instances, documentation for the specific functionality can be found [here](../user-guide/advanced/Pandas_API.ipynb). - Added `.pykx.setdefault` to `pykx.q` which allows the default conversion type to be set without using environment variables. ## PyKX 1.3.2 -#### Release Date +**Release Date** 2023-01-06 -### Features and Fixes +**Features and Fixes** - Fixed support for using TLS with `SyncQConnection` instances. ## PyKX 1.3.1 -#### Release Date +**Release Date** 2022-11-16 -### Features and Fixes +**Features and Fixes** - Added environment variable `PYKX_Q_LIB_LOCATION` to specify a path to load the PyKX q libraries from. - Required files in this directory @@ -4674,11 +5097,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.3.0 -#### Release Date +**Release Date** 2022-10-20 -### Features and Fixes +**Features and Fixes** - Support for converting `datetime.datetime` objects with time zone information into `pykx.TimestampAtom`s and `pykx.TimestampVector`s. - Added a magic command to run cells of q code in a Jupyter Notebook. The addition of `%%q` at the start of a Jupyter Notebook cell will allow a user to execute q code locally similarly to loading a q file. @@ -4687,38 +5110,38 @@ the following reads a CSV file and specifies the types of the three columns name - Fix to memory leak seen when converting Pandas Dataframes to q tables. - Removed unnecessary copy when sending `q` objects over IPC. -### Beta Features +**Beta Features** - EmbedPy replacement functionality `pykx.q` updated significantly to provide parity with embedPy from a syntax perspective. Documentation of the interface [here](../pykx-under-q/intro.md) provides API usage. Note that initialization requires the first version of Python to be retrieved on a users `PATH` to have PyKX installed. Additional flexibility with respect to installation location is expected in `1.4.0` please provide any feedback to `pykx@kx.com` ## PyKX 1.2.2 -#### Release Date +**Release Date** 2022-10-01 -### Features and Fixes +**Features and Fixes** - Fixed an issue causing the timeout argument for `QConnection` instances to not work properly. ## PyKX 1.2.1 -#### Release Date +**Release Date** 2022-09-27 -### Features and Fixes +**Features and Fixes** - Added support for OpenSSLv3 for IPC connections created when in 'licensed' mode. - Updated conversion functionality for timestamps to support conversions within Pandas 1.5.0 ## PyKX 1.2.0 -#### Release Date +**Release Date** 2022-09-01 -### Features and Fixes +**Features and Fixes** - Support for converting any python type to a `q` Foreign object has been added. - Support for converting Pandas categorical types into `pykx.EnumVector` type objects. @@ -4734,7 +5157,7 @@ the following reads a CSV file and specifies the types of the three columns name - Fixed conversion of `None` Python objects into analogous null `K` types if a `ktype` is specified. - Added `event_loop` parameter to `pykx.AsyncQConnection` that takes a running event loop as a parameter and allows the event loop to manage `pykx.QFuture` objects. -### Beta Features +**Beta Features** - Added extra functionality to `pykx.q` related to the calling and use of python foreign objects directly within a `q` process. - Support for [NEP-49](https://numpy.org/neps/nep-0049.html), which allows Numpy arrays to be converted into `q` Vectors without copying the underlying data. This behavior is opt-in and you can do so by setting the environment variable `PYKX_ALLOCATOR` to 1, "1" or True or by adding the flag `--pykxalloc` to the `QARGS` environment variable. Note: This feature also requires a python version of at least 3.8. @@ -4742,25 +5165,25 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.1.1 -#### Release Date +**Release Date** 2022-06-13 -### Features & Fixes +**Features & Fixes** - Added ability to skip symlinking `$QHOME` to `PyKX`'s local `$QHOME` by setting the environment variable `IGNORE_QHOME`. ## PyKX 1.1.0 -#### Release Date +**Release Date** 2022-06-07 -### Dependencies +**Dependencies** - The dependency on the system library `libcurl` has been made optional for Linux. If it is missing on Linux, a warning will be emitted instead of an error being raised, and the KX Insights Core library `kurl` will not be fully loaded. Windows and macOS are unaffected, as they don't support the KX Insights Core features to begin with. -### Features & Fixes +**Features & Fixes** - Splayed and partitioned tables no longer emit warnings when instantiated. - Added `pykx.Q.sql`, which is a wrapper around [KXI Core SQL](https://code.kx.com/insights/core/sql.html#sql-language-support). @@ -4787,15 +5210,15 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.0.1 -#### Release Date +**Release Date** 2022-03-18 -### Deprecations & Removals +**Deprecations & Removals** - The `sync` parameter for `pykx.QConnection` and `pykx.QConnection.__call__` has been renamed to the less confusing name `wait`. The `sync` parameter remains, but its usage will result in a `DeprecationWarning` being emitted. The `sync` parameter will be removed in a future version. -### Features & Fixes +**Features & Fixes** - Updated to stable classifier (`Development Status :: 5 - Production/Stable`) in project metadata. Despite this update being done in version 1.0.1, version 1.0.0 is still the first stable release of PyKX. - PyKX now provides source distributions (`sdist`). It can be downloaded from PyPI using `pip download --no-binary=:all: --no-deps pykx`. As noted in [the installation docs](../getting-started/installing.md#supported-environments), installations built from the source will only receive support on a best-effort basis. - Fixed Pandas NaT conversion to q types. Now `pykx.toq(pandas.NaT, ktype=ktype)` produces a null temporal atom for any given `ktype` (e.g. `pykx.TimeAtom`). @@ -4810,11 +5233,11 @@ the following reads a CSV file and specifies the types of the three columns name ## PyKX 1.0.0 -#### Release Date +**Release Date** 2022-02-14 -### Migration Notes +**Migration Notes** To switch from Pykdb to PyKX, you will need to update the name of the dependency from `pykdb` to `pykx` in your `pyproject.toml`/`requirements.txt`/`setup.cfg`/etc. When Pykdb was renamed to PyKX, its version number was reset. The first public release of PyKX has the version number 1.0.0, and will employ [semantic versioning](https://semver.org/). @@ -4848,7 +5271,7 @@ kx.QARGS # List of arguments provided to the embedded q instance at startup, set You can no longer rely on the [context](../api/pykx-execution/ctx.md) being reset to the global context after each call into embedded q, however IPC calls are unaffected. -### Renames +**Renames** - Pykdb has been renamed to PyKX. `Pykdb` -> `PyKX`; `PYKDB` -> `PYKX`; `pykdb` -> `pykx`. - The `adapt` module has been renamed to `toq`, and it can be called directly. Instead of `pykdb.adapt.adapt(x)` one should write `pykx.toq(x)`. - The `k` module has been renamed to `wrappers`. All wrapper classes can be accessed from the top-level, i.e. `pykx.K`, `pykx.SymbolAtom`, etc. @@ -4858,18 +5281,18 @@ You can no longer rely on the [context](../api/pykx-execution/ctx.md) being rese - The `Connection` class in the IPC module has been renamed to `QConnection`, and is now available at the top-level, i.e. `pykx.QConnection`. - The q type wrapper `DynamicLoad` has been renamed to `Foreign` (`pykdb.k.DynamicLoad` -> `pykx.Foreign`). -### Deprecations & Removals +**Deprecations & Removals** - The `pykdb.q.ipc` attribute has been removed. The IPC module can be accessed directly instead at `pykx.ipc`, but generally one will only need to access the `QConnection` class, which can be accessed at the top-level: `pykx.QConnection`. - The `pykdb.q.K` attribute has been removed. Instead, `K` types can be used as constructors for that type by leveraging the `toq` module. For example, instead of `pykdb.q.K(x)` one should write `pykx.K(x)`. Instead of `pykx.q.K(x, k_type=pykx.k.SymbolAtom)` one should write `pykx.SymbolAtom(x)` or `pykx.toq(x, ktype=pykx.SymbolAtom)`. - Most `KdbError`/`QError` subclasses have been removed, as identifying them is error prone, and we are unable to provide helpful error messages for most of them. - The `pykx.kdb` singleton class has been removed. -### Dependencies +**Dependencies** - More Numpy, Pandas, and PyArrow versions are supported. Current `pandas~=1.0`, `numpy~=1.20,<1.22`, and `pyarrow>=3.0.0` are supported. PyArrow remains an optional dependency. - A dependency on `find-libpython~=0.2` was added. This is only used when running PyKX under a q process (see details in the section below about new alpha features). - A dependency on the system library `libcurl` was added for Linux. This dependency will be made optional in a future release. -### Features & Fixes +**Features & Fixes** - The `pykx.Q` class has been added as the base class for `pykx.EmbeddedQ` (the class for `pykx.q`) and `pykx.QConnection`. - The `pykx.EmbeddedQ` process now persists its [context](../api/pykx-execution/ctx.md) between calls. - The console now works over IPC. @@ -4894,13 +5317,13 @@ You can no longer rely on the [context](../api/pykx-execution/ctx.md) being rese - Misc other bug fixes. - Misc doc improvements. -### Performance Improvements +**Performance Improvements** - Converting nested lists from q to Python is much faster. - Internally, PyKX now calls q functions with arguments directly instead of creating a `pykx.Function` instance then calling it. This results in modest performance benefits in some cases. - The context interface no longer loads every element of a context when the context is first accessed, thereby removing the computation spike, which could be particularly intense for large q contexts. -### New Alpha Features +**New Alpha Features** !!! danger "Alpha features are subject to change" diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index e5b3e51..144b45c 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -1,28 +1,38 @@ # PyKX under q Changelog -This changelog provides updates from PyKX 2.0.0 and above, for information relating to versions of PyKX prior to this version see the changelog linked below. +This changelog covers PyKX under q from version 2.0.0 onwards. !!! Note - The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). + For changes to PyKX within a Python environment, refer to the [Python changelog](./changelog.md). + +## PyKX 3.2.0 + +**Release Date** + +2026-06-24 + +**Fixes and Improvements** + +- Fixed a memory leak when using `.pykx.getattr` on foreign object in `q`. ## PyKX 3.1.5 -#### Release Date +**Release Date** 2025-10-21 -### Fixes and Improvements +**Fixes and Improvements** - Added `.pykx.noalloc` which can be used to wrap a foreign object. When used with `.pykx.toq` the conversion will not use the `PYKX_ALLOCATOR`. See [here](../help/issues.md#known-issues) for details. ## PyKX 3.1.4 -#### Release Date +**Release Date** 2025-07-17 -### Fixes and Improvements +**Fixes and Improvements** - Resolved `object has no attribute 't'` error for certain conversions @@ -49,11 +59,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.1.3 -#### Release Date +**Release Date** 2025-06-12 -### Fixes and Improvements +**Fixes and Improvements** - Dashboards integration now lists missing libraries by name. @@ -78,17 +88,17 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat - Resolved `double free or corruption (out)` error when `pykx.q` was loaded in `QINIT` or `q.q`. - `pykx.q` load time has been roughly halved. -### Deprecations & Removals +**Deprecations & Removals** - `.pykx.console[]` has been removed on Windows due to incompatibility. Will now error with `'.pykx.console is not available on Windows` if called. ## PyKX 3.1.1 -#### Release Date +**Release Date** 2025-02-14 -### Fixes and Improvements +**Fixes and Improvements** - Fix throwing of errors for `.pykx.safeReimport`, now throws an error instead of returning a value. @@ -116,11 +126,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.1.0 -#### Release Date +**Release Date** 2025-02-11 -### Additions +**Additions** - Addition of `.pykx.typepy` which returns an objects datatype as a `CharVector` after being passed to python. @@ -133,7 +143,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat "" ``` -### Fixes and Improvements +**Fixes and Improvements** - Using `.pykx.toq`/`.pykx.toq0` now return the q representation of an object when passed a wrapped type conversion object @@ -189,7 +199,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat - When attempting to load PyKX after embedPy has already been loaded, an error will be thrown and PyKX will not continue to load. -### Beta Features +**Beta Features** - Added ability for users to convert between PyKX numeric vectors or N-Dimensional Lists and PyTorch Tensor objects using the `.pykx.topt` function. @@ -200,11 +210,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.0.1 -#### Release Date +**Release Date** 2024-12-04 -### Additions +**Additions** - Added a function `.pykx.loadPy` which can be used to execute/load a Python file following expected Python syntax, this removes limitations which exist due to use of q parsing syntax when loading a Python file using `system"l /path/to/file.p"` for example @@ -222,16 +232,16 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 3.0.0 -#### Release Date +**Release Date** 2024-11-12 -### Additions +**Additions** - Added `cloud_libraries` kwarg to `install_into_QHOME` allowing installation of the kdb Insights cloud libraries to QHOME. - Addition of support for new environment variable `PYKX_USE_FIND_LIBPYTHON` which will use the Python package [`find_libpython`](https://pypi.org/project/find-libpython/) to specify the location from which `libpython.[so|dll]` will be taken. -### Fixes and Improvements +**Fixes and Improvements** - Addition of function `.pykx.toq0` to support conversion of Python strings to q strings rather than q symbols as is the default behavior @@ -293,21 +303,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.5.4 -#### Release Date +**Release Date** 2024-10-22 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.util.loadfile` now loads a file using it's full path unless it contains a space. This is to avoid issues loading scripts which are sensitive to their working directory. ## PyKX 2.5.3 -#### Release Date +**Release Date** 2024-08-20 -### Fixes and Improvements +**Fixes and Improvements** - Previously PyKX conversions of generic lists (type 0h) would convert this data to it's `raw` representation rather than it's `python` representation as documented. This had the effect of restricting the usability of some types within PyKX under q in non-trivial use-cases. With the `2.5.2` changes to more accurately represent `raw` data at depth this became more obvious as an issue. @@ -329,21 +339,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.5.0 -#### Release Date +**Release Date** 2024-05-15 -### Fixes and Improvements +**Fixes and Improvements** - When loading PyKX under from a source file path containing a space initialisation would fail with an `nyi` error message, this has now been resolved. ## PyKX 2.4.1 -#### Release Date +**Release Date** 2024-03-27 -### Fixes and Improvements +**Fixes and Improvements** - When loading PyKX under q users who had previously loaded [embedPy](https://github.com/KxSystems/embedPy) into their process would cause a segfault of unspecified origin. With this release we have added a warning prior to loading of PyKX which specifies that if a value of `.p.e` has been specified which does not match that expected of PyKX a user should consider installing PyKX under q fully: @@ -359,11 +369,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.3.1 -#### Release Date +**Release Date** 2024-02-07 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.eval` is now variadic allowing an optional second parameter to be passed to define return type. Previously would error with `rank`. @@ -424,11 +434,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.3.0 -#### Release Date +**Release Date** 2024-01-22 -### Fixes and Improvements +**Fixes and Improvements** - A bug was fixed when using `.pykx.console`, it is now possible to access python variables set using the console with `.pykx.(eval|pyexec|pyeval)` functions. @@ -455,21 +465,21 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.2.2 -#### Release Date +**Release Date** 2023-12-07 -### Fixes and Improvements +**Fixes and Improvements** - When loaded in a q process loading `pykx.q` would not allow `Ctrl+C` (SIGINT) interruption. ## PyKX 2.2.1 -#### Release Date +**Release Date** 2023-11-30 -### Fixes and Improvements +**Fixes and Improvements** - `.pykx.print` was using `repr` representation for some objects. Now consistently calls `print`. - `.pykx.safeReimport` now resets environment variables correctly before throwing any error raised by the function supplied to it. @@ -519,15 +529,15 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.2.0 -#### Release Date +**Release Date** 2023-11-09 -### Additions +**Additions** - Addition of `PYKX_EXECUTABLE` environment/configuration variable to allow control of which Python executable is used under q. -### Fixes and Improvements +**Fixes and Improvements** - Failure to access and load PyKX resulting in an `os` error now returns Python backtrace outlining the underlying Python error allowing for easier debugging @@ -578,11 +588,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.1.0 -#### Release Date +**Release Date** 2023-10-09 -### Fixes and Improvements +**Fixes and Improvements** - Update to default conversion logic for q objects passed to PyKX functions to more closely match embedPy based conversion expectations.For version <=2.0 conversions of KX lists would produce N Dimensional Numpy arrays of singular type. This results in issues when applying to many analytic libraries which rely on lists of lists rather than singular N Dimensional arrays. Additionally q tables and keyed tables would be converted to Numpy recarrays, these are now converted to Pandas DataFrames. To maintain previous behavior please set the following environment variable `PYKX_DEFAULT_CONVERSION="np"`. @@ -606,11 +616,11 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat ## PyKX 2.0.0 -#### Release Date +**Release Date** 2023-09-18 -### Additions +**Additions** - Addition of `.pykx.qcallable` and `.pykx.pycallable` functions which allow wrapping of a foreign Python callable function returning the result as q or Python foreign respectively. - Addition of `.pykx.version` allowing users to programmatically access their version from a q process. @@ -628,7 +638,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat 4 0.691953 0.375638 ``` -### Fixes and Improvements +**Fixes and Improvements** - Application of object setting on a Python list returns generic null rather than wrapped foreign object. - Use of environment variables relating to `PyKX under q` must use `"true"` as accepted value, previously any value set for such environment variables would be supported. diff --git a/docs/scripts/collapse-code.js b/docs/scripts/collapse-code.js new file mode 100644 index 0000000..92db1e7 --- /dev/null +++ b/docs/scripts/collapse-code.js @@ -0,0 +1,65 @@ +document$.subscribe(function () { + const THRESHOLD = 25; + const COLLAPSE_LINES = 10; + + document.querySelectorAll("pre code").forEach(function (code) { + const pre = code.parentElement; + + if (pre.closest(".code-compare, .code-compare-three")) return; + + const lines = code.textContent.split("\n").filter(function (l, i, arr) { + return !(i === arr.length - 1 && l === ""); + }); + if (lines.length < THRESHOLD) return; + + const wrapper = document.createElement("div"); + wrapper.className = "code-collapse"; + pre.parentNode.insertBefore(wrapper, pre); + wrapper.appendChild(pre); + + const lineHeight = parseFloat(getComputedStyle(code).lineHeight); + const collapsedHeight = lineHeight * COLLAPSE_LINES; + pre.style.maxHeight = collapsedHeight + "px"; + pre.style.overflow = "hidden"; + + const toggle = document.createElement("button"); + toggle.className = "code-collapse-toggle code-collapse-toggle--bottom"; + toggle.setAttribute("aria-expanded", "false"); + toggle.textContent = "▼ Show all " + lines.length + " lines"; + wrapper.appendChild(toggle); + + function expand() { + pre.style.maxHeight = "none"; + pre.style.overflow = ""; + toggle.setAttribute("aria-expanded", "true"); + toggle.textContent = "▲ Show fewer lines"; + wrapper.classList.add("code-collapse--expanded"); + } + + function collapse() { + pre.style.maxHeight = collapsedHeight + "px"; + pre.style.overflow = "hidden"; + toggle.setAttribute("aria-expanded", "false"); + toggle.textContent = "▼ Show all " + lines.length + " lines"; + wrapper.classList.remove("code-collapse--expanded"); + } + + toggle.addEventListener("click", function () { + pre.style.overflow === "hidden" ? expand() : collapse(); + }); + + // Auto-expand if search.highlight injects a inside a collapsed block. + // MutationObserver catches marks added after collapse runs (search nav is async). + if (pre.querySelector("mark")) { + expand(); + } else { + var observer = new MutationObserver(function () { + if (pre.querySelector("mark")) { + expand(); + observer.disconnect(); + } + }); + observer.observe(pre, { childList: true, subtree: true }); + } + }); +}); diff --git a/docs/scripts/init_kapa_widget.js b/docs/scripts/init_kapa_widget.js new file mode 100644 index 0000000..6b62d52 --- /dev/null +++ b/docs/scripts/init_kapa_widget.js @@ -0,0 +1,52 @@ +document.addEventListener("DOMContentLoaded", function () { + var script = document.createElement("script"); + script.src = "https://widget.kapa.ai/kapa-widget.bundle.js"; + script.setAttribute("data-website-id", "90e95ec3-2524-43ab-93da-3e94fdd01d59"); + script.setAttribute("data-project-name", "KX"); + script.setAttribute("data-modal-title", "Ask AI"); + script.setAttribute("data-project-color", "#FFC300"); + //script.setAttribute("data-project-color-dark", "#00B0F0"); + //script.setAttribute("data-color-scheme-selector", "[data-md-color-scheme='slate']"); + script.setAttribute("data-color-scheme", "light"); + script.setAttribute("data-surface-color", "#ffffff"); + script.setAttribute("data-surface-elevated-color", "#f8f9fa"); + script.setAttribute("data-surface-hover-color", "#f1f3f5"); + script.setAttribute("data-text-color", "#111111"); + script.setAttribute("data-text-muted-color", "#666666"); + script.setAttribute("data-border-color", "#dee2e6"); + script.setAttribute("data-anchor-color", "#2563eb"); + //script.setAttribute("data-surface-color-dark", "#17171a"); + //script.setAttribute("data-surface-elevated-color-dark", "#25252a"); + //script.setAttribute("data-surface-hover-color-dark", "#32323a"); + //script.setAttribute("data-text-color-dark", "#e4e4e7"); + //script.setAttribute("data-text-muted-color-dark", "#a1a1aa"); + //script.setAttribute("data-border-color-dark", "#3f3f45"); + //script.setAttribute("data-anchor-color-dark", "#60a5fa"); + script.setAttribute("data-modal-header-background-color", "#ffffff"); + script.setAttribute("data-modal-header-color", "#1a1a1a"); + //script.setAttribute("data-modal-header-background-color-dark", "#1a1a1a"); + //script.setAttribute("data-modal-header-color-dark", "#ffffff"); + script.setAttribute("data-project-logo", "https://images.seeklogo.com/logo-png/45/2/kx-logo-png_seeklogo-454185.png"); + script.setAttribute('data-modal-override-open-id', 'ask-ai-input'); + script.setAttribute('data-modal-override-open-class', 'search-input'); + script.setAttribute('data-modal-open-by-default', 'false'); + //script.setAttribute('data-user-analytics-fingerprint-enabled', 'true'); + script.setAttribute("data-modal-z-index", "10001"); + script.setAttribute("data-modal-size", "800px"); + script.setAttribute("data-scale-factor", "0.8"); + script.setAttribute("data-modal-title-font-size", "18px"); + script.setAttribute("data-button-text-color", "#060000"); + script.setAttribute("data-modal-disclaimer", "This is an experimental custom LLM for answering technical questions about KX products. Answers are based **only** on KX documentation and support sources, but may not be fully accurate so please use your best judgement. Don't forget to rate the answers and give feedback!"); + script.setAttribute("data-modal-example-questions", "How do I get started with PyKX?,What's new in PyKX?,How do I get started with KDB-X?,How do I learn q?"); + script.setAttribute("data-mcp-enabled", "true"); + script.setAttribute("data-mcp-server-url", "https://kx.mcp.kapa.ai"); + script.setAttribute("data-search-mode-enabled", "true"); + //script.setAttribute("data-consent-required", "true"); + //script.setAttribute("data-user-analytics-cookie-enabled", "true"); + //script.setAttribute("data-consent-screen-title", "Help us improve our AI assistant"); + //script.setAttribute("data-consent-screen-disclaimer", "By clicking 'Allow tracking', you consent to anonymous user tracking which helps us improve our service. We don't collect any personally identifiable information."); + //script.setAttribute("data-consent-screen-accept-button-text", "Allow tracking"); + //script.setAttribute("data-consent-screen-reject-button-text", "No, thanks"); + script.async = true; + document.head.appendChild(script); +}); \ No newline at end of file diff --git a/docs/scripts/search-highlight-clear.js b/docs/scripts/search-highlight-clear.js new file mode 100644 index 0000000..90a29f4 --- /dev/null +++ b/docs/scripts/search-highlight-clear.js @@ -0,0 +1,19 @@ +// Clear search.highlight marks on first click after they appear. +// Marks are injected asynchronously after navigation, so a MutationObserver +// watches for them rather than polling with a fixed timeout. +document$.subscribe(function () { + var content = document.querySelector(".md-content") || document.body; + + var observer = new MutationObserver(function () { + if (!content.querySelector("mark")) return; + observer.disconnect(); + document.addEventListener("click", function clearMarks() { + content.querySelectorAll("mark").forEach(function (mark) { + mark.replaceWith(document.createTextNode(mark.textContent)); + }); + document.removeEventListener("click", clearMarks); + }); + }); + + observer.observe(content, { childList: true, subtree: true }); +}); diff --git a/docs/stylesheets/2026-edited.css b/docs/stylesheets/2026-edited.css index cf82b53..83386ba 100644 --- a/docs/stylesheets/2026-edited.css +++ b/docs/stylesheets/2026-edited.css @@ -361,4 +361,74 @@ .md-tabs__link { user-select: none; +} + +/* Collapsible code blocks */ +.code-collapse { + position: relative; +} + +.code-collapse::after { + content: ""; + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 6em; + background: linear-gradient(to bottom, transparent 0%, var(--md-code-bg-color) 80%); + pointer-events: none; +} + +[data-md-color-scheme="slate"] .code-collapse::after { + background: linear-gradient(to bottom, transparent, var(--kx-blue-grey)); +} + +.code-collapse--expanded::after { + display: none; +} + +.code-collapse-toggle--bottom { + position: absolute; + bottom: 2em; + left: 50%; + transform: translateX(-50%); + margin-bottom: 0; + z-index: 1; + background: var(--md-code-bg-color); +} + +[data-md-color-scheme="slate"] .code-collapse-toggle--bottom { + background: var(--kx-blue-grey); +} + +[data-md-color-scheme="kx-light"] .code-collapse-toggle--bottom { + background: var(--md-code-bg-color); +} + +.code-collapse-toggle { + display: inline-block; + margin-bottom: 0.4em; + cursor: pointer; + color: var(--kx-blue); + background: none; + border: 1px solid var(--kx-blue); + border-radius: 0.2em; + font-weight: 500; + font-size: 0.8em; + padding: 0.2em 0.6em; +} + +.code-collapse-toggle:hover { + background: var(--kx-blue); + color: white; +} + +[data-md-color-scheme="slate"] .code-collapse-toggle { + color: var(--kx-yellow); + border-color: var(--kx-yellow); +} + +[data-md-color-scheme="slate"] .code-collapse-toggle:hover { + background: var(--kx-yellow); + color: var(--kx-black); } \ No newline at end of file diff --git a/docs/stylesheets/prism-edited.css b/docs/stylesheets/prism-edited.css index 2eb16d2..66a0be5 100644 --- a/docs/stylesheets/prism-edited.css +++ b/docs/stylesheets/prism-edited.css @@ -55,12 +55,12 @@ pre[class*="language-"] { pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection, code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection { - background: #073642; /* base02 */ + background: var(--md-typeset-mark-color); } pre[class*="language-"]::selection, pre[class*="language-"] ::selection, code[class*="language-"]::selection, code[class*="language-"] ::selection { - background: #073642; /* base02 */ + background: var(--md-typeset-mark-color); } /* Code blocks */ @@ -120,7 +120,6 @@ pre[class*="language-"] { .token.entity { color: #657b83; /* base00 */ - background: #eee8d5; /* base2 */ } .token.atrule, diff --git a/docs/user-guide/advanced/Pandas_API.ipynb b/docs/user-guide/advanced/Pandas_API.ipynb index 7cb4833..9e54ad2 100644 --- a/docs/user-guide/advanced/Pandas_API.ipynb +++ b/docs/user-guide/advanced/Pandas_API.ipynb @@ -2171,7 +2171,7 @@ "| right_on | str | The column name in the right table to join on. | None |\n", "| left_index | bool | Use the index of the left Table. | False |\n", "| right_index | bool | Use the index of the right Table. | False |\n", - "| by | str | Not yet implemented. | None |\n", + "| by | Union[str/list] | Match on these columns before performing merge operation. | None |\n", "| left_by | str | Field names to match on in the left table. | None |\n", "| right_by | str | Field names to match on in the right table. | None |\n", "| suffixes | Tuple(str, str) | The number of rows to return. | ('\\_x', '\\_y') |\n", @@ -2303,6 +2303,16 @@ "trades.merge_asof(quotes, on=\"time\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc094351", + "metadata": {}, + "outputs": [], + "source": [ + "trades.merge_asof(quotes, on=\"time\", by=\"ticker\")" + ] + }, { "cell_type": "markdown", "id": "e6280a9a", diff --git a/docs/user-guide/advanced/license.md b/docs/user-guide/advanced/license.md index 4c5d77c..9c6e503 100644 --- a/docs/user-guide/advanced/license.md +++ b/docs/user-guide/advanced/license.md @@ -17,7 +17,7 @@ In the [install page](../../getting-started/installing.md) you can follow along ## When will your license expire? -For personal edition licenses your license will expire one year after initial download, for enterprise/commercial licenses the expiry date will vary. To provide some forewarning of when your license will expire PyKX provides the following: +To provide some forewarning of when your license will expire PyKX provides the following: - For the 10 days before expiry PyKX will print a message at start up indicating the license will expire in N days - The utility function `#!python kx.license.expires` provides a programmatic method of finding days to expiry. @@ -46,7 +46,7 @@ When your license is expired/expiring you will need to update it to ensure you c There are three methods by which updating your license is possible with PyKX. - You have allowed your license to expire and on restart of PyKX you will be presented with an new license install walkthrough similar to [installing](../../getting-started/installing.md). -- You pre-emptively install a newly downloaded license using `#!python kx.license.install`. +- You preemptively install a newly downloaded license using `#!python kx.license.install`. === "After Expiry" @@ -90,9 +90,9 @@ There are three methods by which updating your license is possible with PyKX. ... force=True) ``` -## Upgrade to a commercial license +## Upgrade to a `k4.lic` from a `kc.lic` license -If you are currently using a `kc.lic` personal license and need to upgrade to a `k4.lic` license the following steps allow you to ensure this can be done effectively. +If you are currently using a `kc.lic` license and need to change to a `k4.lic` license the following steps allow you to ensure this can be done effectively. 1. Delete your existing `kc.lic` license. 2. Install your new license following the [license installation](../../getting-started/installing.md) diff --git a/docs/user-guide/advanced/modes.md b/docs/user-guide/advanced/modes.md index fafa356..b1a5b14 100644 --- a/docs/user-guide/advanced/modes.md +++ b/docs/user-guide/advanced/modes.md @@ -8,21 +8,21 @@ tags: PyKX, q language, Python, licensed, unlincensed, # Modes of operation -_This page explains how to operate PyKX in Python and q, with or without a KDB Insights license._ +_This page explains how to operate PyKX in Python and q, with or without a license._ PyKX can operate in different modes. Each mode has its limitations and requirements, so we're breaking them down into the following: 1. PyKX within Python - 1.a Unlicensed - 1.b Licensed -2. PyKX within q with a kdb Insights license +2. PyKX within q - licensed ## 1. PyKX within Python The purpose of operating PyKX within a Python session is to replace [qPython](https://github.com/exxeleron/qPython) and [PyQ](https://github.com/kxsystems/pyq). Within Python, PyKX has two modes of operation: -- `#!python Licensed` (this means you have a kdb Insights license with PyKX enabled) -- `#!python Unlicensed` (this means you don't have a kdb Insights license or a license in which PyKX is not enabled) +- `#!python Licensed` (this means you have a license with PyKX enabled) +- `#!python Unlicensed` (this means you don't have a license or a license in which PyKX is not enabled) The main difference between the two is that the `#!python Unlicensed` mode is for IPC-based communication. This mean that it allows to create IPC connections and convert data from Pythonic representations to PyKX objects. @@ -55,9 +55,9 @@ This mode cannot run q embedded within it. Also, it lacks the ability to run q c ### 1.b Running in Licensed mode -Licensed mode is the standard way to operate PyKX, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#2-install-a-kdb-insights-license). This modality aims to replace PyQ as the Python-first library for KX. All PyKX features are available in this mode. +Licensed mode is the standard way to operate PyKX, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#2-install-a-license). This modality aims to replace PyQ as the Python-first library for KX. All PyKX features are available in this mode. -The differences provided through operating with a valid kdb Insights license are: +The differences provided through operating with a valid license are: 1. You can execute PyKX/q functionalities directly within a Python session. 2. PyKX objects can be represented in a human readable format rather than as a memory address, namely: diff --git a/docs/user-guide/advanced/remote-functions.md b/docs/user-guide/advanced/remote-functions.md index 4fe83dc..e2c300e 100644 --- a/docs/user-guide/advanced/remote-functions.md +++ b/docs/user-guide/advanced/remote-functions.md @@ -35,11 +35,11 @@ This walkthrough demonstrates the following steps: 1. Import PyKX and generate a remote session object which denotes the process against which the Python functions will be executed. 1. Define a number of Python functions which will be executed on the remote q/kdb+ server. -### Initializea q/kdb+ server with PyKX under q +### Initialize a q/kdb+ server with PyKX under q This step ensures you have a q process running with PyKX under q, as well as having a kdb+ table available to query. If you have this already, proceed to the next step. -Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/), retrieving your license following the instructions provided [here](https://kx.com/kdb-insights-sdk-personal-edition-download). +Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/). Install PyKX under q using the following command. diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 567d6cd..dfcd9e8 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -65,7 +65,7 @@ To add values to your configuration file you can modify the file directly or alt >>> import pykx as kx >>> kx.util.add_to_config({'PYKX_GC': 'True', 'PYKX_BETA_FEATURES': 'True'}) -Configuration updated at: /Users/conormccarthy/.pykx-config. +Configuration updated at: /Users/user/.pykx-config. Profile updated: default. Successfully added: - PYKX_GC = True diff --git a/docs_archive/kx_for_databricks/clusterSetups.md b/docs_archive/kx_for_databricks/clusterSetups.md deleted file mode 100644 index 6ffdd07..0000000 --- a/docs_archive/kx_for_databricks/clusterSetups.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: KX for Databricks cluster setups -description: How to use KX for Databricks -date: September 2024 -author: KX Systems, Inc., -tags: PyKX, Databricks, single node, multi node, workflows ---- - -# Cluster setups - -_The purpose of this page is to help you set up your single node or multi node clusters._ - -You can run KX for Databricks on both single node and multi-node computes. Choose the compute and RAM requirements depending on your data/pipeline requirements. - -For the best experience, read the [KX for Databricks Overview](introduction.md) and the [Get started](getStarted.md) guide first. - -Now let's explore examples for two types of cluster setups: - -1. [Single node workflow](#1-single-node-workflow) -2. [Multi node workflow](#2-multi-node-workflow) - -!!! info "Pre-requisites" - - The following pre-requisites must be satisfied in order to run the below notebooks: - - - Configuration of single node or multi node compute - - Latest version of PyKX must be installed - - QLIC env variable must point to a valid kc.lic file - - Load Trade, Quote & Execution data from a Databricks Unity Catalogue. - - For more information refer to the [Get Started documentation](getStarted.md). - -## 1. Single node workflow - -A single node compute runs spark locally and the driver works as both master and worker, with no worker nodes. -It is intended for jobs that use small amounts of data or non-distributed workloads such as single-node machine learning libraries. - -Import the required Python libraries: - -```python -import pykx as kx -import numpy as np -import pandas as pd -from datetime import datetime, timedelta -``` - -### Hourly OHLC - Pythonic vs. q magic - -!!! note "OHLC is the open, high, low, and close price for a given period." - -The example below calculates the hourly OHLC in two ways: using regular Python syntax and then the q magic. - -!!! example "" - - === "Pythonic" - - Calculate the OHLC using regular Python syntax: - - ```python - kx.q("{select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from x}", trade).head() - ``` - - This should produce the following output: - - ``` - bucket sym Open High Low Close - 2024.07.04D04:00:00.000000000 AEF 617f 617f 617f 617f - BKR 93.21 93.21 93.21 93.21 - CLMT 2327.99 2327.99 2327.99 2327.99 - CMG 780f 780f 777.6 780f - F.PRA 333.99 336.79 332.6 336.69 - ``` - - === "q magic" - - Calculate the OHLC using q magic: - - ```python - %%q - select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from trade - ``` - - This should produce the following output: - - ```python - bucket sym | Open High Low Close - ------------------------------------| ------------------------------- - 2024.07.04D04:00:00.000000000 AEF | 617 617 617 617 - 2024.07.04D04:00:00.000000000 BKR | 93.21 93.21 93.21 93.21 - 2024.07.04D04:00:00.000000000 CLMT | 2327.99 2327.99 2327.99 2327.99 - 2024.07.04D04:00:00.000000000 CMG | 780 780 777.6 780 - 2024.07.04D04:00:00.000000000 F.PRA | 333.99 336.79 332.6 336.69 - ``` - -Notice how both examples produce the same result. - -### VWAP - -!!! note "VWAP - The average price a security has traded at throughout the day, based on both volume and price." - -The example below calculates the volume-weighted average price (VWAP) using q magic: - -```python -%%q -select vwap:size wavg price by sym from trade -``` - -??? example "Output" - - ``` - sym | vwap - ------| -------- - AEF | 625.727 - AGND | 228.8684 - AUY | 251.6101 - AXJL | 494.2899 - AXX | 326.6949 - ``` - -### Volatility - -!!! note "Volatility - A statistical measure of the dispersion of returns for a given security or market index." - -The example below calculates volatility for the stock `AEF`. - -Define the functions used to calculate the volatility: - -```python -get_volatility_pykx = kx.q("{[bid;ask] {r:0^log[x]-log next x;sqrt ema[0.001] r*r } 0.5*ask+bid}") - -def get_volatility(quote_tbl, sym) -> dict: - subset_quotes = quote_tbl.loc[ - (quote_tbl["sym"] == sym) - & (quote_tbl["time"] >= timedelta(hours=9, minutes=25)) - & (quote_tbl["time"] < timedelta(hours=16, minutes=0)) - ] - return { - "datetime": subset_quotes["time"], - "volatility": get_volatility_pykx(subset_quotes["bid"], subset_quotes["ask"]) - } -``` - -Pass the `quote` table and the `AEF` symbol to the `get_volatility` function: - -```python -get_volatility(quote, 'AEF') -``` - -??? example "Output" - - ```python - {'datetime': pykx.TimestampVector(pykx.q('2024.07.04D09:25:03.491000000 2024.07.04D09:25:46.166000000 2024.07.04D09:27:..')), - 'volatility': pykx.FloatVector(pykx.q('0.003473635 0.003496871 0.003534634 0.003532866 0.003612577 0.003686721 0.003..'))} - ``` - -### Spread - -!!! note "Spread - The difference or gap that exists between two prices, rates, or yields." - -The example below calculates the moving average over the spread for the stock `AEF`. - -Define the functions used to calculate the moving average over the spread: - -```python -def get_mavg_of_spread(bid, ask, mavg_size: int): - spreads = ask - bid - return np.convolve(spreads, np.ones(mavg_size), mode="same") / mavg_size - -kx.q["mavg_of_spread"] = get_mavg_of_spread - -def get_spread(quotes: kx.Table, sym: str) -> kx.Table: - return kx.q.qsql.select( - quotes, - columns={"datetime": "time", "spread": "mavg_of_spread[bid; ask; 1000]", "bid":"bid", "ask":"ask"}, - where=[f'sym=`$"{sym}"', "bid>0", "ask>0", "time>=09:25", "time <16:00"] - ) -``` - -Pass the `quote` table and the `AEF` symbol to the `get_spread` function: - -```python -get_spread(quote,'AEF').head() -``` - -??? example "Output" - - ```python - datetime spread bid ask - 0 2024.07.04D09:25:03.491000000 15.07707 580.67 653.61 - 1 2024.07.04D09:25:46.166000000 15.07735 600f 630f - 2 2024.07.04D09:27:01.015000000 15.07766 588.88 625f - 3 2024.07.04D09:27:10.556000000 15.07844 580.67 653.61 - 4 2024.07.04D09:27:10.557000000 15.07873 580.67 653.61 - ``` - -### Slippage - -!!! note "Slippage - The difference between the expected price of a trade and the price at which the trade is executed." - -The example below calculates the Slippage based on the midpoint between the prevailing quoted bid and ask prices. - -Create the `merged` table by performing an as-of-join on the `execs` and `quote` tables: - -```python -merged = kx.q.aj(kx.SymbolVector(["sym", "time"]), execs, quote) -``` - -Calculate the Mid,Delta and Slippage columns: - -```python -merged["mid"] = 0.5 * (merged["bid"] + merged["ask"]) -merged["diff"] = np.where( - merged["side"] == "BUY", merged["mid"] - merged["price"], merged["price"] - merged["mid"] -) -merged["slippage"] = (merged["diff"] / merged["mid"]) * 10000 - -merged.head() -``` - -??? example "Output" - - ```python - date sym time price size side venue bid ask bsize asize mode ex mid diff slippage - 0 2022.03.31 GE.PRA 2024.07.04D09:30:02.676000000 397.6131 12 BUY venue1 396.51 397.96 1 1 R N 397.235 -0.3781 -9.518295 - 1 2022.03.31 MQC 2024.07.04D09:30:02.851000000 254.8359 258 SELL venue3 254.32 254.99 2 1 R V 254.655 0.1809 7.103729 - 2 2022.03.31 BKR 2024.07.04D09:30:03.426000000 93.64192 154 BUY venue2 93.64 93.72 1 1 R N 93.68 0.03808 4.064902 - 3 2022.03.31 GE.PRA 2024.07.04D09:30:04.489000000 397.5469 12 SELL venue2 397.21 398.8 1 2 R T 398.005 -0.4581 -11.50991 - 4 2022.03.31 DXGE 2024.07.04D09:30:06.677000000 570.1041 60 SELL venue3 569.08 571.34 2 3 R P 570.21 -0.1059 -1.857211 - ``` - -## 2. Multi node workflow - -A multi node (distributed) compute consists of one driver node and one or more worker nodes. - -The driver node maintains state information of all notebooks attached to the compute, maintains the SparkContext, interprets all the commands you run from a notebook or a library on the compute, and runs the Apache Spark master that coordinates with the Spark executors. -Worker nodes run the Spark executors and other services required for proper functioning compute. - -Use multi-node compute for larger jobs with distributed workloads. - -Before you start, make sure you meet the pre-requisites at the top of this page. - -### Import required libraries - -Import a PySpark and Python libraries by running: - -```python -import pykx as kx -from pyspark.sql import functions as F -from pyspark.sql import SparkSession -from pyspark.sql.functions import pandas_udf, PandasUDFType, lit -from pyspark.sql.types import StructType, StructField, DoubleType, StringType, DateType, IntegerType -spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", True) -import copy as cp -``` - -### Prepare the data - -!!! note "The below examples will make use of the full dataset while using PySpark and a subset of the data while using the Python Driver." - -Create `trade_sdf_all_syms` and `quote_sdf_all_syms` Spark Dataframes containing trade and quote data respectively using Spark SQL: - -```python -trade_sdf_all_syms = spark.sql('SELECT TTime as time, Symbol as sym, cast(Trade_Volume AS INT) as size, Trade_Price as price FROM default.nyse_trades') -quote_sdf_all_syms = spark.sql('SELECT TTime as time, Symbol as sym, Bid_Size as bsize, Bid_Price as bid, Offer_Size as asize, Offer_Price as ask FROM default.nyse_quotes') -``` - -Filter the data to the a subset of the tickers: - -```python -SYMBOLS = ['GOOG', 'AAPL', 'FB', 'AMZN', 'MSFT'] -trade_sdf_few_syms = trade_sdf_all_syms.where(F.col("Symbol").isin(SYMBOLS)).orderBy(["Symbol","TTime"]) -quote_sdf_few_syms = quote_sdf_all_syms.where(F.col("Symbol").isin(SYMBOLS)).orderBy(["Symbol","TTime"]) -``` - -Convert the Spark Dataframes to Pandas Dataframes: - -```python -trade_pdf = trade_sdf_few_syms.toPandas() -quote_pdf = quote_sdf_few_syms.toPandas() -``` - -### Slippage markouts (Python Driver/Single Node) - -!!! note "Slippage - The difference between the expected price of a trade and the price at which the trade is executed." - -In the below example, we will use PyKX to calculate the execution slippage markouts for a subset of the data (5 tickers) using just the Python Driver (Single node). - -??? example "Define the required functions used to calculate the slippage markouts" - - ```python - # adjust the time column in the past/future by offset e.g 10 20 30 seconds, 1 5 10 minutes - def time_offset(trade, offset): - offset_trade = cp.copy(trade) - offset_trade['time'] = (offset_trade['time'] + offset).sorted() - return offset_trade[['sym', 'time']] - - # compute N markouts and calculate slippage using mid price - def trade_to_mid_markouts(trade, quote): - import os - os.environ['QLIC'] = '/dbfs/tmp/' - import pykx as kx - - trade = kx.toq(trade) - quote = kx.toq(quote) - - trade = trade.sort_values(['time']).grouped('sym') - quote = quote.sort_values(['time']).grouped('sym') - quote['mid'] = (quote['ask'] + quote['bid']) / 2 - - # markouts we are interested in - seconds = [1, 10, 30] - minutes = [1, 5, 10, 30] - - qseconds = kx.toq(seconds).cast(kx.SecondVector) - qminutes = kx.toq(minutes).cast(kx.MinuteVector) - - for s, q in zip(seconds, qseconds): - trade['tp'+str(s)+'s'] = trade['price'] - time_offset(trade, q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - trade['tm'+str(s)+'s'] = trade['price'] - time_offset(trade, -q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - - for s, q in zip(minutes, qminutes): - trade['tp'+str(s)+'m'] = trade['price'] - time_offset(trade, q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - trade['tm'+str(s)+'m'] = trade['price'] - time_offset(trade, q).merge_asof(quote[['sym', 'time', 'mid']], on=['sym', 'time'])['mid'] - - return trade.pd() - ``` - -Pass the `trade_pdf` and `quote_pdf` Pandas Dataframes to the `trade_to_mid_markouts` function in order to calculate the slippage: - -```python -result = trade_to_mid_markouts(trade_pdf, quote_pdf) - -print('Executions input row count - ', f'{trade_pdf.shape[0]:,}') -print('Market quotes input row count - ', f'{quote_pdf.shape[0]:,}') -print('TCA report result row count - ', f'{kx.q.count(result).np():,}') - -result.tail() -``` - -!!! note "Since we have supplied a Pandas dataframe, the workload will be performed by only the Python driver." - -??? example "Output" - - ```python - Executions input row count - 1,851,406 - Market quotes input row count - 27,424,147 - TCA report result row count - 1,851,406 - - time sym size price tp1s tm1s tp10s tm10s tp30s tm30s tp1m tm1m tp5m tm5m tp10m tm10m tp30m tm30m - 1851401 2022-10-03 19:59:55.559408 AAPL 1 142.935 0.000 0.000 -0.025 0.010 -0.025 0.045 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 - 1851402 2022-10-03 19:59:58.444119 MSFT 25 241.700 0.185 0.185 0.185 0.185 0.185 3.200 0.185 0.185 0.185 0.185 0.185 0.185 0.185 0.185 - 1851403 2022-10-03 19:59:59.146947 AAPL 1 142.935 -0.025 0.000 -0.025 0.010 -0.025 0.055 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 -0.025 - 1851404 2022-10-03 19:59:59.551589 MSFT 1 241.700 0.185 0.185 0.185 0.185 0.185 3.200 0.185 0.185 0.185 0.185 0.185 0.185 0.185 0.185 - 1851405 2022-10-03 19:59:59.668143 MSFT 150 241.700 0.185 0.185 0.185 0.185 0.185 3.200 0.185 0.185 0.185 0.185 0.185 0.185 0.185 0.1815 - ``` - -### Slippage markouts (PySpark/Multi node) - -In the below example, we will use PyKX to calculate the execution slippage markouts for all of the data (11686 tickers) using PySpark (Multi node). - -Prepare the schema for the results dataframe: - -```python -full_schema = "time timestamp, sym string, size int, price double" - -seconds = [1, 10, 30] -minutes = [1, 5, 10, 30] - -for s in seconds: - full_schema += ', tp'+str(s)+'s float' - full_schema += ', tm'+str(s)+'s float' -for m in minutes: - full_schema += ', tp'+str(m)+'m float' - full_schema += ', tm'+str(m)+'m float' -``` - -Pass the `trade_sdf_all_syms` and `quote_sdf_all_syms` Spark Dataframes to the `trade_to_mid_markouts` function in order to calculate the slippage: - -```python -result_pyspark = trade_sdf_all_syms.groupby('sym').cogroup(quote_sdf_all_syms.groupby('sym')).applyInPandas(trade_to_mid_markouts, schema=full_schema) - -print('Executions input row count - ', f'{trade_sdf_all_syms.count():,}') -print('Market quotes input row count - ', f'{quote_sdf_all_syms.count():,}') -print('TCA report result row count - ', f'{result_pyspark.count():,}') - -result_pyspark.show(5) -``` - -!!! note "Since we have supplied a Spark dataframe, the workload will be farmed out to the configured worker nodes for processing before returning the result to the Python driver." - -??? example "Output" - - ```python - Executions input row count - 77,026,358 - Market quotes input row count - 1,541,781,424 - TCA report result row count - 77,026,358 - - +--------------------+------+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+-----+-----+ - | time| sym|size|price| tp1s| tm1s|tp10s|tm10s|tp30s|tm30s| tp1m| tm1m| tp5m| tm5m| tp10m| tm10m|tp30m|tm30m| - +--------------------+------+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+-----+-----+ - |2022-10-03 09:16:...|AKIC U| 85| 9.95| 0.21| 0.21| 0.21| 0.21| 0.21| 0.21| 0.21| 0.21| 0.17| 0.17|-0.265|-0.265|-0.03|-0.03| - |2022-10-03 11:20:...|AKIC U| 9| 9.96| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| - |2022-10-03 11:56:...|AKIC U| 21| 9.96| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -1.0| -0.02| -0.02|-0.02|-0.02| - |2022-10-03 13:36:...|AKIC U| 21| 9.95|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01| -0.01| -0.01|-0.01|-0.01| - |2022-10-03 13:36:...|AKIC U| 21| 9.95|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01|-0.01| -0.01| -0.01|-0.01|-0.01| - +--------------------+------+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------+------+-----+-----+ - ``` \ No newline at end of file diff --git a/docs_archive/kx_for_databricks/getStarted.md b/docs_archive/kx_for_databricks/getStarted.md deleted file mode 100644 index 4d0654c..0000000 --- a/docs_archive/kx_for_databricks/getStarted.md +++ /dev/null @@ -1,228 +0,0 @@ ---- -title: Get started with KX for Databricks -description: How to install and use KX for Databricks -date: September 2024 -author: KX Systems, Inc., -tags: PyKX, setup, install, ---- - -# Get started with KX for Databricks - -_This page explains how to deploy and get started with KX for Databricks._ - -1. [Install PyKX](#1-install-pykx) -2. [Licensing](#2-licensing) -3. [Load data using Spark Dataframes](#3-load-data-using-spark-dataframes) -4. [PyKX pythonic vs. q magic (%%)](#4-pykx-pythonic-vs-q-magic-) - - -## 1. Install PyKX - -Install PyKX on the cluster using one of the following methods: - -!!! Note "" - - === "Install PyKX from PyPI" - - Create a new notebook and install using `pip`: - - ```python - pip install pykx - ``` - - === "Install PyKX from Anaconda" - - Create a new notebook and install using `conda`: - - ```python - conda install -c kx pykx - ``` - - === "Install PyKX from the Python Wheel" - - Download the Python Wheel file from the [Download Portal](https://portal.dl.kx.com/assets/pypi/pykx) and store it on the cluster: - - ```shell - > ls /path/to/folder - pykx-2.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - ``` - - Create a new notebook and install from the wheel file: - - ```python - pip install /path/to/folder/pykx-2.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - ``` -!!! tip "Tip: After installing the packages, you may need to restart the kernel by running the following command:" - - - ```python - dbutils.library.restartPython() - ``` - -## 2. Licensing - -To use all PyKX functionalities, you need to download and install a KDB Insights license. - -1. Choose a license type: - - !!! Note "" - - === "Personal License" - - For personal usage, navigate to [Personal License Download](https://kx.com/kdb-insights-personal-edition-license-download/) and complete the form. - - === "Commercial License" - - For commercial usage, contact your KX sales representative, email [sales@kx.com](mailto:sales@kx.com), or apply through [Book demo](https://kx.com/book-demo). - - - !!! info "More on the [license installation process](https://code.kx.com/pykx/2.5/getting-started/installing.html#2-install-a-kdb-insights-license)." - -2. On receipt of an email from KX, download and save the `kc.lic` license file to a secure location on your cluster: - - ```shell - > ls /path/to/folder - kc.lic - ``` - -3. Set an environment variable `QLIC` pointing to the folder containing the license file: - - ```python - import os - os.environ['QLIC'] = '/path/to/folder' - ``` - That's it! To verify if you successfully installed PyKX, run the following: - - ```python - import pykx as kx - print('PyKX Version: ', kx.__version__) - ``` - You should have the following output: - - ```python - PyKX Version: 2.5.1 - ``` - !!! note "Note: The license requires internet connectivity." - - Update the workspace security policies to whitelist the following: - - - The remote servers should treat kdb+ on-demand as a CDN-fronted service and employ DNS-based filtering. - - The current list of hostnames required for kdb+ on-demand operation is {h,g}.kdbod.{com,net,org}. - -## 3. Load data using Spark Dataframes - -**a. From the Databricks Unity Catalogue:** - -```python -dfTrades = spark.table('..') -dfQuotes = spark.table('..') -``` - -**b. Directly from a CSV file:** - -```python -dfTrades = spark.read.format("csv").option("header", "true").option("delimiter", ",").load("/path/to/file") -dfQuotes = spark.read.format("csv").option("header", "true").option("delimiter", ",").load("/path/to/file") -``` - -To convert the Spark Dataframe to a Pandas Dataframe, use the `toPandas()` function: - -```python -pandas_trade = dfTrades.toPandas() -pandas_quote = dfQuotes.toPandas() -``` - -Convert the Pandas Dataframe to a q table with the `toq()` function: - -```python -pykx_trades = kx.toq(pandas_trade) -pykx_quotes = kx.toq(pandas_quote) -``` - -## 4. PyKX pythonic vs. q magic (%%) - -Reference q functions in PyKX with the standard python syntax: - -```python -kx.q('til 10') -``` - -This produces a sequence of numbers from 0 to 9: - -```python -pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) -``` - -Another way to achieve this is by using native q by leveraging q magic (`%%q`) : - -```python -%%q -til 10 -``` - -You will get the same output: - -```python -0 1 2 3 4 5 6 7 8 9 -``` - -??? example "Simple load and query example using PyKX" - - Consider a `samples` catalogue with a `finance` schema that has a `trades` table. Load data from the table into a Spark Dataframe `dfTrade`: - - ```python - dfTrade = spark.table('samples.finance.trades') - ``` - - Next, convert the Spark Dataframe to a Pandas Dataframe `panda_trade` using the `toPandas()` function: - - ```python - panda_trade = dfTrade.toPandas() - ``` - - Using the `toq()` function to convert the Pandas Dataframe to a q table `pykx_trade`: - - ```python - pykx_trade = kx.toq(panda_trade) - ``` - - To perform a simple OHLC query on the data, use regular python syntax: - - ```python - kx.q("{select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from x}", pykx_trade).head() - ``` - - This should produce the following output: - - ```python - bucket sym Open High Low Close - 2017.08.31D00:00:00.000000000 A 347.8271 349.256 346.0935 348.2853 - AAN 347.4742 349.1463 346.2195 348.3558 - ACA 347.4005 348.4313 346.5526 347.2568 - ACP 346.8897 350.9313 345.8342 347.8243 - ``` - - You can achieve the same result with q magic by storing the table: - - ```python - kx.q['pykx_trade'] = pykx_trade - ``` - - and performing the same query: - - ```python - %%q - select Open:first price,High:max price,Low:min price,Close:last price by bucket:0D01 xbar time,sym from pykx_trade - ``` - - In the end, you get the same output: - - ```python - bucket sym | Open High Low Close - ----------------------------------| ----------------------------------- - 2017.08.31D00:00:00.000000000 A | 347.8271 349.256 346.0935 348.2853 - 2017.08.31D00:00:00.000000000 AAN | 347.4742 349.1463 346.2195 348.3558 - 2017.08.31D00:00:00.000000000 ACA | 347.4005 348.4313 346.5526 347.2568 - 2017.08.31D00:00:00.000000000 ACP | 346.8897 350.9313 345.8342 347.8243 - 2017.08.31D00:00:00.000000000 ACRE| 349.8609 351.621 345.0562 345.9471 - ``` \ No newline at end of file diff --git a/docs_archive/kx_for_databricks/introduction.md b/docs_archive/kx_for_databricks/introduction.md deleted file mode 100644 index f9f1149..0000000 --- a/docs_archive/kx_for_databricks/introduction.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: KX for Databricks -description: Kx, PyKX, and Databricks -date: September 2024 -author: KX Systems, Inc., -tags: kx, PyKX, Databricks ---- - -# KX for Databricks -_This page briefly describes PyKX on Databricks use cases and capabilities._ - -KX for Databricks brings the full capability of KX's time series engine to Databricks. KX's PyKX, the Python-first interface already familiar to data scientists and quants, turbocharges workloads and research on use cases such as: - -- Algorithmic trading strategy development and backtesting​ -- Large-scale pre- and post-trade analytics -- Macro-economic research and analysis using large datasets​ -- Cross-asset analytics with deep historical data​ - -KX for Databricks leverages PyKX directly inside Databricks, for Python and distributed Spark workloads. Use it on existing Delta Lake datasets without external KX dependencies, or choose to flexibly interop with other KX products to leverage the power of your broader KX systems estate. - -![KX for Databricks reference architecture](kx-databricks-refarch.png) - -## Get started and workflows - -Refer to the [Get started](getStarted.md) page if you wish to complete the following actions: - -- [Install](getStarted.md#1-install-pykx) PyKX on Databricks. -- Check what type of [license](getStarted.md#2-licensing) you need. -- [Load data](getStarted.md#3-load-data-using-spark-dataframes) using Spark Dataframes. -- [Use PyKX pythonic vs. q magic (%%)](getStarted.md#4-pykx-pythonic-vs-q-magic-). - -Next, learn how to [set up a cluster](clusterSetups.md), depending on the type of workflow you need: - -- Work with a [single node/driver](clusterSetups.md#1-single-node-workflow) on analytic calculation of OHLC, VWAP pricing, volatility, spread, and trade execution analysis (slippage). -- Work with a [multi/distributed nodes](clusterSetups.md#2-multi-node-workflow) cluster on slippage calculation with PySpark. diff --git a/docs_archive/kx_for_databricks/kx-databricks-refarch.png b/docs_archive/kx_for_databricks/kx-databricks-refarch.png deleted file mode 100644 index f24db3b..0000000 Binary files a/docs_archive/kx_for_databricks/kx-databricks-refarch.png and /dev/null differ diff --git a/licenses/QSO.txt b/licenses/QSO.txt index ee31923..9e57305 100644 --- a/licenses/QSO.txt +++ b/licenses/QSO.txt @@ -1,7 +1,7 @@ Software License for q.so -Version Number: 1.0 -Date Last Revised: April 2023 +Version Number: 1.1 +Date Last Revised: June 2026 Software License Agreement for use of q.so (“Agreement”) @@ -13,7 +13,7 @@ This Agreement is made between KX Systems, Inc. (“KX” or “we”) and the E  1. LICENSE GRANTS -1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. +1.1 Grant of License. KX hereby grants End User a non-transferable, non-exclusive license, without right of sublicense, to install and use the Software solely for the purpose of compiling and running the PyKX or KDB-X software made available by KX under separate licensing terms. End User will not attempt to circumvent any restrictions imposed on the Software or use the Software for any purpose other than stated above. If End User is using the PyKX or KDB-X software under the terms of a commercial licence from KX, the End User must obtain a separate licence from KX for the use of the Software. 1.2 Software Use Restrictions. End User may not: (a) modify any part of the Software or create derivative works thereof, (b) sell, lease, license or distribute the Software to any third party, (c) attempt to decompile, disassemble or reverse engineer the Software, (d) copy the Software, except for purposes of installing and executing it within the limitations set out at clause 1.1, (e) use or attempt to use the Software in any way that is unlawful or fraudulent or has any unlawful or fraudulent purpose or effect, (f) use or attempt to use the Software in any way that would breach the license granted herein. diff --git a/mkdocs.yml b/mkdocs.yml index 6e2f759..f6f1e74 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -41,9 +41,14 @@ extra_javascript: - 'https://code.jquery.com/jquery-3.6.0.min.js' - 'https://use.fontawesome.com/3dbf29ed12.js' - scripts/strip-q-prompt-copy.js - # this relative path is needed to source from code.kx.com!!! - - "../../pykx/scripts/versions.js" - # Uncomment only for local test + - scripts/init_kapa_widget.js + - scripts/collapse-code.js + - scripts/search-highlight-clear.js + # absolute URL so the shared, symlinked versions.js loads on every page. + # The old "." relative hack is double-escaped to "&period;" by + # newer mkdocs-material, which 404s and breaks the version dropdown. + - https://code.kx.com/pykx/scripts/versions.js + # Uncomment only for local test # - 'scripts/versions.js' @@ -82,7 +87,8 @@ markdown_extensions: format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tilde - toc: - permalink: false + permalink: true + permalink_title: "" title: On this page plugins: @@ -127,11 +133,12 @@ plugins: - search - exclude-search: exclude: - - user-guide/advanced/Pandas_API.ipynb - - examples/interface-overview.ipynb - - examples/jupyter-integration.ipynb - - examples/db-management.ipynb - - examples/streaming/Evolving System.ipynb + - user-guide/advanced/Pandas_API* + - examples/interface-overview* + - examples/jupyter-integration* + - examples/db-management* + - "examples/streaming/Evolving System*" + - examples/charting* - spellcheck: known_words: spelling.txt ignore_code: true # Ignore words in tags @@ -147,7 +154,56 @@ plugins: - examples/db-management.ipynb - examples/streaming/Evolving System.ipynb - examples/charting.ipynb - + - redirects: + redirect_maps: + 'api/console.md': 'api/pykx-execution/console.md' + 'api/ctx.md': 'api/pykx-execution/ctx.md' + 'api/embedded_q.md': 'api/pykx-execution/embedded_q.md' + 'api/pykx-save-load/read.md': 'api/pykx-save-load/fileio.md' + 'api/pykx-save-load/write.md': 'api/pykx-save-load/fileio.md' + 'api/pykx_under_q.md': 'api/pykx-execution/ctx.md' + 'api/q/q.md': 'api/pykx-execution/q.md' + 'api/read.md': 'api/pykx-save-load/fileio.md' + 'api/toq.md': 'api/pykx-q-data/toq.md' + 'api/type_conversions.md': 'api/pykx-q-data/type_conversions.md' + 'api/wrappers.md': 'api/pykx-q-data/wrappers.md' + 'api/write.md': 'api/pykx-save-load/fileio.md' + 'beta-features/compress-encypt.md': 'user-guide/advanced/compress-encrypt.md' + 'beta-features/db-management.md': 'user-guide/advanced/database/index.md' + 'beta-features/remote-functions.md': 'user-guide/advanced/remote-functions.md' + 'beta-features/streamlit.md': 'user-guide/advanced/streamlit.md' + 'beta-features/threading.md': 'user-guide/advanced/threading.md' + 'changelog.md': 'release-notes/changelog.md' + 'comparisons.md': 'getting-started/what_is_pykx.md' + 'evaluating.md': 'user-guide/fundamentals/evaluating.md' + 'examples/threaded_execution/index.md': 'examples/threaded_execution/threading.md' + 'examples/threaded_execution/threaded_execution.md': 'examples/threaded_execution/threading.md' + 'extras/comparisons.md': 'getting-started/what_is_pykx.md' + 'extras/faq.md': 'help/faq.md' + 'extras/known_issues.md': 'help/issues.md' + 'faq.md': 'help/faq.md' + 'getting-started/interface_overview.md': 'getting-started/what_is_pykx.md' + 'getting-started/q_magic_command.md': 'examples/jupyter-integration.ipynb' + 'indexing.md': 'user-guide/fundamentals/indexing.md' + 'installing.md': 'getting-started/installing.md' + 'interface_overview.md': 'getting-started/what_is_pykx.md' + 'known_issues.md': 'help/issues.md' + 'licenses.md': 'license.md' + 'limitations.md': 'help/issues.md' + 'modes.md': 'user-guide/advanced/modes.md' + 'nulls_and_infinities.md': 'user-guide/fundamentals/nulls_and_infinities.md' + 'parallelization.md': 'user-guide/advanced/performance.md' + 'performance.md': 'user-guide/advanced/performance.md' + 'pykx-under-q/known_issues.md': 'help/issues.md' + 'troubleshooting.md': 'help/troubleshooting.md' + 'user-guide/advanced/environment_variables.md': 'user-guide/configuration.md' + 'user-guide/advanced/limitations.md': 'help/issues.md' + 'user-guide/advanced/running_under_q.md': 'pykx-under-q/intro.md' + 'user-guide/fundamentals/querying.md': 'user-guide/fundamentals/query/index.md' + 'user-guide/index.md': 'index.md' + '1.0/licenses.md': 'license.md' + '1.5/api/type_conversion.md': 'api/pykx-q-data/type_conversions.md' + '3.0/pykx-under-q/type_conversions.md': 'api/pykx-q-data/type_conversions.md' theme: name: material @@ -166,6 +222,10 @@ theme: - content.code.annotate - content.action.edit - content.code.copy + - search.highlight + - navigation.tracking + - navigation.top + - toc.follow palette: - media: "(prefers-color-scheme: light)" scheme: kx-light diff --git a/pyproject.toml b/pyproject.toml index 567110c..4e9c097 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "pykx" description = "An interface between Python and q" readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.9" license = { file = "LICENSE.txt" } authors = [ { name = "KX", email = "pykx@kx.com" }, @@ -31,11 +31,12 @@ classifiers = [ "Programming Language :: Cython", "Programming Language :: Other", "Programming Language :: Other Scripting Engines", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Implementation :: CPython", # "Programming Language :: Python :: Implementation :: PyPy", @@ -48,15 +49,12 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "numpy~=1.20; python_version=='3.7'", - "numpy~=1.22; python_version=='3.8'", "numpy~=1.22; python_version=='3.9'", "numpy>=1.22; python_version=='3.10'", "numpy>=1.23; python_version=='3.11'", "numpy>=1.26; python_version=='3.12'", "numpy>=1.26; python_version=='3.13'", "numpy>=2.3.0; python_version=='3.14'", - "pandas>=1.2, <2.0.0; python_version=='3.8'", "pandas>=1.2, <3.0; python_version>'3.8' and python_version<'3.14'", "pandas>= 2.3.3, <3.0; python_version=='3.14'", "pytz>=2022.1", @@ -80,6 +78,7 @@ doc = [ "mkdocs-exclude-search==0.6.6", "mkdocs-jupyter~=0.24", "mkdocs-material~=9.4.5", + "mkdocs-redirects>=1.2.0", "mkdocs-render-swagger-plugin==0.0.3", "mkdocs-spellcheck==0.2.0", "mkdocstrings[python]==0.18.0", @@ -112,7 +111,7 @@ dashboards = [ "ast2json~=0.3", ] streamlit = [ - "streamlit~=1.28; python_version>'3.7'" + "streamlit~=1.28" ] torch = [ "torch>2.1; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version <= '3.12'" @@ -145,8 +144,6 @@ changelog = "https://code.kx.com/pykx/release-notes/changelog.html" [build-system] requires = [ "Cython~=3.0.0", - "numpy~=1.20.0; python_version=='3.7'", # Use numpy version 1.20.x for building the python 3.7 wheel - "numpy~=1.22.0; python_version=='3.8'", # Use the highest patch version of numpy 1.22.x, this will still support a user using numpy version 1.22.0 "numpy~=2.0; python_version=='3.9'", "numpy~=2.0; python_version=='3.10'", "numpy~=2.0; python_version=='3.11'", @@ -156,10 +153,8 @@ requires = [ "numpy~=2.0; python_version=='3.11'", "numpy~=2.0; python_version=='3.12'", "numpy~=2.0; python_version=='3.13'", - "setuptools~=68.0.0; python_version=='3.7'", - "setuptools~=69.0.2; python_version!='3.7'", - "setuptools-scm[toml]~=7.1.0; python_version=='3.7'", - "setuptools-scm[toml]~=8.0; python_version!='3.7'", + "setuptools~=69.0.2", + "setuptools-scm[toml]~=8.0", "tomli>=2.0.1", "wheel>=0.42.0", ] diff --git a/setup.py b/setup.py index ffb1742..db30d9a 100755 --- a/setup.py +++ b/setup.py @@ -256,8 +256,7 @@ def ext(name: str, ]), ] - if py_minor_version >= 8: # python 3.8 or higher is required for NEP-49 - exts.append(ext('_numpy', numpy=True, cython=False, libraries=['dl', *windows_libraries])) + exts.append(ext('_numpy', numpy=True, cython=False, libraries=['dl', *windows_libraries])) exts.append(ext('numpy_conversions', numpy=True, cython=False, diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index 1d8c1bd..79839d1 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -362,11 +362,9 @@ def merge_asof(left, *args, **kwargs): return left.merge_asof(*args, **kwargs) -if sys.version_info[1] < 8: +if sys.version_info[1] < 9: warn( - 'Python 3.7 has reach its end of life period and is no longer supported.' - 'Please consider upgrading to Python 3.8, as PyKX will no longer support issues for this ' - 'Python version.', + 'Python 3.9 is the minimum support version. Please upgrade.', exceptions.PyKXWarning ) diff --git a/src/pykx/config.py b/src/pykx/config.py index 9bd625b..d5d6ba9 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -12,7 +12,7 @@ import toml import pandas as pd -from .exceptions import PyKXWarning, QError +from .exceptions import QError system = platform.system() @@ -219,8 +219,8 @@ def _license_check(lic_type, lic_encoding, lic_variable): # pragma: no cover def _unlicensed_config(unlicensed_message): - choice = input('\nWould you like us to remember this choice? [Y/n]: ') - if choice in ('y', 'Y', ''): + choice = input('\nWould you like us to remember this choice? [N/y]: ') + if choice in ('y', 'Y'): fpath = Path(os.path.expanduser('~')) / '.pykx-config' try: os.access(fpath, os.W_OK) @@ -271,8 +271,8 @@ def _license_install(intro=None, return_value=False, license_check=False, licens print(install_message) return True - personal_url = "https://kx.com/kdb-insights-sdk-personal-edition-download" - commercial_url = "https://kx.com/book-demo" + lic_url = 'https://developer.kx.com/products/kdb-x/install' + lic_type = 'kc.lic' unlicensed_message = '\nPyKX unlicensed mode enabled. To set this as your default behavior '\ "set the following environment variable PYKX_UNLICENSED='true'" first_user = '\nThank you for installing PyKX!\n\n'\ @@ -296,69 +296,24 @@ def _license_install(intro=None, return_value=False, license_check=False, licens if existing_license not in ('Y', 'y', 'N', 'n', ''): raise Exception('Invalid input provided please try again') if existing_license in ('N', 'n', ''): - commercial = input('\nIs the intended use of this software for:' - '\n [1] Personal use (Default)' - '\n [2] Commercial use' - '\nEnter your choice here [1/2]: ').strip().lower() - if commercial not in ('1', '2', ''): - raise Exception('User provided option was not one of [1/2]') - - personal = commercial in ('1', '') - - lic_url = personal_url if personal else commercial_url - lic_type = _kc_lic if personal else _k4_lic - - if personal: - redirect = input(f'\nTo apply for your PyKX license, navigate to {lic_url}.\n' - 'Shortly after you submit your application, you will receive a ' - 'welcome email containing your license information.\n' - 'Would you like to open this page? [Y/n]: ') - else: - redirect = input('\nTo apply for your PyKX license, contact your ' - 'KX sales representative or sales@kx.com.\n' - f'Alternately apply through {lic_url}.\n' - 'Would you like to open this page? [Y/n]: ') - + redirect = input('\n** It is recommended to upgrade to KDB-X Python pykx>=4.0 **\n\n' + f'For instructions and to obtain a license visit: {lic_url}\n' + 'Would you like to open this page? [Y/n]: ') if redirect.lower() in ('y', ''): try: webbrowser.open(lic_url) time.sleep(2) except BaseException: raise Exception('Unable to open web browser') - - install_type = input('\nPlease select the method you wish to use to activate your ' - 'license:\n [1] Download the license file provided in your ' - 'welcome email and input the file path (Default)' - '\n [2] Input the activation key (base64 encoded string) ' - 'provided in your welcome email' - '\n [3] Proceed with unlicensed mode' - '\nEnter your choice here [1/2/3]: ').strip().lower() - - if install_type not in ('1', '2', '3', ''): - raise Exception('User provided option was not one of [1/2/3]') - - if install_type in ('1', ''): - license = input('\nProvide the download location of your license ' - f'(for example, {root}{lic_type}) : ').strip() - download_location = os.path.expanduser(Path(license)) - _license_install_path(download_location, qlic) - - elif install_type == '2': - license = input('\nProvide your activation key (base64 encoded string) ' - 'provided with your welcome email : ').strip() - - _license_install_B64(license, lic_type) - - print('\nPyKX license successfully installed to: {qlic / lic_type}\n') # noqa: E501 - elif install_type == '3': - _unlicensed_config(unlicensed_message) - if return_value: - return False + print('\nPyKX will proceed in unlicensed mode') + _unlicensed_config(unlicensed_message) + if return_value: + return False else: install_type = input( '\nPlease select the method you wish to use to activate your license:\n' - ' [1] Provide the location of your license\n' - ' [2] Input the activation key\n' + ' [1] Provide the location of your license file\n' + ' [2] Paste the license key\n' 'Enter your choice here [1/2]: ') if install_type not in ('1', '2', ''): @@ -370,16 +325,15 @@ def _license_install(intro=None, return_value=False, license_check=False, licens _license_install_path(download_location, qlic) else: - commercial = input('\nPlease confirm the license type:\n' - f' [1] Personal use ({_kc_lic})\n' - f' [2] Commercial use ({_k4_lic})\n' - 'Enter your choice here [1/2]: ') - if commercial not in ('1', '2', ''): + lic_type_choice = input('\nPlease confirm the license type:\n' + f' [1] {_kc_lic} - The default\n' + f' [2] {_k4_lic} - Used in some scenarios\n' + 'Enter your choice here [1/2]: ') + if lic_type_choice not in ('1', '2', ''): raise Exception('User provided option was not one of [1/2]') - personal = commercial in ('1', '') - lic_type = _kc_lic if personal else _k4_lic - license = input('\nProvide your activation key (base64 encoded string) : ').strip() + lic_type = _k4_lic if lic_type_choice == '2' else _kc_lic + license = input('f\nProvide your {lic_type} license key (base64 encoded string) : ').strip() # noqa: E501 _license_install_B64(license, lic_type) @@ -420,15 +374,7 @@ def _license_install(intro=None, return_value=False, license_check=False, licens keep_local_times = _is_enabled('PYKX_KEEP_LOCAL_TIMES') max_error_length = int(_get_config_value('PYKX_MAX_ERROR_LENGTH', 256)) -allocator = _is_enabled('PYKX_ALLOCATOR', '--pykxalloc') -if allocator: - if sys.version_info[1] <= 7: - raise PyKXWarning('A python version of at least 3.8 is required to use the PyKX allocators') # noqa nocov - k_allocator = False # nocov - else: - k_allocator = True -else: - k_allocator = False +k_allocator = _is_enabled('PYKX_ALLOCATOR', '--pykxalloc') k_gc = _is_enabled('PYKX_GC', '--pykxgc') release_gil = _is_enabled('PYKX_RELEASE_GIL', '--release-gil') diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx index cfede9e..f35bbe7 100644 --- a/src/pykx/core.pyx +++ b/src/pykx/core.pyx @@ -312,20 +312,19 @@ if not pykx_threading: 'Would you like to renew your license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' _license_message = _license_install(_exp_license, True, True, 'exp') elif re.compile('licen[cs]e error: embedq').search(_capout_msg): - _ce_license = 'You appear to be using a non kdb Insights license.\n\n'\ + _ce_license = 'You appear to be using a non PyKX enabled license.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ - 'Running PyKX in the absence of a kdb Insights license '\ + 'Running PyKX in the absence of a license '\ 'has reduced functionality.\nWould you like to install '\ - 'a kdb Insights personal license? [Y/n]: ' + 'a license? [Y/n]: ' _license_message = _license_install(_ce_license, True) elif re.compile('licen[cs]e error: upd').search(_capout_msg): _upd_license = 'Your installed license is out of date for this version'\ ' of PyKX and must be updated.\n\n'\ f'{_capout_msg}\n\n'\ f'{_lic_location}\n\n'\ - 'Would you like to install an updated kdb '\ - 'Insights personal license? [Y/n]: ' + 'Would you like to install an updated license? [Y/n]: ' _license_message = _license_install(_upd_license, True) elif re.compile('licen[cs]e error: k[xc4].lic').search(_capout_msg) or re.compile('licen[cs]e error: badmsg').search(_capout_msg): _k_license = '\nThe PyKX license found is corrupt or incompatible with'\ diff --git a/src/pykx/help.py b/src/pykx/help.py index 9163713..75ef532 100644 --- a/src/pykx/help.py +++ b/src/pykx/help.py @@ -1,14 +1,10 @@ import os import re -from urllib.request import Request, urlopen -from urllib.error import HTTPError, URLError __all__ = ["help"] _filepath = os.path.join( - os.path.dirname(__file__), '..', '..', 'docs', 'api', - 'pykx-execution', 'q.md' + os.path.dirname(__file__), 'docs', 'api', 'pykx-execution', 'q.md' ) -_base_url = "https://code.kx.com/pykx/4.0/api/pykx-execution/q.html" def __dir__(): @@ -20,73 +16,6 @@ def _init(_q): q = _q -def _fetch_web_documentation(keyword): - """Fetch documentation from the PyKX website.""" - url = _base_url - try: - req = Request(url, headers={'User-Agent': 'PyKX/help'}) - with urlopen(req, timeout=5) as response: - html = response.read().decode('utf-8') - - # Extract the article content - article_match = re.search( - r']*class="[^"]*md-content__inner[^"]*"[^>]*>' - r'(.*?)', - html, re.DOTALL - ) - if not article_match: - return None - - article_html = article_match.group(1) - - # Find the h3 section for this keyword - # Pattern:

...

... content ...

- pattern = rf']*id="{keyword}"[^>]*>.*?

(.*?)(?=]*id="|$)' - section_match = re.search(pattern, article_html, re.DOTALL) - - if not section_match: - return None - - section_html = section_match.group(1) - text = f" • {keyword}\n\n" - - # Extract paragraphs - for para_match in re.finditer(r']*)?>(.*?)

', section_html, re.DOTALL): - para_text = re.sub(r'<[^>]+>', '', para_match.group(1)) - para_text = _decode_html_entities(para_text).strip() - if para_text: - text += para_text + "\n\n" - - # Extract code blocks - for code_match in re.finditer(r']*>]*>(.*?)', - section_html, re.DOTALL): - code_text = _decode_html_entities(code_match.group(1)).strip() - if code_text: - text += "\n" - for line in code_text.split("\n"): - text += " " + line + "\n" - text += "\n" - - return text - - except (URLError, HTTPError, TimeoutError): - return None - except Exception: - return None - - -def _decode_html_entities(text): - """Decode common HTML entities.""" - return (text.replace('<', '<') - .replace('>', '>') - .replace('&', '&') - .replace('"', '"') - .replace(''', "'") - .replace(' ', ' ') - .replace('¶', '') # Remove paragraph symbols - .replace('·', '·')) - - def _load_markdown_section(keyword, file_path=_filepath): """Load and parse markdown section for a keyword.""" try: @@ -174,15 +103,9 @@ def qhelp(keyword): return text - # Fallback to web scraping if local file fails - web_result = _fetch_web_documentation(keyword) - if web_result: - return web_result - # If both local and web sources fail return (f" • {keyword}\n\n" - f"Documentation not available. Please check your internet connection\n" - f"or visit {_base_url}#{keyword} for documentation.") + f"Documentation not available locally. Visit https://code.kx.com for documentation.") def _parse_markdown_table(lines): diff --git a/src/pykx/lib/4-1-libs/q.k b/src/pykx/lib/4-1-libs/q.k index b48fde4..c37ff6f 100644 --- a/src/pykx/lib/4-1-libs/q.k +++ b/src/pykx/lib/4-1-libs/q.k @@ -68,7 +68,7 @@ show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! \d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") -k:4.1;K:0Nd;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc +k:4.1;K:0Nd;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;ipc:{-40!(x;y)};w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp`from`by addmonths:{("d"$m+y)+x-"d"$m:"m"$x} f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} @@ -111,7 +111,7 @@ hap:{@[(0;h;(h:1+i 1)^(1+*&"@"=x)&e;e:(#x)^e)_x,(^e:(i:&"/"=x:(":"=*x)_x)2)#"/"; hmb:{x:$[10=@x;x;1_$x];p:{$[#y;y;x]}/'getenv@+`$_:\("HTTP";"NO"),\:"_PROXY";u:hap@x;t:~(~#*p)||/(*":"\:u 2)like/:{(("."=*x)#"*"),x}'","\:p 1;a:$[t;p:hap@*p;u]1;r:(`$":",,/($[t;p;u]0 2))($y)," ",$[t;x;u 3]," HTTP/1.1",s,(s/:(~![-35;::])_("Accept-Encoding: gzip";"Connection: close";"Host: ",u 2),((0<#a)#,$[t;"Proxy-";""],"Authorization: Basic ",btoa a),$[#z;("Content-type: ",z 0;"Content-length: ",$#z 1);()]),(d:s,s:"\r\n"),$[#z;z 1;""];(0;n)_$[#ss[(n:4+*r ss d)#r;"content-encoding: gzip\r\n"];-35!r;r]} hg:{hmb[x;`GET;()]1};hp:{hmb[x;`POST;(y;z)]1} -a0:(#:;*:;last;sum;prd;min;max;.q.all;.q.any;?:),a1:(avg;wsum;wavg;var;dev;cov;cor),`.q `svar`sdev`scov`med +a0:(#:;*:;last;+/[0;];sum;prd;min;max;.q.all;.q.any;?:),a1:(avg;wsum;wavg;var;dev;cov;cor),`.q `svar`sdev`scov`med IN:{$[99h<@x;x in y;0]};qa:{$[qb x;0;IN[*x;a0];1;|/qa'1_x]};qb:{(2>#x)|(@x)&~11=@x} / CAN EXIT HERE FOR SMALL Q @@ -142,7 +142,7 @@ fp:{+((,*x)!,(#z)#$[-7h=@y;y;(*|x)$y]),+z} foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/r(::;i)1=#i:&0<#:'r:$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} / select{u's|a's[by[date,]u's]}from t where[date..],[sym{=|in}..],.. -a2:({(%;(sum;("f"$;x));(sum;(~^:;x)))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} +a2:({(%;(sum;("f"$;x));(-;(#:;x);(+/[0;];(^:;x))))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} {(-;(avg;(*;("f"$;x);y));(*;(avg;x);(avg;y)))};{(%;(cov;x;y);(*;(dev;x);(dev;y)))};{(.q.scov;x;x)};{(sqrt;(.q.svar;x))};{(*;(%;(#:;`i);(+;-1;(#:;`i)));(cov;x;y))};{'`part}) / x0 translate;x1 aggrs;x2 translate ?terminal (subselect/exec within partition) unknown assumed uniform? diff --git a/src/pykx/license.py b/src/pykx/license.py index 9a532d1..c639a21 100644 --- a/src/pykx/license.py +++ b/src/pykx/license.py @@ -39,8 +39,7 @@ def check(license: str, format: Is the license check being completed using a downloaded file or base64 encoded string. Accepted inputs are "FILE"(default) or "STRING". license_type: The license file type/name which is to be checked, by default this - is 'kc.lic' which is the version provided with personal and commercial - evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + is 'kc.lic' but can be changed to 'k4.lic' or 'kx.lic' if appropriate Returns: A boolean indicating if the license is correct or not and a printed message describing @@ -166,8 +165,7 @@ def install(license: str, format: Is the license check being completed using a downloaded file or base64 encoded string. Accepted inputs are "FILE"(default) or "STRING". license_type: The license file type/name which is to be checked, by default this - is 'kc.lic' which is the version provided with personal and commercial - evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + is 'kc.lic' but can be changed to 'k4.lic' or 'kx.lic' if appropriate force: Enforce overwrite without opt-in message for overwrite Returns: diff --git a/src/pykx/pandas_api/pandas_conversions.py b/src/pykx/pandas_api/pandas_conversions.py index e26b6eb..3dc5e3a 100644 --- a/src/pykx/pandas_api/pandas_conversions.py +++ b/src/pykx/pandas_api/pandas_conversions.py @@ -145,20 +145,20 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c1:()!(); if[any b1; dCols1:dictCols where b1; - f1:{(`$';x)}; c1:dCols1!(f1 each dCols1)]; + f1:{({(0b;`$'x)};x)}; c1:dCols1!(f1 each dCols1)]; // Check casting to symbol, run `$string col // (also covers any symbol -> symbol cases) b2:(dictColTypes=11h) & not (b1 or tabColTypes=0h); c2:()!(); if[any b2; dCols2:dictCols where b2; - f2:{(`$string; x)}; c2:dCols2!(f2 each dCols2)]; + f2:{({(0b;`$string x)}; x)}; c2:dCols2!(f2 each dCols2)]; // Casting to string covering all cases except mixed lists b3: (dictColTypes=10h) & not tabColTypes=0h; c3:()!(); if[any b3; dCols3:dictCols where b3; - f3:{(string; x)}; c3:dCols3!(f3 each dCols3)]; + f3:{({(0b;string x)}; x)}; c3:dCols3!(f3 each dCols3)]; // Check mixed lists // if string column then allow cast to symbol // Check at beginning of method @@ -168,31 +168,32 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c4:()!(); if[any b4; dCols4:dictCols where b4; - f4:{(`$; x)}; c4:dCols4!(f4 each dCols4)]; + f4:{({(0b;`$x)}; x)}; c4:dCols4!(f4 each dCols4)]; // Any matches that meet the vanilla case // and don't have additional needs --> not any (bools) b5:not any (b1;b2;b3;b4); - .pykx.i.errorList:(); if[any b5; dCols5:dictCols where b5; dictColTypes5:dictColTypes where b5; f5:{[c;t;tvd] ({[cl;t;tvd] - @[t$;cl; - {[cl;ty;tvd;err].pykx.i.errorList,:enlist + @[{(0b;x$y)}t;cl; + {[cl;ty;tvd;err] (1b; "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], - " with q error: ", err;}[cl;t;tvd;]]}[;t;tvd]; + " with q error: ", err)}[cl;t;tvd;]]}[;t;tvd]; c)}; f5:f5[;;typeValDict]; c5:dCols5!(f5\'[dCols5;dictColTypes5])]; // Grab all cols c:c1,c2,c3,c4,c5; - tableOutput:tabColsOrig xcols ![tab;();0b;c]; - $[count .pykx.i.errorList; - .pykx.i.errorList; - tableOutput] + res:?[tab;();();c]; + $[any value[res][;0]; + {x[;1] where x[;0]} value[res]; + tabColsOrig xcols flip res[;1], + $[count cls:cols[tab] except key c; + ?[tab;();();{x!x} cls];()]] }''', self, dict_grab, type_number_to_pykx_k_type) else: @@ -234,21 +235,21 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c1:()!(); if[any b1; tCols1:tabCols where b1; - f1:{(`$';x)}; c1:tCols1!(f1 each tCols1) + f1:{({(0b;`$'x)};x)}; c1:tCols1!(f1 each tCols1) ]; // Support casting to symbol b2:(dtype=11h) & not (b1 or tabColTypes=0h); c2:()!(); if[any b2; tCols2:tabCols where b2; - f2:{(`$string@; x)}; c2:tCols2!(f2 each tCols2) + f2:{({(0b;`$string x)}; x)}; c2:tCols2!(f2 each tCols2) ]; // Support casting to string except for mixed lists b3:(dtype=10h) & not tabColTypes=0h; c3:()!(); if[any b3; tCols3:tabCols where b3; - f3:{(string; x)}; c3:tCols3!(f3 each tCols3) + f3:{({(0b;string x)}; x)}; c3:tCols3!(f3 each tCols3) ]; // For mixed lists support casting strings to symbols b4:(dtype=11h) & (tabColTypes=0h) @@ -256,30 +257,31 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 c4:()!(); if[any b4; tCols4:tabCols where b4; - f4:{(`$; x)}; c4:tCols4!(f4 each tCols4) + f4:{({(0b;`$x)}; x)}; c4:tCols4!(f4 each tCols4) ]; // Any other combination not matching b1-4 b5:not any (b1;b2;b3;b4); - .pykx.i.errorList:(); c5:()!(); if[any b5; tCols5: tabCols where b5; f5:{[c;t;tvd] ({[cl;t;tvd] - @[t$;cl; - {[cl;ty;tvd;err].pykx.i.errorList,:enlist + @[{(0b;x$y)}t;cl; + {[cl;ty;tvd;err] (1b; "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], " with q error: ", - err;}[cl;t;tvd;]]}[;t;tvd]; + err)}[cl;t;tvd;]]}[;t;tvd]; c)}; c5:tCols5!(f5[;dtype;typeValDict] each tCols5) ]; c:c1,c2,c3,c4,c5; - tableOutput:tabCols xcols ![tab;();0b;c]; - $[count .pykx.i.errorList; - .pykx.i.errorList; - tableOutput] + res:?[tab;();();c]; + $[any value[res][;0]; + {x[;1] where x[;0]} value[res]; + tabCols xcols flip res[;1], + $[count cls:cols[tab] except key c; + ?[tab;();();{x!x} cls];()]] }''', self, dtype_val, type_number_to_pykx_k_type) diff --git a/src/pykx/pandas_api/pandas_merge.py b/src/pykx/pandas_api/pandas_merge.py index 56e5cbe..c893291 100644 --- a/src/pykx/pandas_api/pandas_merge.py +++ b/src/pykx/pandas_api/pandas_merge.py @@ -383,7 +383,6 @@ def merge_asof( direction != 'backward' or not allow_exact_matches or tolerance is not None - or by is not None or left_by is not None or right_by is not None ): @@ -395,7 +394,9 @@ def merge_asof( ) else: raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() only implemented for direction='backward', \ - allow_exact_matches=True, tolerance=None, by=None, left_by=None, right_by=None.") # noqa: E501 + allow_exact_matches=True, tolerance=None, left_by=None, right_by=None.") # noqa: E501 + if by is not None: + on = ([by] if isinstance(by, str) else by) + [on] (left, right, on, added_idx) = _parse_input( self, right, diff --git a/src/pykx/pandas_api/pandas_meta.py b/src/pykx/pandas_api/pandas_meta.py index 92f2427..e3bad0c 100644 --- a/src/pykx/pandas_api/pandas_meta.py +++ b/src/pykx/pandas_api/pandas_meta.py @@ -437,6 +437,14 @@ def count(self, axis=0, numeric_only=False): def isna(self): return q.null(self) + @api_return + def fills(self): + return q.fills(self) + + @api_return + def ungroup(self): + return q.ungroup(self) + @api_return def isnull(self): return self.isna() diff --git a/src/pykx/pykx.c b/src/pykx/pykx.c index 36ffdbd..9ae3b49 100644 --- a/src/pykx/pykx.c +++ b/src/pykx/pykx.c @@ -550,6 +550,8 @@ EXPORT K get_attr(K f, K attr) { } K res = create_foreign(pres); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(pres); PyGILState_Release(gstate); return res; } diff --git a/src/pykx/pykxq.c b/src/pykx/pykxq.c index 10fad12..50c8d0c 100644 --- a/src/pykx/pykxq.c +++ b/src/pykx/pykxq.c @@ -506,8 +506,11 @@ EXPORT K get_attr(K f, K attr) { int gstate = PyGILState_Ensure(); P p = get_py_ptr(f); P _attr = Py_BuildValue("s", attr->s); - K res = create_foreign(PyObject_GetAttr(p, _attr)); + P __attr = PyObject_GetAttr(p, _attr); + K res = create_foreign(__attr); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(__attr); if ((k = k_py_error())) { PyGILState_Release(gstate); return k; diff --git a/src/pykx/pykxq_m.c b/src/pykx/pykxq_m.c index 48de3bc..7f2d038 100644 --- a/src/pykx/pykxq_m.c +++ b/src/pykx/pykxq_m.c @@ -664,8 +664,11 @@ K _get_attr(K f, K attr) { } P p = get_py_ptr(f); P _attr = Py_BuildValue("s", attr->s); - K res = create_foreign(PyObject_GetAttr(p, _attr)); + P __attr = PyObject_GetAttr(p, _attr); + K res = create_foreign(__attr); Py_XDECREF(_attr); + // INCREF is called in create_foreign, decref here to avoid object not being freed + Py_XDECREF(__attr); if ((k = k_py_error())) { return k; } diff --git a/src/pykx/q.so/libs/4-1/l64/libq.so b/src/pykx/q.so/libs/4-1/l64/libq.so index 56684db..1680946 100755 Binary files a/src/pykx/q.so/libs/4-1/l64/libq.so and b/src/pykx/q.so/libs/4-1/l64/libq.so differ diff --git a/src/pykx/q.so/libs/4-1/l64arm/libq.so b/src/pykx/q.so/libs/4-1/l64arm/libq.so index 3e2e4f9..8963529 100755 Binary files a/src/pykx/q.so/libs/4-1/l64arm/libq.so and b/src/pykx/q.so/libs/4-1/l64arm/libq.so differ diff --git a/src/pykx/q.so/libs/4-1/m64/libq.dylib b/src/pykx/q.so/libs/4-1/m64/libq.dylib index 5358dba..50d360b 100755 Binary files a/src/pykx/q.so/libs/4-1/m64/libq.dylib and b/src/pykx/q.so/libs/4-1/m64/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib b/src/pykx/q.so/libs/4-1/m64arm/libq.dylib index 21660d4..5aaaf7d 100755 Binary files a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib and b/src/pykx/q.so/libs/4-1/m64arm/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-1/w64/q.dll b/src/pykx/q.so/libs/4-1/w64/q.dll index 35c9677..ba406f0 100644 Binary files a/src/pykx/q.so/libs/4-1/w64/q.dll and b/src/pykx/q.so/libs/4-1/w64/q.dll differ diff --git a/src/pykx/util.py b/src/pykx/util.py index 8d4560b..54451ec 100644 --- a/src/pykx/util.py +++ b/src/pykx/util.py @@ -21,7 +21,7 @@ from .config import ( _executable, _get_qexecutable, _get_qhome, _pykx_config_location, _pykx_profile_content, - allocator, beta_features, ignore_qhome, jupyterq, k_gc, keep_local_times, + beta_features, ignore_qhome, jupyterq, k_allocator, k_gc, keep_local_times, load_pyarrow_unsafe, max_error_length, no_pykx_signal, no_qce, pykx_4_1, pykx_config_location, pykx_config_profile, pykx_debug_insights, pykx_dir, pykx_lib_dir, pykx_qdebug, pykx_threading, q_executable, qargs, qhome, qlic, release_gil, @@ -467,7 +467,7 @@ def env_information(): env_info = '\n**** PyKX Configuration Variables ****\n' global_config = {'PYKX_IGNORE_QHOME': ignore_qhome, 'PYKX_KEEP_LOCAL_TIMES': keep_local_times, - 'PYKX_ALLOCATOR': allocator, 'PYKX_GC': k_gc, + 'PYKX_ALLOCATOR': k_allocator, 'PYKX_GC': k_gc, 'PYKX_LOAD_PYARROW_UNSAFE': load_pyarrow_unsafe, 'PYKX_MAX_ERROR_LENGTH': max_error_length, 'PYKX_NOQCE': no_qce, 'PYKX_RELEASE_GIL': release_gil, 'PYKX_Q_LIB_LOCATION': pykx_lib_dir, diff --git a/src/pykx/wrappers.py b/src/pykx/wrappers.py index bd1020d..65f4f0e 100644 --- a/src/pykx/wrappers.py +++ b/src/pykx/wrappers.py @@ -1989,6 +1989,9 @@ def mean(self): def median(self): return q.med(self) + def fills(self): + return q.fills(self) + def mode(self): return q('{where max[c]=c:count each d:group x}', self) @@ -4320,7 +4323,7 @@ def __getitem__(self, key): def __setitem__(self, key, val): if isinstance(key, tuple): raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() is not implemented, .pykx.Dictionary objects do not support tuple key assignment") # noqa: E501 - self.__dict__.update(q('{x,((),y)!((),z)}', self, key, val).__dict__) + self.__dict__.update(q('{ye:(),y;x,ye!$[y~ye;z;enlist z]}', self, key, val).__dict__) def _repr_html_(self): if not licensed: @@ -5197,9 +5200,6 @@ def each_left(self): vs = each_left -Function.scan.__doc__ = help.qhelp('scan') - - class Lambda(Function): """Wrapper for q lambda functions. @@ -5894,6 +5894,10 @@ def __init__(self, column=None, name=None, data=None, is_tree=False, value=None) warn("The kx.Column 'value' keyword is deprecated, please use 'data' instead", DeprecationWarning) elif data is not None: + if isinstance(data, (SymbolAtom, SymbolVector, str)): + data = [data] + elif isinstance(data, list) and all(isinstance(x, str) for x in data): + data = [data] self._data = data self._renamed = True else: @@ -5909,8 +5913,45 @@ def __repr__(self): preamble = f'pykx.{type(self).__name__}' return f"{preamble}(name='{self._name}', data={type(self._data)})" - """Function for building up a function call off a Column""" + def __getitem__(self, key): + res = copy.deepcopy(self) + if isinstance(key, (SymbolAtom, SymbolVector, str)): + key = [key] + elif isinstance(key, list) and all(isinstance(x, str) for x in key): + key = [key] + res._data = [res._data, key] + return res + def call(self, op, *other, iterator=None, col_arg_ind=0, project_args=None): + """ + Used to makes function calls off a Column object. + + Parameters: + op: The function to apply. + *other: The other column(s) or variable(s) (Python/q) to be used + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Examples: + + Find the maximum values in nested lists. + + ```python + >>> t = kx.q('([] n:(1 2;4 3;10 10))') + >>> t.select(kx.Column('n').call('max', iterator='each')) + pykx.Table(pykx.q(' + n + -- + 2 + 4 + 10 + ')) + ``` + """ params = [] for param in other: if isinstance(param, Column): @@ -6164,7 +6205,7 @@ def _and(self, other, iterator=None, col_arg_ind=0, project_args=None): Use `&` or `[]` in general to build queries. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -6488,7 +6529,7 @@ def cor(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -6632,7 +6673,7 @@ def cov(self, other, sample=False, iterator=None, col_arg_ind=0, project_args=No - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. sample: Should calculations of covariance return the sample covariance (set True) covariance (set False {default}) iterator: What iterator to use when operating on the column @@ -6694,7 +6735,7 @@ def cross(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7018,7 +7059,7 @@ def div(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7467,7 +7508,7 @@ def inter(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7516,7 +7557,7 @@ def isin(self, other, iterator=None, col_arg_ind=0, project_args=None): Most commonly this function is used in where clauses to filter data Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7619,7 +7660,7 @@ def like(self, other, iterator=None, col_arg_ind=0, project_args=None): clauses to filter data. Parameters: - other: A string/byte array defining a regex pattern to be used for query + other: A string/byte array defining a regex pattern to be used for query. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -7767,7 +7808,7 @@ def mavg(self, other, iterator=None, col_arg_ind=1, project_args=None): The results are returned as a floating point. Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving average iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -7905,7 +7946,7 @@ def mcount(self, other, iterator=None, col_arg_ind=1, project_args=None): so far, thereafter the result is the moving average Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving count iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -7978,7 +8019,7 @@ def mdev(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving standard deviation Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving standard deviation iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8152,7 +8193,7 @@ def mmax(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving maximum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving maximum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8185,7 +8226,7 @@ def mmin(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving minimum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving minimum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8216,11 +8257,11 @@ def mod(self, other, iterator=None, col_arg_ind=0, project_args=None): Calculate the modulus of items in a column for a given value. Parameters: - other: An integer denoting the divisor to be used when calculating the modulus + other: An integer denoting the divisor to be used when calculating the modulus. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 1. + where the column parameter will be used. Default 0. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -8248,7 +8289,7 @@ def msum(self, other, iterator=None, col_arg_ind=1, project_args=None): of items so far, thereafter the result is the moving sum Parameters: - other: An integer denoting the window to be used for calculation of + other: An integer denoting the window to be used for calculation of. the moving sum iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -8341,7 +8382,7 @@ def _or(self, other, iterator=None, col_arg_ind=0, project_args=None): Return the larger of the underlying boolean values between two columns: Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -8617,7 +8658,7 @@ def rotate(self, other, iterator=None, col_arg_ind=1, project_args=None): by the parameter other. Parameters: - other: An integer denoting the number of elements left(positive) or right(negative) + other: An integer denoting the number of elements left(positive) or right(negative). which the column list will be shifted iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -9189,7 +9230,7 @@ def union(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9307,7 +9348,7 @@ def var(self, iterator=None, sample=False): fn = 'svar' if sample else 'var' return self.call(fn, iterator=iterator) - def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): + def wavg(self, weight, iterator=None, col_arg_ind=0, project_args=None): """ Return the weighted average between a column and: @@ -9316,7 +9357,7 @@ def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + weight: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9351,7 +9392,7 @@ def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): pykx.FloatAtom(pykx.q('2.431111')) ``` """ - return self.call('wavg', other, iterator=iterator, col_arg_ind=col_arg_ind, + return self.call('wavg', weight, iterator=iterator, col_arg_ind=col_arg_ind, project_args=project_args) def within(self, lower, upper, iterator=None, col_arg_ind=0, project_args=None): @@ -9417,7 +9458,7 @@ def wsum(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9461,7 +9502,7 @@ def xbar(self, other, iterator=None, col_arg_ind=1, project_args=None): parameter other. Parameters: - other: An integer denoting the multiple to which all values will be rounded + other: An integer denoting the multiple to which all values will be rounded. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9519,7 +9560,7 @@ def xexp(self, other, iterator=None, col_arg_ind=1, project_args=None): the parameter other. Parameters: - other: An integer denoting the power to which all values will be raised + other: An integer denoting the power to which all values will be raised. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9559,7 +9600,7 @@ def xlog(self, other, iterator=None, col_arg_ind=1, project_args=None): by the parameter other. Parameters: - other: An integer denoting the logarithmic base to which all values will be set + other: An integer denoting the logarithmic base to which all values will be set. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9598,7 +9639,7 @@ def xprev(self, other, iterator=None, col_arg_ind=1, project_args=None): before it. Where N is specified by the parameter other. Parameters: - other: An integer denoting the number of indices before elements in the list + other: An integer denoting the number of indices before elements in the list. to retrieve the value of iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -9841,7 +9882,7 @@ def add(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -9974,6 +10015,10 @@ def data(self, data): ``` """ cpy = copy.deepcopy(self) + if isinstance(data, (SymbolAtom, SymbolVector, str)): + data = [data] + elif isinstance(data, list) and all(isinstance(x, str) for x in data): + data = [data] cpy._data = data cpy._renamed=True return cpy @@ -10029,13 +10074,13 @@ def cast(self, other, iterator=None, col_arg_ind=1, project_args=None): Convert the content of a column to another PyKX type Parameters: - other: The name of the type to which your column should be cast + other: The name of the type to which your column should be cast. or the lower case letter used to define it in q, for more information see [here](https://code.kx.com/q/ref/cast/). iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10071,6 +10116,8 @@ def cast(self, other, iterator=None, col_arg_ind=1, project_args=None): """ if not isinstance(other, str): raise QError('Supplied value other must be a str') + if other == 'symbol': + other = '' if 1 == len(other): other = other.encode('UTF-8') return self.call('$', other, iterator=iterator, col_arg_ind=col_arg_ind, @@ -10085,7 +10132,7 @@ def correlation(self, other, iterator=None, col_arg_ind=0, project_args=None): - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10145,7 +10192,7 @@ def covariance(self, other, sample=False, iterator=None, col_arg_ind=0, project_ - A PyKX variable in q memory Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. sample: Should calculations of covariance return the sample covariance (set True) covariance (set False {default}) iterator: What iterator to use when operating on the column @@ -10216,7 +10263,7 @@ def divide(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10296,11 +10343,11 @@ def drop(self, other, iterator=None, col_arg_ind=1, project_args=None): an iterator. Where N is specified by the other parameter. Parameters: - other: An integer defining the number of elements to drop + other: An integer defining the number of elements to drop. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10334,11 +10381,11 @@ def fill(self, other, iterator=None, col_arg_ind=1, project_args=None): Replace all null values in a column with a specified 'other' parameter Parameters: - other: The value which should replace nulls within a column + other: The value which should replace nulls within a column. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -10427,7 +10474,7 @@ def join(self, other, iterator=None, col_arg_ind=0, project_args=None): - A variable in q memory Parameters: - other: The Column, list, item or variable to be joined to the + other: The Column, list, item or variable to be joined to the. original column iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -10514,7 +10561,7 @@ def modulus(self, other, iterator=None, col_arg_ind=1, project_args=None): Calculate the modulus of items in a column for a given value. Parameters: - other: An integer denoting the divisor to be used when calculating the modulus + other: An integer denoting the divisor to be used when calculating the modulus. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10557,7 +10604,7 @@ def multiply(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10864,7 +10911,7 @@ def subtract(self, other, iterator=None, col_arg_ind=0, project_args=None): when adding elements. Parameters: - other: The second column or variable (Python/q) to be used + other: The second column or variable (Python/q) to be used. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10944,11 +10991,11 @@ def take(self, other, iterator=None, col_arg_ind=1, project_args=None): from a column using an iterator. Where N is specified by the other parameter. Parameters: - other: An integer defining the number of elements to retrieve + other: An integer defining the number of elements to retrieve. iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function - where the column parameter will be used. Default 0. + where the column parameter will be used. Default 1. project_args: The argument indices of a multivariate function which will be projected on the function before evocation with use of an iterator. @@ -11072,10 +11119,416 @@ def bin(self, other, iterator=None, col_arg_ind=1, project_args=None): 0 1 ')) + ``` """ return self.call('bin', other, iterator=iterator, col_arg_ind=col_arg_ind, project_args=project_args) + def enlist(self, iterator=None): + """ + Return column values as a list. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 2 4)') + >>> t.exec(kx.Column('n').enlist(iterator='each')) + pykx.List(pykx.q(' + 2 + 4 + ')) + ``` + """ + return self.call('enlist', iterator=iterator) + + def all(self, iterator=None): + """ + Returns a boolean atom 1b if all items are nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 5#1b)') + >>> t.exec(kx.Column('n').all()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + """ + return self.call('all', iterator=iterator) + + def any(self, iterator=None): + """ + Returns a boolean atom 1b if at least one item is nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 0010b)') + >>> t.exec(kx.Column('n').any()) + pykx.BooleanAtom(pykx.q('1b')) + ``` + """ + return self.call('any', iterator=iterator) + + def _in(self, other, iterator=None, col_arg_ind=0, project_args=None): + """ + Returns a boolean atom 1b if at least one item is nonzero, else retuns 0b. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:1 2 3 4 5)') + >>> t.select(kx.Column('n')._in([0,2,4], iterator='\\:')) + pykx.Table(pykx.q(' + n + - + 0 + 1 + 0 + 1 + 0 + ')) + ``` + """ + return self.call('in', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def _except(self, other, iterator=None, col_arg_ind=0, project_args=None): + """ + Returns a list of all items of column that are not items of `other`. + + Parameters: + other: A list or atom to be excepted from the column. + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: til 5)') + >>> t.exec(kx.Column('n')._except(3)) + pykx.LongVector(pykx.q('0 1 2 4')) + ``` + """ + return self.call('except', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def hsym(self, iterator=None): + """ + Returns a list of all items of column prefixed with a colon. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: `a`b`c)') + >>> t.exec(kx.Column('n').hsym()) + pykx.SymbolVector(pykx.q('`:a`:b`:c')) + ``` + """ + return self.call('hsym', iterator=iterator) + + def raze(self, iterator=None): + """ + Returns the collapsed items of the column. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').raze()) + pykx.List(pykx.q(' + 1 + 2 + 3 4 + 5 6 + 7 + 8 + ')) + ``` + + If you want to flatten all levels use the `over` or `\\` option for iterator. + ```python + >>> t.exec(kx.Column('n').raze(iterator='over')) + pykx.LongVector(pykx.q('1 2 3 4 5 6 7 8')) + ``` + """ + return self.call('raze', iterator=iterator) + + def type(self, iterator=None): + """ + Returns the type of the column. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.ShortVector(pykx.q('7 0 -7 -7h')) + ``` + """ + return self.call('type', iterator=iterator) + + def key(self, iterator=None): + """ + Where a column is made of dictionaries, return the keys. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> d1 = kx.q('`a`b`c ! 1 2 3') + >>> d2 = kx.q('`d`e`f ! 4 5 6') + >>> t = kx.Table(data={'n':[d1, d2]}) + >>> t.exec(kx.Column('n').type(iterator='each')) + pykx.Table(pykx.q(' + n + ----- + a b c + d e f + ')) + ``` + """ + return self.call('key', iterator=iterator) + + def get(self, iterator=None): + """ + Read value from a column of named global variables or file names. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:tb1 set ([] n: `a`b`c)') + pykx.SymbolAtom(pykx.q('`:tb1')) + >>> kx.q('`:tb2 set ([] n: 1 2 3)') + pykx.SymbolAtom(pykx.q('`:tb2')) + >>> t = kx.q('([] n: `:tb1`:tb2)') + >>> t.exec(kx.Column('n').get(iterator='each')) + pykx.List(pykx.q(' + +(,`n)!,`a`b`c + +(,`n)!,1 2 3 + ')) + ``` + """ + return self.call('get', iterator=iterator) + + def read0(self, iterator=None): + """ + Read text from a file or process handle. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read0(iterator='each')) + pykx.List(pykx.q(' + "hello" + "goodbye" + ')) + ``` + """ + return self.call('read0', iterator=iterator) + + def read1(self, iterator=None): + """ + Read bytes from a file or named pipe. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> kx.q('`:t1.txt 0: enlist "hello"') + pykx.SymbolAtom(pykx.q('`:t1.txt')) + >>> kx.q('`:t2.txt 0: enlist "goodbye"') + pykx.SymbolAtom(pykx.q('`:t2.txt')) + >>> t = kx.q('([] n: `:t1.txt`:t2.txt)') + >>> t.exec(kx.Column('n').read1(iterator='each')) + pykx.List(pykx.q(' + 0x68656c6c6f0a + 0x676f6f646279650a + ')) + ``` + """ + return self.call('read1', iterator=iterator) + + def next(self, iterator=None): + """ + For each item in the column, return the next item. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n:3 6 9 12)') + >>> t.exec(kx.Column('n').next())) + pykx.LongVector(pykx.q('6 9 12 0N')) + ``` + """ + return self.call('next', iterator=iterator) + + def prior(self, other, iterator=None, col_arg_ind=1, project_args=None): + """ + Applies passed function to value in column and the item preceding it. + + Parameters: + other: A binary function. + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 1. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + Example: + + ```python + >>> import pykx as kx + >>> t = kx.q('([] n: 1 2 3)') + >>> t.select(kx.Column('n').prior(kx.q('+'))) + pykx.Table(pykx.q(' + n + - + 1 + 3 + 5 + ')) + ``` + """ + return self.call('prior', other, iterator=iterator, col_arg_ind=col_arg_ind, + project_args=project_args) + + def where(self, iterator=None): + """ + Return indexes of True values. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each` + + Examples: + + Find indexes where values are True + + ```python + >>> import pykx as kx + >>> tab = kx.q('([] a:101010b)') + >>> tab.exec(kx.Column('a').where()) + pykx.LongVector(pykx.q('0 2 4')) + ``` + """ + return self.call('where', iterator=iterator) + class QueryPhrase: """Special wrapper for a list which will be treated as a QueryPhrase. diff --git a/tests/qcumber_tests/memory.quke b/tests/qcumber_tests/memory.quke index 0abf799..21b7274 100644 --- a/tests/qcumber_tests/memory.quke +++ b/tests/qcumber_tests/memory.quke @@ -23,3 +23,11 @@ feature Memory Allocation tests initmem:pymem[]; do[100000;.pykx.eval"bbb(10, 100)"]; 10000 > abs initmem - pymem[] + + expect repeated getting of an objects attribute to not increase memory + .pykx.pyexec"class Foo:\n def __init__(self):\n self.a = 1\n self.b = list(range(10000))"; + f: .pykx.eval"Foo()"; + initmem:pymem[]; + do[10000;f[`:a]]; + do[10000;f[`:b]]; + 10000 > abs initmem-pymem[] diff --git a/tests/test_pandas_api.py b/tests/test_pandas_api.py index 1d8c3a7..f8f43ea 100644 --- a/tests/test_pandas_api.py +++ b/tests/test_pandas_api.py @@ -935,25 +935,25 @@ def test_table_outer_merge(kx, q): assert df_res.equals(res) -def test_merge_sort(kx, q): - d = kx.q.z.D - a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) - b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) - res = a.merge(b, how="outer", on="date", sort=True) - assert all(res[0]['date'] == kx.q('enlist', d-3)) - - -def test_merge_sort_single_key(kx, q): - d = kx.q.z.D - a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) - b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) - a = kx.q.xkey('date', a) - res = a.merge(b, how="outer", on="date", sort=True) - assert all(res.keys().columns == ['date']) - res = kx.q('0!', res) - assert all(res[0]['date'] == kx.q('enlist', d-3)) - - +def test_merge_sort(kx, q): + d = kx.q.z.D + a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) + b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) + res = a.merge(b, how="outer", on="date", sort=True) + assert all(res[0]['date'] == kx.q('enlist', d-3)) + + +def test_merge_sort_single_key(kx, q): + d = kx.q.z.D + a = kx.toq(pd.DataFrame({'r': [2, 3, 4], 'date': [d, d-3, d-1]})) + b = kx.toq(pd.DataFrame({'r': [5, 6, 7], 'date': [d-1, d, d-3]})) + a = kx.q.xkey('date', a) + res = a.merge(b, how="outer", on="date", sort=True) + assert all(res.keys().columns == ['date']) + res = kx.q('0!', res) + assert all(res[0]['date'] == kx.q('enlist', d-3)) + + def test_cross_merge(kx, q): df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) @@ -1978,13 +1978,16 @@ def compare_q_to_pd(tab, df): def test_table_merge_asof(kx, q): - left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]}) - right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]}) + left = pd.DataFrame({"a": [1, 5, 10], "b": ["a", "a", "b"], "left_val": ["a", "b", "c"]}) + right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "b": ["a", "a", "b", "b", "a"], + "right_val": [1, 2, 3, 6, 7]}) qleft = kx.toq(left) qright = kx.toq(right) assert (pd.merge_asof(left, right, on='a') == kx.merge_asof(qleft, qright, on='a').pd()).all().all() + assert (pd.merge_asof(left, right, on='a', by="b") + == kx.merge_asof(qleft, qright, on='a', by="b").pd()).all().all() assert (pd.merge_asof(left, right, on='a') == qleft.merge_asof(qright, on='a').pd()).all().all() assert (pd.merge_asof(left, right, on='a') @@ -2658,3 +2661,13 @@ def test_merge_qjoin_errors(kx): match=r"Right Join requires a keyed table" " for the left dataset."): assert tab1.merge(tab2, how='right', q_join=True) + + +def test_fills(kx, q): + t = q('([] n:0N 2 3 0N 0N 7 0N;b:``h`h```j`;g:0N 1 0N 2 0N 3 0N)') + assert kx.q('~', t.fills(), kx.q('fills', t)) + + +def test_ungroup(kx, q): + t = q('([] n:1 2 3;nn:(1 1;2 2 2;3 3 3))') + assert kx.q('~', t.ungroup(), kx.q('ungroup', t)) diff --git a/tests/test_pykx.py b/tests/test_pykx.py index 22b744d..0e4ed70 100644 --- a/tests/test_pykx.py +++ b/tests/test_pykx.py @@ -292,10 +292,6 @@ def test_pykx_star(): os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) -@pytest.mark.skipif( - (sys.version_info.major == 3) and (sys.version_info.minor == 8), - reason="python3.8 subprocess behavior inconsistent with newest versions" -) def test_pykx_stdout_stderr(): output = subprocess.run( (str(Path(sys.executable).as_posix()), '-c', @@ -314,10 +310,6 @@ def test_pykx_stdout_stderr(): os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) -@pytest.mark.skipif( - (sys.version_info.major == 3) and (sys.version_info.minor == 8), - reason="python3.8 subprocess behavior inconsistent with newest versions" -) def test_pykx_stdout_stderr_under_q(): subprocess.run( (str(Path(sys.executable).as_posix()), '-c', @@ -448,31 +440,31 @@ def test_kx_versions(kx): if test_vars == ('Linux', 'x86_64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Linux', 'x86_64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') + assert kx.q.z.k == kx.q('2026.07.06') # elif test_vars == ('Linux', 'x86_64', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Linux', 'aarch64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Linux', 'aarch64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') + assert kx.q.z.k == kx.q('2026.07.06') # elif test_vars == ('Linux', 'aarch64', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Darwin', 'x86_64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Darwin', 'x86_64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') + assert kx.q.z.k == kx.q('2026.07.06') # elif test_vars == 'Darwin', 'x86_64', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Darwin', 'arm64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Darwin', 'arm64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') + assert kx.q.z.k == kx.q('2026.07.06') # elif test_vars == ('Darwin', 'arm', '4.2'): # assert kx.q.z.k == kx.q('?') elif test_vars == ('Windows', 'AMD64', '4.0'): assert kx.q.z.k == kx.q('2025.02.18') elif test_vars == ('Windows', 'AMD64', '4.1'): - assert kx.q.z.k == kx.q('2026.04.01') + assert kx.q.z.k == kx.q('2026.07.06') # elif test_vars == ('Windows', 'AMD64', '4.2'): # assert kx.q.z.k == kx.q('?') else: diff --git a/tests/test_streamlit.py b/tests/test_streamlit.py index 41970cd..03cc6d9 100644 --- a/tests/test_streamlit.py +++ b/tests/test_streamlit.py @@ -7,7 +7,6 @@ import streamlit as st -@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher") def test_streamlit(kx, q_port): conn = st.connection('pykx', type=kx.streamlit.PyKXConnection, host='localhost', port=q_port) diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index 4824067..d5379cc 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -3504,6 +3504,17 @@ def test_nested_keyed_dict(self, q): '(`c`d)!([] metaa: (19; 14); metab: (17; 16)));' 'data: (1; 2))').py() == double_nested + def test_setitem(self, kx): + a = kx.q('`a`b!(`;2)') + c = kx.q('([] a:enlist 2;b:enlist 6)') + a['c'] = c + assert isinstance(a['c'], kx.Table) + + a = kx.q('`a`b!(`;2)') + d = kx.q('([] a:1 2;b:3 6)') + a['d'] = d + assert isinstance(a['d'], kx.Table) + class Test_KeyedTable: kt = '([k1:100+til 3] x:til 3; y:`singly`keyed`table)' @@ -5760,23 +5771,152 @@ def test_column_time(self, kx): assert all(tab.exec(kx.Column('a').time) == kx.TimeVector(kx.q('20:31:15.070 11:09:13.020'))) # noqa E:501 + def test_column_bin(self, kx): + t = kx.q('([] n: 2 4)') + assert all(t.select(kx.Column('n').bin([1, 3, 5])) == kx.q('select 1 3 5 bin n from ([] n: 2 4)')) # noqa E:501 -def test_column_types(kx): - a = kx.Column(name='age', data=(1, 2, 3)) - assert (a._name=='age' and a._data==(1, 2, 3)) - - with pytest.raises(TypeError) as err: - kx.Column(data=(1, 2, 3)) - assert "'name' cannot be None" in str(err) - - with pytest.raises(TypeError) as err: - kx.Column(age=5, data=(1, 2, 3)) - assert "name can only be of type Str and pykx.SymbolAtom" in str(err) - + with pytest.raises(kx.QError): + t.select(kx.Column('n').bin(['a', 'b', 'c'])) + + def test_column_enlist(self, kx): + t = kx.q('([] n: 2 4)') + assert all(t.select(kx.Column('n').enlist(iterator='each')) == kx.q('select enlist each n from ([] n: 2 4)')) # noqa E:501 + t2 = kx.q('([] n:((2 3);(4 5)))') + assert (t2.exec(kx.Column('n').enlist(iterator='each')) == kx.q('exec enlist each n from ([] n:((2 3);(4 5)))')).all() # noqa E:501 + + def test_column_any(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n').any()) == kx.q('select any n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_all(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n').all()) == kx.q('select all n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_not(self, kx): + t1 = kx.q('([] n: 3#1b)') + assert all(t1.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 3#1b)')) + + t2 = kx.q('([] n: 3#0b)') + assert all(t2.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 3#0b)')) + + t3 = kx.q('([] n: 0b, 0b, 1b, 0b)') + assert all(t3.select(kx.Column('n')._not()) == kx.q('select not n from ([] n: 0b, 0b, 1b, 0b)')) # noqa E:501 + + def test_column_in(self, kx): + t = kx.q('([] n: til 5)') + assert (t.exec(kx.Column('n')._in(3)) == kx.q('exec n in 3 from ([] n: til 5)')).all() + + def test_column_except(self, kx): + t = kx.q('([] n: til 5)') + assert (t.exec(kx.Column('n')._except(3)) == kx.q('exec n except 3 from ([] n: til 5)')).all() # noqa E:501 + + def test_column_hsym(self, kx): + t = kx.q('([] n: `a`b`c)') + assert (t.exec(kx.Column('n').hsym()) == kx.q('exec hsym n from ([] n : `a`b`c)')).all() + + def test_column_raze(self, kx): + t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + assert (t.exec(kx.Column('n').raze()) == kx.q('exec raze n from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + assert (t.exec(kx.Column('n').raze(iterator='over')) == kx.q('exec raze/[n] from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + + def test_column_type(self, kx): + t = kx.q('([] n:(1 2;(3 4;5 6);7;8))') + assert (t.exec(kx.Column('n').type(iterator='each')) == kx.q('exec type each n from ([] n:(1 2;(3 4;5 6);7;8))')).all() # noqa E:501 + + def test_column_key(self, kx): + d1 = kx.q('`a`b`c ! 1 2 3') + kx.q['d1'] = d1 + d2 = kx.q('`d`e`f ! 4 5 6') + kx.q['d2'] = d2 + t = kx.Table(data={'n': [d1, d2]}) + assert all(t.select(kx.Column('n').key(iterator='each')) == kx.q('select key each n from ([] n:(d1;d2))')) # noqa E:501 + + def test_column_get(self, kx): + kx.q('`:tb1 set (`a`b`c)') + kx.q('`:tb2 set (1 2 3)') + t = kx.q('([] n: `:tb1`:tb2)') + assert (t.exec(kx.Column('n').get(iterator='each')) == kx.q('exec get each n from ([] n: `:tb1`:tb2)')).all() # noqa E:501 + os.remove('tb1') + os.remove('tb2') + + def test_column_read0(self, kx): + kx.q('`:t1.txt 0: enlist "hello"') + kx.q('`:t2.txt 0: enlist "goodbye"') + t = kx.q('([] n:`:t1.txt`:t2.txt)') + assert (t.exec(kx.Column('n').read0(iterator='each')) == kx.q('exec read0 each n from ([]n:`:t1.txt`:t2.txt)')).all() # noqa E:501 + os.remove('t1.txt') + os.remove('t2.txt') + + def test_column_read1(self, kx): + kx.q('`:t1.txt 0: enlist "hello"') + kx.q('`:t2.txt 0: enlist "goodbye"') + t = kx.q('([] n:`:t1.txt`:t2.txt)') + assert (t.exec(kx.Column('n').read1(iterator='each')) == kx.q('exec read1 each n from ([]n:`:t1.txt`:t2.txt)')).all() # noqa E:501 + os.remove('t1.txt') + os.remove('t2.txt') + + def test_column_next(self, kx): + t = kx.q('([] n:3 6 9 12)') + assert (t.exec(kx.Column('n').call('next')) == kx.q('exec next n from ([]n: 3 6 9 12)')).all() # noqa E:501 + + def test_column_prior(self, kx): + t = kx.q('([] n:1 2 3 3 2 1)') + assert t.exec(kx.Column('n').prior(kx.q('<')) == kx.q('000011b')).all() + + def test_column_where(self, kx): + t = kx.q('([] a:101010b)') + assert (t.exec(kx.Column('a').where()) == kx.q('0 2 4')).all() + + def test_column_types(self, kx): + a = kx.Column(name='age', data=(1, 2, 3)) + assert (a._name=='age' and a._data==(1, 2, 3)) -def test_column_bin(kx): - t = kx.q('([] n: 2 4)') - assert all(t.select(kx.Column('n').bin([1, 3, 5])) == kx.q('select 1 3 5 bin n from ([] n: 2 4)')) # noqa E:501 + with pytest.raises(TypeError) as err: + kx.Column(data=(1, 2, 3)) + assert "'name' cannot be None" in str(err) - with pytest.raises(kx.QError): - t.select(kx.Column('n').bin(['a', 'b', 'c'])) + with pytest.raises(TypeError) as err: + kx.Column(age=5, data=(1, 2, 3)) + assert "name can only be of type Str and pykx.SymbolAtom" in str(err) + + def test_column_data_str(self, kx): + t = kx.q('([] a:1 2 3)') + r = kx.q('([] a:1 2 3;n:`t`t`t)') + kx.q['s'] = 't' + kx.q['svec'] = ['t', 't', 't'] + assert kx.q('~', r, t.update(kx.Column('n', data='t'))) + assert kx.q('~', r, t.update(kx.Column('n', data=['t', 't', 't']))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.SymbolAtom('t')))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.SymbolVector(['t', 't', 't'])))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.Variable('s')))) + assert kx.q('~', r, t.update(kx.Column('n', data=kx.Variable('svec')))) + assert kx.q('~', r, t.update(kx.Column('n').data('t'))) + assert kx.q('~', r, t.update(kx.Column('n').data(['t', 't', 't']))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.SymbolAtom('t')))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.SymbolVector(['t', 't', 't'])))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.Variable('s')))) + assert kx.q('~', r, t.update(kx.Column('n').data(kx.Variable('svec')))) + + def test_column_getitem(self, kx): + t = kx.q('([] a:5 10 15 20)') + assert kx.q('~', kx.q('([] a:10 15)'), t.select(kx.Column('a')[[1, 2]])) + kx.q['myvar'] = [1, 2] + assert kx.q('~', kx.q('([] a:10 15)'), t.select(kx.Column('a')[kx.Variable('myvar')])) + + def test_column_cast(self, kx): + t = kx.q('([] a:("this";"that"))') + assert kx.q('~', kx.q('([] a:`this`that)'), t.select(kx.Column('a').cast('symbol'))) + t = kx.q('([] a:1 2 3)') + assert kx.q('~', kx.q('([] a:1 2 3i)'), t.select(kx.Column('a').cast('int')))