+ Don't worry – even the best Python code sometimes raises an exception. + This page may have moved, been renamed, or become unavailable. +
+ ++ Use the site search above, or choose a KDB-X Python documentation area. +
+ + tags
@@ -142,7 +149,6 @@ plugins:
- comparisons.md # Skipped due to false positives
- performance.md # Skipped due to false positives
- user-guide/advanced/Pandas_API.ipynb
- - examples/interface-overview.ipynb
- examples/jupyter-integration.ipynb
- examples/db-management.ipynb
- examples/streaming/Evolving System.ipynb
@@ -167,19 +173,20 @@ plugins:
'beta-features/streamlit.md': 'user-guide/advanced/streamlit.md'
'beta-features/threading.md': 'user-guide/advanced/threading.md'
'changelog.md': 'release-notes/changelog.md'
- 'comparisons.md': 'getting-started/what_is_pykx.md'
+ 'comparisons.md': 'getting-started/what-is-kdb-x-python.md'
'evaluating.md': 'user-guide/fundamentals/evaluating.md'
'examples/threaded_execution/index.md': 'examples/threaded_execution/threading.md'
'examples/threaded_execution/threaded_execution.md': 'examples/threaded_execution/threading.md'
- 'extras/comparisons.md': 'getting-started/what_is_pykx.md'
+ 'extras/comparisons.md': 'getting-started/what-is-kdb-x-python.md'
'extras/faq.md': 'help/faq.md'
'extras/known_issues.md': 'help/issues.md'
'faq.md': 'help/faq.md'
- 'getting-started/interface_overview.md': 'getting-started/what_is_pykx.md'
+ 'getting-started/interface_overview.md': 'examples/interface-overview.md'
+ 'getting-started/what_is_pykx.md': 'getting-started/what-is-kdb-x-python.md'
'getting-started/q_magic_command.md': 'examples/jupyter-integration.ipynb'
'indexing.md': 'user-guide/fundamentals/indexing.md'
'installing.md': 'getting-started/installing.md'
- 'interface_overview.md': 'getting-started/what_is_pykx.md'
+ 'interface_overview.md': 'examples/interface-overview.md'
'known_issues.md': 'help/issues.md'
'licenses.md': 'license.md'
'limitations.md': 'help/issues.md'
@@ -215,6 +222,10 @@ theme:
- content.code.annotate
- content.action.edit
- content.code.copy
+ - search.highlight
+ - navigation.tracking
+ - navigation.top
+ - toc.follow
palette:
- media: "(prefers-color-scheme: light)"
scheme: kx-light
@@ -231,13 +242,13 @@ theme:
docs_dir: docs
nav:
- - Home:
+ - Home:
- Overview: index.md
- - Get Started:
- - Install: getting-started/installing.md
- - Quickstart guide: getting-started/quickstart.md
+ - About: getting-started/what-is-kdb-x-python.md
+ - Install: getting-started/installing.md
+ - Quickstart: getting-started/quickstart.md
- Learn:
- - What is KDB-X Python?: getting-started/what_is_pykx.md
+ - Fundamentals: examples/interface-overview.md
- Objects and attributes: learn/objects.md
- Databases: user-guide/advanced/database/index.md
- Modes of operation: user-guide/advanced/modes.md
@@ -313,7 +324,7 @@ nav:
- Python to KDB-X: api/pykx-q-data/toq.md
- Wrappers: api/pykx-q-data/wrappers.md
- KDB-X to Python: api/pykx-q-data/type_conversions.md
- - Pandas API: user-guide/advanced/Pandas_API.ipynb
+ - pandas API: user-guide/advanced/Pandas_API.ipynb
- Exceptions: api/exceptions.md
- Integrations:
- NumPy: user-guide/advanced/numpy.md
@@ -321,7 +332,6 @@ nav:
- Python charting libraries: examples/charting.ipynb
- Jupyter Integration: examples/jupyter-integration.ipynb
- Examples:
- - KDB-X Python Introduction Notebook: examples/interface-overview.ipynb
- Subscriber: examples/subscriber/readme.md
- Compress and encrypt: examples/compress_and_encrypt/readme.md
- Database creation and management: examples/db-management.ipynb
diff --git a/setup.py b/setup.py
index 3f2cd03..a08c460 100755
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,6 @@
'Windows': 'w64',
}[system]
-py_minor_version = sys.version_info[1]
debug = os.environ.get('PYKX_DEBUG', '').lower() in {'true', '1'}
windows_vcpkg_content = script_dir.resolve()/'vcpkg'/'installed'/'x64-windows-static-md'
@@ -50,7 +49,7 @@
windows_library_dirs = () if system != 'Windows' else (str(Path(sys.exec_prefix)/'libs'),
str(Path(sys.base_exec_prefix)/'libs'),
str(windows_vcpkg_content/'lib'))
-windows_libraries = () if system != 'Windows' else ('psapi', 'q')
+windows_libraries = () if system != 'Windows' else ('psapi', 'qdll')
class CustomInstallCommand(install):
@@ -111,7 +110,7 @@ def build_q_c_extension(self, compiler, lib, lib_ext, library=None):
extra_preargs=[
*(
('-undefined dynamic_lookup',) if system == 'Darwin' else
- ('/LD /Fepykxq.dll q.lib -I include',) if system == 'Windows' else ()
+ ('/LD /Fepykxq.dll qdll.lib -I include',) if system == 'Windows' else ()
),
],
),
diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py
index 08fd45e..8898c01 100644
--- a/src/pykx/__init__.py
+++ b/src/pykx/__init__.py
@@ -7,12 +7,20 @@
import os
import sys
+if os.getenv('PYKX_LOADING_Q', '') == '':
+ os.environ['PYKX_LOADING_PYTHON'] = 'true'
+
+if os.getenv('PYKX_OLD_QHOME', '') != '' and os.getenv('PYKX_LOADING_PYTHON', '') != '':
+ os.environ['QHOME'] = os.getenv('PYKX_OLD_QHOME', "")
+if os.getenv('PYKX_OLD_QPATH', '') != '' and os.getenv('PYKX_LOADING_PYTHON', '') != '':
+ os.environ['QPATH'] = os.getenv('PYKX_OLD_QPATH', "")
+
if os.getenv('PYKX_LOADED_UNDER_Q') == 'True':
sys.exit(0)
# Attempt to import PyArrow early to get ahead of others (e.g. Pandas) who would try to import it
# without guarding against segfaults. Skip this if we're just doing a qinit check.
-if os.environ.get('PYKX_QINIT_CHECK') is None:
+if os.getenv('PYKX_QINIT_CHECK', '') == '':
try:
from ._pyarrow import pyarrow
except ImportError: # nocov
@@ -561,3 +569,20 @@ def deactivate_numpy_allocator():
def __dir__():
return __all__
+
+
+envlist = (
+ 'PYKX_UNDER_Q',
+ 'PYKX_UNDER_PYTHON',
+ 'PYKX_Q_LOADED_MARKER',
+ 'PYKX_LOADED_UNDER_Q',
+ 'PYKX_LOADING_PYTHON',
+ 'PYKX_LOADING_Q',
+ 'PYKX_QINIT_CHECK',
+)
+if os.getenv('PYKX_LOADING_PYTHON', '') != '':
+ for x in envlist:
+ os.environ[x] = ''
+ del os.environ[x]
+ if licensed:
+ q(f'setenv[`{x}; ""]')
diff --git a/src/pykx/_tcore.c b/src/pykx/_tcore.c
index cbced2a..3105955 100644
--- a/src/pykx/_tcore.c
+++ b/src/pykx/_tcore.c
@@ -487,6 +487,7 @@ struct QInit {
void* q_thread_init(void* _qini) {
struct QInit* qini = (struct QInit*)_qini;
qinit_rc = _qinit(qini->argc, qini->argv, qini->qhome, qini->qlic, qini->qqq);
+ // `qini` is owned and freed by q_init once it observes qinit_rc; do not free here.
pthread_mutex_lock(&init_mutex);
pthread_cond_signal(&init);
pthread_mutex_unlock(&init_mutex);
@@ -546,12 +547,19 @@ int q_init(int argc, char** argv, char* qhome, char* qlic, char* qqq) {
qini->qqq = qqq; // TODO: ADD COMMENT
qinit_rc = -256;
int rc = pthread_create(&q_thread, NULL, q_thread_init, (void*)qini);
+ if (rc != 0) {
+ // q_thread_init never runs, so free here and avoid waiting on `init` forever.
+ free(qini);
+ return rc;
+ }
pthread_mutex_lock(&init_mutex);
while (qinit_rc == -256) {
pthread_cond_wait(&init, &init_mutex);
}
pthread_mutex_unlock(&init_mutex);
+ // Safe to free now: q_thread_init finished reading `qini` before signalling `init`.
+ free(qini);
return qinit_rc;
}
diff --git a/src/pykx/config.py b/src/pykx/config.py
index ea5009c..c4a9b9e 100644
--- a/src/pykx/config.py
+++ b/src/pykx/config.py
@@ -34,6 +34,7 @@
# If PYKX_CONFIGURATION_LOCATION is not set it will search '.'
pykx_config_location = os.path.expanduser(os.getenv('PYKX_CONFIGURATION_LOCATION', ''))
pykx_config_profile = os.getenv('PYKX_PROFILE', 'default')
+unlicensed_reason = None
def _get_config_value(param, default):
@@ -55,10 +56,6 @@ def _is_enabled(param, cmdflag=False):
return env_config or (cmdflag and cmdflag in qargs)
-def _is_set(envvar):
- return os.getenv(envvar, None)
-
-
pykx_config_locs = [Path.home()/'.kx/config-pykx']
if pykx_config_location != '':
pykx_config_locs = [Path(pykx_config_location)] + pykx_config_locs
@@ -157,7 +154,7 @@ def _get_qcfg_values():
def _get_qhome():
qhome_value = _get_config_value('QHOME', None)
- if qhome_value is not None:
+ if qhome_value:
try:
qhome = Path(qhome_value).resolve(strict=True)
return qhome
@@ -174,7 +171,15 @@ def _get_qhome():
def _get_qpath(qhome):
qpath = _get_config_value('QPATH', None)
- os.environ['QPATH'] = str(qhome/'mod' if qpath is None else qpath+':'+str(qhome/'mod'))
+ mod = str(qhome / 'mod')
+ if qpath is None or qpath == '':
+ result = mod
+ elif mod in qpath.split(os.pathsep):
+ result = qpath
+ else:
+ result = qpath + os.pathsep + mod
+ os.environ['QPATH'] = result
+ return result
if 'QHOME' in _qcfg_values:
@@ -187,6 +192,8 @@ def _get_qpath(qhome):
qhome = _get_qhome()
qpath = _get_qpath(qhome)
+os.environ['PYKX_OLD_QHOME'] = str(qhome) if qhome is not None else ""
+os.environ['PYKX_OLD_QPATH'] = str(qpath) if qpath is not None else ""
# License search
if 'QLIC' in _qcfg_values:
@@ -285,6 +292,7 @@ def _license_install_B64(license, license_type): # pragma: no cover
def _license_check(lic_type, lic_encoding, lic_variable): # pragma: no cover
+ global unlicensed_reason
license_content = None
lic_name = lic_type + '.lic'
lic_file = qlic / lic_name
@@ -299,6 +307,7 @@ def _license_check(lic_type, lic_encoding, lic_variable): # pragma: no cover
f' License location: {qlic}/{lic_type}.lic\n'\
'Reason: License content matches supplied Environment variable'
print(conflict_message)
+ unlicensed_reason = f"Updating license failed. {lic_variable} contents matched contents of {qlic}/{lic_type}.lic." # noqa: E501
return False
else:
return _license_install_B64(lic_encoding, lic_name)
@@ -333,7 +342,7 @@ def _unlicensed_config(unlicensed_message):
def _license_install(intro=None, return_value=False, license_check=False, license_error=None): # noqa:
-
+ global unlicensed_reason
if not hasattr(sys, 'ps1'): # Exit if running in a non-interactive session
return False
@@ -371,6 +380,7 @@ def _license_install(intro=None, return_value=False, license_check=False, licens
f' QHOME {qhome if qhome else "Not Set"}\n\n'\
'Running KDB-X Python in unlicensed mode has reduced functionality.\n'\
'Would you like to install a license? [Y/n]: '
+ unlicensed_reason = "No license was found."
root = 'C:\\path\\to\\' if platform.system() == 'Windows' else '~/path/to/'
continue_license = input(first_user if intro is None else intro)
if continue_license in ('n', 'N'):
@@ -458,6 +468,22 @@ def _license_install(intro=None, return_value=False, license_check=False, licens
licensed = False
_pykx_force_unlicensed = ('--unlicensed' in qargs or _is_enabled('PYKX_UNLICENSED', '--unlicensed')) or light_load # noqa: E501
+
+try:
+ config_unlicensed_flag = pykx_profile_content['PYKX_UNLICENSED']
+except KeyError:
+ config_unlicensed_flag = False
+
+if _pykx_force_unlicensed:
+ if '--unlicensed' in qargs:
+ unlicensed_reason = ("--unlicensed passed to args.")
+ elif light_load:
+ unlicensed_reason = ("light_load enabled.")
+ elif os.environ.get('PYKX_UNLICENSED') and unlicensed_reason is None:
+ unlicensed_reason = ("The environment variable PYKX_UNLICENSED is set to True.")
+ elif config_unlicensed_flag:
+ unlicensed_reason = (f"PYKX_UNLICENSED is set to True in the config file {pykx_config_location}.") # noqa: E501
+
_pykx_force_licensed = ('--licensed' in qargs or _is_enabled('PYKX_LICENSED', '--licensed')) and not light_load # noqa: E501
qlib_location = Path(_get_config_value('PYKX_Q_LIB_LOCATION', pykx_libs_dir))
@@ -510,6 +536,24 @@ def _set_licensed(licensed_):
licensed = licensed_
+def _set_unlicensed_reason(unlicensed_reason_):
+ global unlicensed_reason
+ unlicensed_reason = unlicensed_reason_
+
+
+def _get_unlicensed_reason():
+ return unlicensed_reason
+
+
+def _set_qdebug(qdebug_):
+ global pykx_qdebug
+ pykx_qdebug = qdebug_
+
+
+def _get_qdebug():
+ return pykx_qdebug
+
+
def _set_keep_local_times(keep_local_times_):
global keep_local_times
keep_local_times = keep_local_times_
diff --git a/src/pykx/core.pxd b/src/pykx/core.pxd
index 3af8dee..845dfb2 100644
--- a/src/pykx/core.pxd
+++ b/src/pykx/core.pxd
@@ -94,7 +94,7 @@ cdef K (*ktj)(short _type, long long x)
cdef K (*ktn)(int _type, long long length)
cdef K (*ku)(U x)
cdef K (*kz)(double x)
-cdef void (*m9)()
+cdef void (*_m9)()
cdef int (*okx)(K x)
cdef K (*orr)(const char*)
cdef void (*r0)(K k)
diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx
index a052f77..a116fc0 100644
--- a/src/pykx/core.pyx
+++ b/src/pykx/core.pyx
@@ -74,8 +74,8 @@ cdef void* _q_handle
# or fallback to unlicensed mode.
# Data is passed in via the `PYKX_QINIT_CHECK` env var to reduce `pykx` import time - why do the
# work to gather that data when the parent process has already done it?
-qinit_check_data = os.environ.get('PYKX_QINIT_CHECK')
-if qinit_check_data is not None: # nocov
+qinit_check_data = os.getenv('PYKX_QINIT_CHECK', '')
+if qinit_check_data != '': # nocov
_core_q_lib_path, _qhome_str, _qlic_str, _qargs = qinit_check_data.split(';') # nocov
import shlex # nocov
_qargs = list(shlex.split(_qargs)) # nocov
@@ -95,7 +95,7 @@ import subprocess
import sys
from .config import find_core_lib, qargs, qhome, qlic, pykx_lib_dir, \
- release_gil, _set_licensed, under_q, use_q_lock, _qlic
+ release_gil, _set_licensed, _set_unlicensed_reason, under_q, use_q_lock, _qlic
from .exceptions import PyKXException, PyKXWarning
final_qhome = str(qhome if ignore_qhome else pykx_lib_dir)
@@ -301,7 +301,7 @@ if not pykx_threading:
_capout_msg = f'Captured output from initialization attempt:\n{_qinit_output}'
_paths_checked = f' QLIC {_qlic if _qlic else "Not Set"}\n'\
f' QHOME {qhome if qhome else "Not Set"}'
- _lic_location = f'License used:\n {lic_path}'
+ _lic_location = f'\n {lic_path}'
else:
_capout_msg = '' # nocov - this can only occur under extremely weird circumstances.
_lic_location = '' # nocov - this additional line is to ensure this code path is covered.
@@ -313,6 +313,7 @@ if not pykx_threading:
f'{_lic_location}\n\n'\
'Would you like to renew your license? (Selecting no will proceed with unlicensed mode) [Y/n]: '
_license_message = _license_install(_exp_license, True, True, 'exp')
+ _set_unlicensed_reason(f"Your license is expired: {_lic_location}")
elif re.compile('licen[cs]e error: embedq').search(_capout_msg):
_ce_license = 'You appear to be using a non KDB-X Python enabled license.\n\n'\
f'{_capout_msg}\n\n'\
@@ -321,6 +322,7 @@ if not pykx_threading:
'has reduced functionality.\nWould you like to install '\
'a license? [Y/n]: '
_license_message = _license_install(_ce_license, True)
+
elif re.compile('licen[cs]e error: upd').search(_capout_msg):
_upd_license = 'Your installed license is out of date for this version'\
' of KDB-X Python and must be updated.\n\n'\
@@ -328,12 +330,14 @@ if not pykx_threading:
f'{_lic_location}\n\n'\
'Would you like to install an updated license? [Y/n]: '
_license_message = _license_install(_upd_license, True)
+ _set_unlicensed_reason(f"Your license requires an update: {_lic_location}")
elif re.compile('licen[cs]e error: k[xc4].lic').search(_capout_msg) or re.compile('licen[cs]e error: badmsg').search(_capout_msg):
_k_license = '\nThe KDB-X Python license found is corrupt or incompatible.\n\n'\
f'{_capout_msg}\n\n'\
f'{_lic_location}\n\n'\
'Would you like to install a new license? [Y/n]: '
_license_message = _license_install(_k_license, True)
+ _set_unlicensed_reason(f"Your license is corrupt or incompatible: {_lic_location}")
else:
warning = f'Failed to initialize KDB-X Python successfully with the following error: {_capout_msg}\n\n'
if _lic_location == '':
@@ -344,6 +348,7 @@ if not pykx_threading:
_missing_license = 'Running KDB-X Python in unlicensed mode has reduced functionality.\n\n'\
'Would you like to install a license? (Selecting no will proceed with unlicensed mode) [Y/n]: '
_license_message = _license_install(_missing_license, True)
+ _set_unlicensed_reason(f"Your license could not be found.")
if (not _license_message) and _qinit_check_proc.returncode:
if _pykx_force_licensed:
raise PyKXException(f'Failed to initialize embedded q.{_capout_msg}')
@@ -360,7 +365,7 @@ if not pykx_threading:
_qinit_args = {'qhome': final_qhome, 'qlic': qlic, 'ignore_qhome': ignore_qhome, 'qargs': list(qargs)}
raise PyKXException(f'Non-zero qinit following license install with configuration: {_qinit_args}\n'
f'failed with output: {_qinit_output}')
- if 'QHOME' in os.environ and not ignore_qhome:
+ if 'QHOME' in os.environ and not ignore_qhome and os.environ['QHOME'] != '':
# Only link the user's QHOME to KDB-X Python's QHOME if the user actually set $QHOME.
# Note that `pykx.qhome` has a default value of `./q`, as that is the behavior
# employed by q.
@@ -432,6 +437,8 @@ cpdef shutdown_thread():
if pykx_threading:
_shutdown_thread()
+cpdef m9():
+ _m9()
b9 = dlsym(_q_handle, sym_name('b9'))
d9 = dlsym(_q_handle, sym_name('d9'))
@@ -471,7 +478,7 @@ ktj = dlsym(_q_handle, sym_name('ktj'))
ktn = dlsym(_q_handle, sym_name('ktn'))
ku = dlsym(_q_handle, sym_name('ku'))
kz = dlsym(_q_handle, sym_name('kz'))
-m9 = dlsym(_q_handle, sym_name('m9'))
+_m9 = dlsym(_q_handle, sym_name('m9'))
okx = dlsym(_q_handle, sym_name('okx'))
orr = dlsym(_q_handle, sym_name('orr'))
r0 = dlsym(_q_handle, sym_name('r0'))
diff --git a/src/pykx/db.py b/src/pykx/db.py
index d49df09..bdebc2d 100644
--- a/src/pykx/db.py
+++ b/src/pykx/db.py
@@ -681,7 +681,7 @@ def add_column(self,
['testTable']
>>> db.list_columns('testTable')
['month', 'sym', 'time', 'price', 'size']
- >>> db.add_column('testTable', 'test', kx.IntAtom.null)
+ >>> db.add_column('testTable', 'test', kx.IntAtom.null)
>>> db.list_columns('testTable')
['month', 'sym', 'time', 'price', 'size']
```
@@ -1171,7 +1171,7 @@ def partition_count(self, *, subview: Optional[list] = None) -> k.Dictionary:
```python
>>> import pykx as kx
>>> db = kx.DB(path = 'newDB')
- >>> db.partition_count(sub_view = kx.q('2020.01 2020.02m'))
+ >>> db.partition_count(subview = kx.q('2020.01 2020.02m'))
pykx.Dictionary(pykx.q('
| trades quotes
-------| -------------
@@ -1218,7 +1218,7 @@ def subview(self, view: list = None) -> None:
```python
>>> import pykx as kx
>>> db = kx.DB(path = 'newDB')
- >>> db.subview(kx.q('2020.02 2020.03m')
+ >>> db.subview(kx.q('2020.02 2020.03m'))
>>> kx.q.qsql.select(db.trades, 'month')
pykx.Table(pykx.q('
month
@@ -1278,7 +1278,7 @@ def enumerate(self, table: str, *, sym_file: Optional[str] = None) -> k.Table:
... 'x': kx.random.random(N, ['a', 'b', 'c']),
... 'x1': kx.random.random(N, 1.0),
... 'x2': kx.random.random(N, 10)
- ... }
+ ... })
>>> tab = db.enumerate(tab)
>>> tab['x']
pykx.EnumVector(pykx.q('`sym$`a`b`a`c`b..'))
@@ -1294,7 +1294,7 @@ def enumerate(self, table: str, *, sym_file: Optional[str] = None) -> k.Table:
... 'x': kx.random.random(N, ['a', 'b', 'c']),
... 'x1': kx.random.random(N, 1.0),
... 'x2': kx.random.random(N, 10)
- ... }
+ ... })
>>> tab = db.enumerate(tab, sym_file = 'mysym')
>>> tab['x']
pykx.EnumVector(pykx.q('`mysym$`a`b`a`c`b..'))
diff --git a/src/pykx/embedded_q.py b/src/pykx/embedded_q.py
index 7838858..f4bdfad 100644
--- a/src/pykx/embedded_q.py
+++ b/src/pykx/embedded_q.py
@@ -11,7 +11,7 @@
from . import toq
from . import wrappers
from . import schema
-from .config import find_core_lib, licensed, pykx_dir, pykx_libs_dir, pykx_qdebug, pykx_threading, qargs, qce, skip_under_q, suppress_warnings, pykx_debug_insights # noqa
+from .config import find_core_lib, _get_qdebug, licensed, pykx_dir, pykx_libs_dir, pykx_threading, qargs, qce, skip_under_q, suppress_warnings, pykx_debug_insights # noqa
from .core import keval as _keval
from .exceptions import FutureCancelled, LicenseException, NoResults, PyKXException, PyKXWarning, QError # noqa
from ._wrappers import _factory as factory
@@ -144,14 +144,14 @@ def __init__(self): # noqa
.pykx.util.loadfile["{kxic_path}";"{kxic_file}"]
];
'''
- if os.getenv('PYKX_UNDER_Q') is None:
+ if os.getenv('PYKX_UNDER_Q', '') == '':
os.environ['PYKX_UNDER_PYTHON'] = 'true'
code += 'setenv[`PYKX_UNDER_PYTHON;"true"];'
code += f'2:[`$"{pykx_qlib_path}";(`k_pykx_init; 2)][`$"{find_core_lib("q").as_posix()}";{"1b" if pykx_threading else "0b"}];' # noqa: E501
code += f'`.pykx.modpow set {{((`$"{pykx_qlib_path}") 2: (`k_modpow; 3))["j"$x;"j"$y;$[z~(::);(::);"j"$z]]}};' # noqa: E501
else:
suffix = ""
- if os.getenv("PYKXQ_THREADING") is not None:
+ if os.getenv("PYKXQ_THREADING", '') != '':
suffix = "_m"
code += f'2:[`$"{pykx_qlib_path}q{suffix}";(`k_pykx_init; 2)][`$"{find_core_lib("q").as_posix()}";{"1b" if pykx_threading else "0b"}];' # noqa: E501
code += f'`.pykx.modpow set {{((`$"{pykx_qlib_path}q{suffix}") 2: (`k_modpow; 3))["j"$x;"j"$y;$[z~(::);(::);"j"$z]]}};' # noqa: E501
@@ -167,8 +167,8 @@ def __init__(self): # noqa
PyKXWarning)
else:
eprint(f'WARN: Failed to load KX Insights Core library {lib!r}.')
- if (not skip_under_q and os.getenv('PYKX_Q_LOADED_MARKER') != 'loaded'
- and os.getenv('PYKX_UNDER_Q') is None
+ if (not skip_under_q and os.getenv('PYKX_Q_LOADED_MARKER', '') != 'loaded'
+ and os.getenv('PYKX_UNDER_Q', '') == ''
):
os.environ['PYKX_Q_LOADED_MARKER'] = 'loaded'
self._call('setenv[`PYKX_Q_LOADED_MARKER; "loaded"]', skip_debug=True)
@@ -191,7 +191,7 @@ def __init__(self): # noqa
)
self._call('.pykx.setdefault[enlist"k"]', skip_debug=True)
if (self._call('@[{x in " " vs .z.l 7};"COMMUNITY";{0b}]', skip_debug=True).py()
- and os.getenv('PYKX_UNDER_Q') is None):
+ and os.getenv('PYKX_UNDER_Q', '') == ''):
print(("\nWelcome to KDB-X Community Edition!\n"
"For Community support, please visit https://kx.com/slack\n"
"Tutorials can be found at https://github.com/KxSystems/tutorials\n"
@@ -235,7 +235,7 @@ def __call__(self,
if len(args) > 8:
raise TypeError('Too many arguments - q queries cannot have more than 8 parameters')
query = wrappers.CharVector(query)
- if (not skip_debug) and (debug or pykx_qdebug):
+ if (not skip_debug) and (debug or _get_qdebug()):
if 0 != len(args):
query = wrappers.List([query, *[wrappers.K(x) for x in args]])
result = _keval(
diff --git a/src/pykx/exceptions.py b/src/pykx/exceptions.py
index 50df431..c7ab89e 100644
--- a/src/pykx/exceptions.py
+++ b/src/pykx/exceptions.py
@@ -34,11 +34,12 @@ class PyKXException(Exception):
class LicenseException(PyKXException):
"""Exception for when a feature that requires a valid q license is used without one."""
def __init__(self, feature_msg='use this feature', *args, **kwargs):
+ # Imported here rather than at module scope as pykx.config imports from this module
+ from .config import _get_unlicensed_reason
super().__init__(
- 'A valid q license must be in a known location (e.g. `$QLIC`) to '
- f'{feature_msg}.',
+ f'Cannot {feature_msg} as KDB-X Python is operating in unlicensed mode. {_get_unlicensed_reason()}', # noqa: E501
*args,
- **kwargs
+ **kwargs,
)
diff --git a/src/pykx/ipc.py b/src/pykx/ipc.py
index 0ec0fa3..8ea2846 100644
--- a/src/pykx/ipc.py
+++ b/src/pykx/ipc.py
@@ -19,7 +19,7 @@
import sys
from . import deserialize, serialize, Q
-from .config import max_error_length, pykx_lib_dir, pykx_qdebug, system
+from .config import _get_qdebug, max_error_length, pykx_lib_dir, system
from .core import licensed
from .exceptions import FutureCancelled, NoResults, PyKXException, QError, UninitializedConnection
from .util import get_default_args, normalize_to_bytes, normalize_to_str
@@ -327,7 +327,7 @@ def result(self) -> Any:
if self._cancelled_message != '':
print('Connection was lost no result', file=sys.stderr)
return None
- if self._debug or pykx_qdebug:
+ if self._debug or _get_qdebug():
if self._result._unlicensed_getitem(0).py() == True:
print((self._result._unlicensed_getitem(1).py()).decode(), file=sys.stderr)
raise QError(self._result._unlicensed_getitem(2).py().decode())
@@ -573,6 +573,8 @@ def _init(self,
reconnection_function: callable = reconnection_function,
connection_timeout: Optional[float] = None,
):
+ if self.__dict__.get('_initialized'):
+ return
credentials = f'{normalize_to_str(username, "Username")}:' \
f'{normalize_to_str(password, "Password")}'
object.__setattr__(self, '_connection_info', {
@@ -696,7 +698,7 @@ def _send(self,
if self.closed:
raise RuntimeError(_ipc_err_warning['closed'])
tquery = type(query)
- debugging = (not skip_debug) and (debug or pykx_qdebug)
+ debugging = (not skip_debug) and (debug or _get_qdebug())
if issubclass(tquery, SymbolicFunction):
if licensed:
query = query.func
@@ -1044,7 +1046,8 @@ def upd(self, table: str, data:Union[list, Table]) -> None:
```python
>>> import pykx as kx
>>> with kx.SyncQConnection(port=5050) as q:
- ... q.upd('trade', [kx.TimespanAtom('now') 'AAPL', 1.0])
+ ... q.upd('trade', [kx.TimespanAtom('now'), 'AAPL', 1.0])
+ >>> N = 100
>>> trades = kx.Table(data = {
... 'time': kx.TimespanAtom('now'),
... 'sym': kx.random.random(N, ['AAPL', 'MSFT', 'GOOG']),
@@ -1359,7 +1362,7 @@ def _call(self,
if not wait:
return K(None)
res = self._recv(locked=True)
- if skip_debug or not (debug or pykx_qdebug):
+ if skip_debug or not (debug or _get_qdebug()):
return res
if res._unlicensed_getitem(0).py() == True:
print((res._unlicensed_getitem(1).py()).decode(), file=sys.stderr)
@@ -1439,10 +1442,13 @@ def close(self) -> None:
self._writer.close()
try:
self._sock.shutdown(socket.SHUT_RDWR)
+ except BaseException:
+ pass
+ try:
self._sock.close()
- self._finalizer()
except BaseException:
pass
+ self._finalizer()
def fileno(self) -> int:
"""The file descriptor or handle of the connection."""
@@ -1948,10 +1954,13 @@ async def close(self) -> None:
self._writer.close()
try:
self._sock.shutdown(socket.SHUT_RDWR)
+ except BaseException:
+ pass
+ try:
self._sock.close()
- self._finalizer()
except BaseException:
pass
+ self._finalizer()
def fileno(self) -> int:
"""The file descriptor or handle of the connection."""
@@ -2020,10 +2029,13 @@ def close(self) -> None:
self._writer.close()
try:
self._sock.shutdown(socket.SHUT_RDWR)
+ except BaseException:
+ pass
+ try:
self._sock.close()
- self._finalizer()
except BaseException:
pass
+ self._finalizer()
def handshake(conn: socket.socket):
@@ -2792,10 +2804,13 @@ async def close(self) -> None:
self._writer.close()
try:
self._sock.shutdown(socket.SHUT_RDWR)
+ except BaseException:
+ pass
+ try:
self._sock.close()
- self._finalizer()
except BaseException:
pass
+ self._finalizer()
class SecureQConnection(QConnection):
@@ -3033,7 +3048,7 @@ def _call(self,
try:
with self._lock if self._lock is not None else nullcontext():
- if debug or pykx_qdebug:
+ if debug or _get_qdebug():
res = handler(
handle,
normalize_to_bytes(
diff --git a/src/pykx/lib/.gitignore b/src/pykx/lib/.gitignore
index 0af2287..77b1995 100644
--- a/src/pykx/lib/.gitignore
+++ b/src/pykx/lib/.gitignore
@@ -3,7 +3,7 @@
**/*q.so
**/*q.dylib
**/q.dll
-**/q.lib
+**/qdll.lib
**/kxreaper
**/libkurl.so
**/libobjstor.so
diff --git a/src/pykx/lib/kdbx-libs/.gitignore b/src/pykx/lib/kdbx-libs/.gitignore
index 0af2287..77b1995 100644
--- a/src/pykx/lib/kdbx-libs/.gitignore
+++ b/src/pykx/lib/kdbx-libs/.gitignore
@@ -3,7 +3,7 @@
**/*q.so
**/*q.dylib
**/q.dll
-**/q.lib
+**/qdll.lib
**/kxreaper
**/libkurl.so
**/libobjstor.so
diff --git a/src/pykx/license.py b/src/pykx/license.py
index be26faa..6d82608 100644
--- a/src/pykx/license.py
+++ b/src/pykx/license.py
@@ -153,7 +153,7 @@ def expires() -> int:
def install(license: str,
*,
format: Optional[str] = 'FILE',
- license_type: Optional[str] = 'kc.lic',
+ license_type: Optional[str] = None,
force: Optional[bool] = False
) -> bool:
"""
@@ -164,9 +164,10 @@ def install(license: str,
If "STRING" this is the base64 encoded string provided in your license email
format: Is the license check being completed using a downloaded file or base64
encoded string. Accepted inputs are "FILE"(default) or "STRING".
- license_type: The license file type/name which is to be checked, by default this
- is 'kc.lic' but can be changed to 'k4.lic' or 'kx.lic' if appropriate
- force: Enforce overwrite without opt-in message for overwrite
+ license_type: The license file type/name which is to be installed. If unspecified
+ and a "FILE" is being installed whose name is one of 'kc.lic', 'k4.lic' or
+ 'kx.lic', that name is used; otherwise it defaults to 'kc.lic'.
+ force: Enforce overwrite without opt-in message for overwrite.
Returns:
A boolean indicating if the license has been correctly overwritten
@@ -194,6 +195,12 @@ def install(license: str,
if format not in ('file', 'string'):
raise Exception('Unsupported option provided for format parameter')
+ if license_type is None:
+ known_types = ('kc.lic', 'k4.lic', 'kx.lic')
+ incoming_name = (os.path.basename(os.path.expanduser(Path(license)))
+ if format == 'file' else None)
+ license_type = incoming_name if incoming_name in known_types else 'kc.lic'
+
license_located = False
installed_lic = qlic/license_type
if os.path.exists(installed_lic):
@@ -210,7 +217,7 @@ def install(license: str,
if not os.path.exists(download_location):
raise Exception(f'Download location provided {download_location} does not exist.')
- shutil.copy(download_location, qlic)
+ shutil.copy(download_location, qlic/license_type)
else:
try:
lic = base64.b64decode(license)
@@ -220,5 +227,5 @@ def install(license: str,
with open(qlic/license_type, 'wb') as binary_file:
binary_file.write(lic)
- print("KDB-X Python license successfully installed!")
+ print(f"KDB-X Python license successfully installed: {qlic/license_type}")
return True
diff --git a/src/pykx/pandas_api/pandas_indexing.py b/src/pykx/pandas_api/pandas_indexing.py
index 9c57129..19cd505 100644
--- a/src/pykx/pandas_api/pandas_indexing.py
+++ b/src/pykx/pandas_api/pandas_indexing.py
@@ -327,9 +327,7 @@ def _rename_columns(tab, labels):
for x in list(labels.keys()):
if type(labels[x]) is not str:
raise ValueError('pykx.Table column names can only be of type pykx.SymbolAtom')
- if type(x) is not str:
- labels.pop(x)
- if x not in tab.columns:
+ if type(x) is not str or x not in tab.columns:
labels.pop(x)
if "Keyed" in str(type(tab)):
return q('''{
diff --git a/src/pykx/pandas_api/pandas_reset_index.py b/src/pykx/pandas_api/pandas_reset_index.py
index 179d46f..b0b1cae 100644
--- a/src/pykx/pandas_api/pandas_reset_index.py
+++ b/src/pykx/pandas_api/pandas_reset_index.py
@@ -62,7 +62,7 @@ def reset_index(self, levels=None, *, drop=False, inplace=False,
levels,
self)
if drop:
- res = q('{x[y;(),cols[key y]z]]}', drop_keys, self, levels) # noqa: E501
+ res = q('{x[y;(),cols[key y]z]}', drop_keys, self, levels) # noqa: E501
else:
res = q('{(cols[key x] except cols[key x]y) xkey x}', self, levels)
else:
diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q
index 86266e7..1082151 100644
--- a/src/pykx/pykx.q
+++ b/src/pykx/pykx.q
@@ -14,6 +14,10 @@ if[`e in key`.p;
]
]
+if["" ~ getenv`PYKX_LOADING_PYTHON; setenv[`PYKX_LOADING_Q; "true"]];
+if[not["" ~ getenv`PYKX_OLD_QHOME] and not["" ~ getenv`PYKX_LOADING_Q]; setenv[`QHOME; getenv`PYKX_OLD_QHOME]];
+if[not["" ~ getenv`PYKX_OLD_QPATH] and not["" ~ getenv`PYKX_LOADING_Q]; setenv[`QPATH; getenv`PYKX_OLD_QPATH]];
+
// @private
// @desc Process context prior to KDB-X Python initialization
.pykx.util.prevCtx:system"d";
@@ -158,8 +162,17 @@ util.CFunctions:flip `qname`cname`args!flip (
util.toPython:{wrap util.pyForeign[$[type[x]in 104 105 112h;wrap[unwrap x]`;x];y;z]}
// @private
-// @desc Check if argument is of specified type
-util.isch :{$[(104= type y);$[x~y[::]0;1b;0b];0b]}
+// @desc
+// Safely extract the pykx marker tag from a value, or ` if it is not a marker.
+util.mtag :{$[104h<>type x;`;
+ 104h<>type v0:(value x)0;`;
+ 2>count vv:value v0;`;
+ enlist~first vv;vv 1;
+ `]}
+
+// @private
+// @desc Check if argument is a pykx marker of the specified type
+util.isch :{x~util.mtag y}
// @private
// @desc
@@ -921,7 +934,6 @@ setdefault:{
x in enlist "default" ;"default";
'"unknown conversion type: ",x
];
- setenv[`PYKX_DEFAULT_CONVERSION;$[-10h=type util.defaultConv;enlist;]util.defaultConv];
}
// @kind function
@@ -1726,14 +1738,7 @@ qcallable:{$[util.isw x;wrap[unwrap[x]](<);util.isf x;wrap[x](<);'"Could not con
// "Hello World"
// ```
safeReimport:{[x]
- reimporter:.pykx.import[`pykx][`:PyKXReimport][];
- envlist:reimporter[`:envlist]`;
- envvals:getenv each envlist;
-
- reimporter[`:reset][];
r: @[{(0b;x y)}x;(::);{(1b;x)}];
-
- setenv'[envlist;envvals];
$[r 0;{'x};::] r 1
}
@@ -2016,3 +2021,13 @@ loadExtension:{[ext]
system"d ",string .pykx.util.prevCtx;
k).Q.pykxld:{x:("#!"~2#*x)_x:-1!'x;+(1+*:'i;)@"\n"/:'x i:(&|1^\|0N 0 1@"/ "?*:'(v:x i),'"/")_i:&~|':(b?-1)#b:+\-/x~\:/:+,"/\\"}
+
+if[not["" ~ getenv`PYKX_LOADING_Q];
+ setenv[`PYKX_UNDER_Q; ""];
+ setenv[`PYKX_UNDER_PYTHON; ""];
+ setenv[`PYKX_Q_LOADED_MARKER; ""];
+ setenv[`PYKX_LOADED_UNDER_Q; ""];
+ setenv[`PYKX_LOADING_Q; ""];
+ setenv[`PYKX_LOADING_PYTHON; ""];
+ setenv[`PYKX_QINIT_CHECK; ""];
+ ];
diff --git a/src/pykx/pykxq_m.c b/src/pykx/pykxq_m.c
index 7f2d038..d8a1ba8 100644
--- a/src/pykx/pykxq_m.c
+++ b/src/pykx/pykxq_m.c
@@ -205,11 +205,8 @@ EXPORT K k_init_python(K x, K y, K z) {
Py_DecRef(a);
}
PyErr_Clear();
- struct CallArgs* init_args = malloc(sizeof(struct CallArgs));
- init_args->arg1 = x;
- init_args->arg2 = y;
- init_args->arg3 = z;
- pthread_create(&thread, NULL, thread_init, (void*)init_args);
+ // thread_init does not read its argument, so no allocation is required here.
+ pthread_create(&thread, NULL, thread_init, NULL);
return (K)0;
}
diff --git a/src/pykx/q.so/libs/kdbx/l64/libq.so b/src/pykx/q.so/libs/kdbx/l64/libq.so
index 99d779b..f0acedb 100755
Binary files a/src/pykx/q.so/libs/kdbx/l64/libq.so and b/src/pykx/q.so/libs/kdbx/l64/libq.so differ
diff --git a/src/pykx/q.so/libs/kdbx/l64arm/libq.so b/src/pykx/q.so/libs/kdbx/l64arm/libq.so
index d9515f0..b6ff66e 100755
Binary files a/src/pykx/q.so/libs/kdbx/l64arm/libq.so and b/src/pykx/q.so/libs/kdbx/l64arm/libq.so differ
diff --git a/src/pykx/q.so/libs/kdbx/m64/libq.dylib b/src/pykx/q.so/libs/kdbx/m64/libq.dylib
index 298295b..2527ddc 100755
Binary files a/src/pykx/q.so/libs/kdbx/m64/libq.dylib and b/src/pykx/q.so/libs/kdbx/m64/libq.dylib differ
diff --git a/src/pykx/q.so/libs/kdbx/m64arm/libq.dylib b/src/pykx/q.so/libs/kdbx/m64arm/libq.dylib
index e5f2278..20f73d0 100755
Binary files a/src/pykx/q.so/libs/kdbx/m64arm/libq.dylib and b/src/pykx/q.so/libs/kdbx/m64arm/libq.dylib differ
diff --git a/src/pykx/q.so/libs/kdbx/w64/q.dll b/src/pykx/q.so/libs/kdbx/w64/q.dll
index f0ad5d4..400d92e 100644
Binary files a/src/pykx/q.so/libs/kdbx/w64/q.dll and b/src/pykx/q.so/libs/kdbx/w64/q.dll differ
diff --git a/src/pykx/q.so/libs/kdbx/w64/q.lib b/src/pykx/q.so/libs/kdbx/w64/q.lib
deleted file mode 100644
index a4beb9d..0000000
Binary files a/src/pykx/q.so/libs/kdbx/w64/q.lib and /dev/null differ
diff --git a/src/pykx/q.so/libs/kdbx/w64/qdll.lib b/src/pykx/q.so/libs/kdbx/w64/qdll.lib
new file mode 100644
index 0000000..8a999ab
Binary files /dev/null and b/src/pykx/q.so/libs/kdbx/w64/qdll.lib differ
diff --git a/src/pykx/q.so/qk/pykx_init.q_ b/src/pykx/q.so/qk/pykx_init.q_
index f9e96c5..682c2bf 100644
Binary files a/src/pykx/q.so/qk/pykx_init.q_ and b/src/pykx/q.so/qk/pykx_init.q_ differ
diff --git a/src/pykx/read.py b/src/pykx/read.py
index ce5d4a5..9fe87a5 100644
--- a/src/pykx/read.py
+++ b/src/pykx/read.py
@@ -59,7 +59,7 @@ def update_types(self, types):
upd_types.append(i)
else:
found = False
- for key in _type_mapping.keys():
+ for key in sorted(_type_mapping.keys(), key=len, reverse=True):
if key in t:
found = True
upd_types.append(_type_mapping[key])
@@ -278,8 +278,8 @@ def fixed(self,
Examples:
- Read a file of fixed width data into a `pykx.List` of two `pykx.LongVectors` the first
- with a size of 1 character and the second with a size of 2 characters.
+ Read a file of fixed width data into a `pykx.List` of two `pykx.LongVector` objects,
+ the first with a size of 1 character and the second with a size of 2 characters.
```python
data = q.read.fixed('example_file', [b'J', b'J'], [1, 2])
diff --git a/src/pykx/register.py b/src/pykx/register.py
index c60c6ae..00a01c8 100644
--- a/src/pykx/register.py
+++ b/src/pykx/register.py
@@ -73,7 +73,7 @@ def py_toq(py_type: Any,
```python
>>> def complex_toq_upd(data):
- ... return kx.q('{`real`imag!(x;y)}', kx.toq(data.real), kx.toq(data.imag)
+ ... return kx.q('{`real`imag!(x;y)}', kx.toq(data.real), kx.toq(data.imag))
>>> kx.register.py_toq(complex, complex_toq_upd, overwrite=True)
>>> kx.toq(complex(1, 2))
pykx.Dictionary(pykx.q('
@@ -138,7 +138,7 @@ def column_function(name: str,
>>> def min_max_scaler(self):
... return self.call('{(x-minData)%max[x]-minData:min x}')
>>> kx.register.column_function('minmax', min_max_scaler)
- >>> tab.select(kx.Column('true') & kx.Column('true').minmax().rename('scaled_true'))
+ >>> tab.select(kx.Column('true') & kx.Column('true').minmax().name('scaled_true'))
```
Register mean-absolute error function to be applied between 'true' and 'pred' columns
diff --git a/src/pykx/reimporter.py b/src/pykx/reimporter.py
index 4b17e86..f4374f9 100644
--- a/src/pykx/reimporter.py
+++ b/src/pykx/reimporter.py
@@ -9,8 +9,6 @@
"""
import os
-from .config import pykx_executable, qhome
-
class PyKXReimport:
"""Helper class to help manage the environment variables around reimporting KDB-X Python in a
@@ -27,19 +25,23 @@ class PyKXReimport:
# This process can safely import KDB-X Python
subprocess.Popen(f"python other_file.py")
```
+
+ Note: Not required in most cases.
+ KDB-X Python now automatically manages these environment variables, so this helper is not
+ needed for the majority of use cases (for example spawning a Python subprocess, or a
+ KDB-X 5.0 q process).
+
+ Note: Still required for spawning kdb+ 4.0/4.1 q subprocesses from Python.
+ When KDB-X Python is imported in a Python process it redirects `QHOME` to its internal
+ directory. A kdb+ 4.0/4.1 `q` child inheriting that `QHOME` fails to start because the
+ pykx-internal `QHOME` does not contain `q.k`. Wrapping the spawn in `PyKXReimport` restores
+ the user's `QHOME` (which contains `q.k`) for the child. This also applies where the QHOME
+ symlinks pykx creates are unavailable or unreliable (e.g. Windows). Alternatively set
+ `PYKX_IGNORE_QHOME=true` so `QHOME` is not redirected in the first place.
"""
def __init__(self):
- self.envlist = ('PYKX_DEFAULT_CONVERSION',
- 'PYKX_UNDER_Q',
- 'PYKX_UNDER_PYTHON',
- 'PYKX_SKIP_UNDERQ',
- 'PYKX_Q_LOADED_MARKER',
- 'PYKX_LOADED_UNDER_Q',
- 'QHOME',
- 'PYKX_EXECUTABLE',
- 'PYKX_DIR')
- self.envvals = [os.getenv(x) for x in self.envlist]
+ self.pykx_qhome = os.getenv('QHOME', '')
def __enter__(self):
self.reset()
@@ -51,12 +53,7 @@ def reset(self):
Note: It is not recommended to use this function directly instead use the `with` syntax.
This will automatically manage setting and restoring the environment variables for you.
"""
- for x, y in zip(self.envlist, self.envvals):
- os.unsetenv(x)
- if y is not None:
- del os.environ[x]
- os.environ['QHOME'] = str(qhome)
- os.environ['PYKX_EXECUTABLE'] = pykx_executable
+ os.environ['QHOME'] = os.getenv('PYKX_OLD_QHOME', self.pykx_qhome)
def restore(self):
"""Restore all the required environment variables.
@@ -64,12 +61,13 @@ def restore(self):
Note: It is not recommended to use this function directly instead use the `with` syntax.
This will automatically manage setting and restoring the environment variables for you.
"""
- for x, y in zip(self.envlist, self.envvals):
- if y is not None:
- os.environ[x] = y
+ os.environ['QHOME'] = self.pykx_qhome
def __exit__(self, exc_type, exc_value, exc_tb):
self.restore()
def __del__(self):
- self.restore()
+ # Best-effort restore; `__init__` may not have completed (e.g. QHOME unset).
+ qhome = getattr(self, 'pykx_qhome', None)
+ if qhome is not None:
+ os.environ['QHOME'] = qhome
diff --git a/src/pykx/schema.py b/src/pykx/schema.py
index d13963e..fb2ccec 100644
--- a/src/pykx/schema.py
+++ b/src/pykx/schema.py
@@ -40,7 +40,7 @@ def __dir__():
k.DatetimeAtom: "datetime", k.DatetimeVector: "datetime",
k.TimespanAtom: "timespan", k.TimespanVector: "timespan",
k.MinuteAtom: "minute", k.MinuteVector: "minute",
- k.SecondAtom: "second", k.SecondVector: "sector",
+ k.SecondAtom: "second", k.SecondVector: "second",
k.TimeAtom: "time", k.TimeVector: "time",
}
diff --git a/src/pykx/streamlit.py b/src/pykx/streamlit.py
index c415bfe..0f42689 100644
--- a/src/pykx/streamlit.py
+++ b/src/pykx/streamlit.py
@@ -167,7 +167,7 @@ def query(self, query: str, *args, format='q', **kwargs):
Parameters:
query: A q expression to be evaluated. This must be valid q, qSQL or SQL in the KX
Insights style.
- *args: Arguments to the query. Each argument will be converted into a `#!python pykx.K`
+ *args: Arguments to the query. Each argument will be converted into a `#!python pykx`
object. Up to 8 arguments can be provided (maximum supported by q functions).
format: Description of query format for internal pre-processing before the query is sent
to the server. This must be one of 'q', 'qsql' or 'sql'.
diff --git a/src/pykx/toq.pyx b/src/pykx/toq.pyx
index d8a2937..3ae5c7d 100644
--- a/src/pykx/toq.pyx
+++ b/src/pykx/toq.pyx
@@ -112,7 +112,7 @@ from . import wrappers as k
from ._pyarrow import pyarrow as pa
from .cast import *
from . import config
-from .config import beta_features, find_core_lib, k_allocator, licensed, pandas_gt1, system
+from .config import beta_features, find_core_lib, keep_local_times, k_allocator, licensed, pandas_gt1, system
from .constants import NULL_INT16, NULL_INT32, NULL_INT64
from .constants import INF_INT16, INF_INT32, INF_INT64, INF_NEG_INT16, INF_NEG_INT32, INF_NEG_INT64
from .exceptions import LicenseException, PyArrowUnavailable, PyKXException, QError
@@ -120,6 +120,12 @@ from .util import df_from_arrays, slice_to_range
import importlib.util
_torch_unavailable = importlib.util.find_spec('torch') is None
+_np_get_handler_name = (np.core.multiarray if np.__version__[0] == '1'
+ else np._core.multiarray).get_handler_name
+
+
+cdef bint _pykx_allocated(x):
+ return _np_get_handler_name(x) == 'pykx_allocator'
__all__ = [
@@ -971,9 +977,7 @@ def from_list(x: list,
if ktype is not None and not ktype is k.List:
try:
np_type = pykx_ktype_to_np_type.get(ktype) if ktype != k.GUIDVector else object
-
if ktype is k.TimestampVector and config.keep_local_times:
-
x = [y.replace(tzinfo=None) for y in x]
return from_numpy_ndarray(np.array(x, dtype=np_type), ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator)
except TypeError as ex:
@@ -1379,11 +1383,19 @@ def from_numpy_ndarray(x: np.ndarray,
Returns:
An instance of a subclass of `pykx.Vector`.
"""
- owndata = False
- writeable = False
+ # Structured arrays (those with named fields) map onto q tables rather than vectors.
+ if x.dtype.names is not None:
+ return from_numpy_recarray(x, ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator)
+ skip_allocator = False
if hasattr(x, 'flags'):
- owndata = x.flags.owndata
- writeable = x.flags.writeable
+ if (
+ not k_allocator
+ or no_allocator
+ or not x.flags.owndata
+ or not x.flags.writeable
+ or not _pykx_allocated(x)
+ ):
+ skip_allocator = True
if str(x.dtype) == "pykx.uuid":
x = x.array
@@ -1483,7 +1495,7 @@ def from_numpy_ndarray(x: np.ndarray,
core.r0(kx)
raise TypeError('Item size mismatch when converting Numpy ndarray to q: q item size '
f'({itemsize}) != Numpy item size ({x.itemsize})')
- if not k_allocator or no_allocator or not owndata or not writeable:
+ if skip_allocator:
kx = core.ktn(ktype.t, n)
data = x.__array_interface__['data'][0]
memcpy( kx.G0, data, n * itemsize)
@@ -1536,12 +1548,12 @@ def from_numpy_ndarray(x: np.ndarray,
core.r0(kx)
raise TypeError('Item size mismatch when converting Numpy ndarray to q: q item size '
f'({itemsize}) != Numpy item size ({x.itemsize})')
- if not k_allocator or no_allocator or not owndata or not writeable:
+ if skip_allocator:
kx = core.ktn(ktype.t, n)
data = x.__array_interface__['data'][0]
memcpy( kx.G0, data, n * itemsize)
return factory(kx, False)
- if not k_allocator or no_allocator or not owndata or not writeable:
+ if skip_allocator:
return factory(kx, False) # nocov
Py_INCREF(x)
data = x.__array_interface__['data'][0]
@@ -1553,6 +1565,100 @@ def from_numpy_ndarray(x: np.ndarray,
return res
+def from_numpy_recarray(x: np.recarray,
+ ktype: Optional[KType] = None,
+ *,
+ cast: bool = False,
+ handle_nulls: bool = False,
+ strings_as_char: bool = False,
+ no_allocator: bool = False,
+) -> Union[k.Table, k.KeyedTable]:
+ """Converts a structured `numpy.ndarray` (including a `numpy.recarray`) into a `pykx.Table`.
+
+ This is the inverse of [`pykx.Table.np`][pykx.wrappers.Table.np], which produces a
+ `numpy.recarray` via `pandas.DataFrame.to_records`. Each named field of the array's dtype
+ becomes a column of the resulting q table, with each field converted independently as a
+ `numpy.ndarray` would be.
+
+ See Also:
+ - [`from_numpy_ndarray`][pykx.toq.from_numpy_ndarray]
+ - [`from_pandas_dataframe`][pykx.toq.from_pandas_dataframe]
+
+ Parameters:
+ x: The structured `numpy.ndarray`/`numpy.recarray` that will be converted into a q table.
+ Its dtype must have named fields (i.e. `x.dtype.names is not None`).
+ ktype: Desired `pykx.K` subclass (or type number) for the returned value. If `None`, a
+ `pykx.Table` is returned. The following values are supported:
+
+ +-------------------+-----------------------------------------------------------------+
+ | `ktype` | Returned value |
+ +===================+=================================================================+
+ | `None` | Same as for `ktype=pykx.Table`. |
+ +-------------------+-----------------------------------------------------------------+
+ | `pykx.Table` | The record array as a q table. |
+ +-------------------+-----------------------------------------------------------------+
+ | `pykx.KeyedTable` | The record array as a q keyed table. |
+ +-------------------+-----------------------------------------------------------------+
+ cast: Unused.
+ handle_nulls: Convert `pd.NaT` to corresponding q null values in Pandas dataframes and
+ Numpy arrays.
+
+ Raises:
+ TypeError: The array is not a structured array, has more than one dimension, or the
+ `ktype` is unsupported.
+
+ Returns:
+ An instance of `pykx.Table` or `pykx.KeyedTable`.
+ """
+ cdef core.K kx
+ if getattr(x, 'dtype', None) is None or x.dtype.names is None:
+ raise _conversion_TypeError(x, repr('numpy.recarray'), ktype)
+ if x.ndim != 1:
+ raise TypeError(
+ 'Only one dimensional record arrays can be converted to a q table, received an '
+ f'array with {x.ndim} dimensions'
+ )
+ # Keyed tables (and any other non-default target) are handled by the more general Pandas
+ # path, which understands how to build the key columns from an index.
+ if not (ktype is None or ktype is k.Table):
+ return from_pandas_dataframe(
+ pd.DataFrame.from_records(x),
+ ktype,
+ cast=cast,
+ handle_nulls=handle_nulls,
+ strings_as_char=strings_as_char,
+ no_allocator=no_allocator,
+ )
+ # Convert each field independently as a numpy array, preserving its dtype (e.g. an 'S1'
+ # field becomes a char column and a 'U1' field a symbol column) and assemble the columns
+ # into a table.
+ columns = {}
+ for name in x.dtype.names:
+ col = from_numpy_ndarray(
+ x[name],
+ cast=cast,
+ handle_nulls=handle_nulls,
+ strings_as_char=strings_as_char,
+ no_allocator=no_allocator,
+ )
+ # A table column must be a vector. A single element field (e.g. a one row 'S1'/bytes
+ # column) converts to an atom, so enlist it into a length one vector.
+ if isinstance(col, k.Atom):
+ col = from_list([col])
+ columns[name] = col
+ kk = from_dict(
+ columns,
+ cast=cast,
+ handle_nulls=handle_nulls,
+ strings_as_char=strings_as_char,
+ no_allocator=no_allocator,
+ )
+ kx = core.xT(core.r1(_k(kk)))
+ if kx == NULL:
+ raise PyKXException('Failed to create table from numpy record array')
+ return factory(kx, False)
+
+
_size_to_nan = {
2: NULL_INT16,
4: NULL_INT32,
@@ -1577,6 +1683,8 @@ def _to_numpy_or_categorical(x, col_name=None, df=None):
if isinstance(x, np.ndarray):
return x
elif isinstance(x, (pd.Series, pd.Index)):
+ if keep_local_times and hasattr(x, 'dt') and hasattr(x.dt, 'tz_localize'):
+ x = x.dt.tz_localize(None)
if isinstance(x.values, pd.Categorical):
return from_pandas_categorical(
x.values,
@@ -1584,8 +1692,12 @@ def _to_numpy_or_categorical(x, col_name=None, df=None):
)
elif isinstance(x.values, pd.core.arrays.ExtensionArray):
attrnull = hasattr(x, 'isnull')
- hasnull = x.isnull().values.any() if attrnull else False
- if isinstance(x.dtype, pd.StringDtype) and attrnull and hasnull:
+ isnull = x.isnull().values if attrnull else None
+ hasnull = isnull.any() if attrnull else False
+ string_dtype = pd.api.types.is_string_dtype(x.dtype)
+ if string_dtype and (len(x) == 0 or (attrnull and isnull.all())):
+ return np.full(len(x), '', dtype='U1')
+ elif string_dtype and attrnull and hasnull:
return np.array(x.to_numpy(copy=False, na_value=None))
elif x.dtype.kind != 'f' and attrnull and hasnull:
if not x.dtype.kind in ['i', 'M', 'm']:
@@ -2279,6 +2391,8 @@ def from_numpy_datetime64(x: np.datetime64,
"""
cdef core.K kx
if isinstance(x, pd._libs.tslibs.timestamps.Timestamp):
+ if keep_local_times and hasattr(x, 'tz_localize'):
+ x = x.tz_localize(None)
x = x.to_datetime64()
if ktype is None or ktype is k.TimestampAtom:
kx = core.ktj(-12, x.astype(np.dtype('datetime64[ns]')).astype(np.int64) - TIMESTAMP_OFFSET) # noqa
@@ -2597,7 +2711,6 @@ def from_ellipsis(x: Ellipsis,
return q('value[(;)]1')
-# TODO: After Python 3.7 support is dropped, use a `typing.Protocol` for `x` (KXI-9158)
def from_fileno(x: Any,
ktype: Optional[KType] = None,
*,
@@ -2925,6 +3038,7 @@ _converter_from_python_type = {
np.ndarray: from_numpy_ndarray,
np.ma.MaskedArray: from_numpy_ndarray,
+ np.recarray: from_numpy_recarray,
pd.DataFrame: from_pandas_dataframe,
pd.Series: from_pandas_series,
diff --git a/src/pykx/util.py b/src/pykx/util.py
index b9f6a7d..882f7fa 100644
--- a/src/pykx/util.py
+++ b/src/pykx/util.py
@@ -1,5 +1,5 @@
from contextlib import contextmanager
-from functools import wraps
+from functools import cached_property, wraps
import inspect
import io
import os
@@ -15,18 +15,23 @@
from warnings import warn
import pandas as pd
-from pandas.core.internals import BlockManager, make_block
+from pandas.core.internals import BlockManager
import numpy as np
import requests
import toml
from .config import (
- _executable, _get_qexecutable, _get_qhome,
+ _executable, _get_qdebug, _get_qexecutable, _get_qhome, _set_qdebug,
beta_features, ignore_qhome, jupyterq, k_allocator, k_gc, keep_local_times,
- load_pyarrow_unsafe, max_error_length, no_pykx_signal, numpy_gt1, pandas_gt2, pykx_config_location, # noqa E501
- pykx_config_profile, pykx_debug_insights, pykx_dir, pykx_lib_dir, pykx_profile_content,
- pykx_qdebug, pykx_threading, q_executable, qargs, qce, qhome, qlic, release_gil,
- skip_under_q, suppress_warnings, use_q_lock)
+ load_pyarrow_unsafe, max_error_length, no_pykx_signal, numpy_gt1, pandas_gt2, # noqa E501
+ pykx_config_location, pykx_config_profile, pykx_debug_insights, pykx_dir, pykx_lib_dir,
+ pykx_profile_content, pykx_threading, q_executable, qargs, qce, qhome, qlic,
+ release_gil, skip_under_q, suppress_warnings, use_q_lock)
+
+if pandas_gt2:
+ from pandas.api.internals import create_dataframe_from_blocks
+else:
+ from pandas.core.internals import make_block
from ._version import version as __version__
from .exceptions import PyKXException
@@ -95,22 +100,6 @@ def num_available_cores() -> int:
return os.cpu_count() # nocov
-# TODO: Once Python 3.7 support is dropped, we can replace this with `functools.cached_property`
-class cached_property:
- """Property-like descriptor which overwrites itself with the first obtained property value."""
- def __init__(self, func):
- self.func = func
- self.attrname = func.__name__
- self.__doc__ = func.__doc__
-
- def __get__(self, instance, owner=None):
- if instance is None:
- return self
- value = self.func(instance)
- instance.__dict__[self.attrname] = value
- return value
-
-
class class_or_instancemethod(classmethod):
def __get__(self, instance, type_):
descr_get = super().__get__ if instance is None else self.__func__.__get__
@@ -274,12 +263,18 @@ def df_from_arrays(columns, arrays, index):
df = pd.DataFrame(index=index)
for col, arr in zip(columns, arrays):
df[col] = arr
+ elif pandas_gt2:
+ blocks = [
+ (a.reshape((1, len(a))), np.array([i]))
+ for i, a in enumerate(arrays)
+ ]
+ df = create_dataframe_from_blocks(blocks, index=index, columns=columns)
else:
blocks = tuple(
make_block(values=a.reshape((1, len(a))), placement=(i,))
for i, a in enumerate(arrays)
)
- df= pd.DataFrame(
+ df = pd.DataFrame(
BlockManagerUnconsolidated(axes=[columns, index], blocks=blocks),
copy=False
)
@@ -480,7 +475,7 @@ def env_information():
'PYKX_RELEASE_GIL': release_gil, 'PYKX_Q_LIB_LOCATION': pykx_lib_dir,
'PYKX_Q_LOCK': use_q_lock, 'PYKX_SKIP_UNDERQ': skip_under_q,
'PYKX_Q_EXECUTABLE': q_executable, 'PYKX_THREADING': pykx_threading,
- 'PYKX_QDEBUG': pykx_qdebug,
+ 'PYKX_QDEBUG': _get_qdebug(),
'PYKX_DEBUG_INSIGHTS_LIBRARIES': pykx_debug_insights,
'PYKX_CONFIGURATION_LOCATION': pykx_config_location,
'PYKX_NO_SIGNAL': no_pykx_signal,
@@ -726,20 +721,21 @@ def start_q_subprocess(port: int,
'Please install q using the function "kx.util.install_q" or following the '
'instructions at:\nhttps://code.kx.com/pykx/getting-started/installing.html'
)
- with PyKXReimport():
- qinit = [q_executable, load_file, '-p', f'{port}']
- if init_args is not None:
- if not isinstance(init_args, list):
- raise TypeError('Supplied additional startup arguments must be a list')
- if not all(isinstance(s, str) for s in init_args):
- raise TypeError('All supplied arguments to init_args must be str type objects')
- qinit.extend(init_args)
- server = subprocess.Popen(
- qinit,
- stdin=subprocess.PIPE,
- stdout=None if process_logs else subprocess.DEVNULL,
- stderr=None)
- time.sleep(2)
+ qinit = [q_executable, load_file, '-p', f'{port}']
+ if init_args is not None:
+ if not isinstance(init_args, list):
+ raise TypeError('Supplied additional startup arguments must be a list')
+ if not all(isinstance(s, str) for s in init_args):
+ raise TypeError('All supplied arguments to init_args must be str type objects')
+ qinit.extend(init_args)
+ server = subprocess.Popen(
+ qinit,
+ stdin=subprocess.PIPE,
+ stdout=None if process_logs else subprocess.DEVNULL,
+ stderr=None,
+ env={**os.environ, 'QHOME': os.getenv('PYKX_OLD_QHOME', '')},
+ )
+ time.sleep(2)
return server
@@ -826,3 +822,16 @@ def delete_q_variable(variable: str, namespace: str = '', garbage_collect: bool
q('{![x;();0b;enlist y]}', ns, variable)
if garbage_collect:
q.Q.gc()
+
+
+def qdebug(enable: bool = True):
+ """
+ Enables or disables q debugging at runtime.
+
+ Parameters:
+ enable: If True enables q debug mode.
+
+ Returns:
+ None
+ """
+ _set_qdebug(enable)
diff --git a/src/pykx/wrappers.py b/src/pykx/wrappers.py
index fd1cb4e..d467e2b 100644
--- a/src/pykx/wrappers.py
+++ b/src/pykx/wrappers.py
@@ -250,6 +250,22 @@ def _rich_convert(x: 'K', stdlib: bool = True, raw=False):
return x.np(raw=raw)
+def _as_array(arr, dtype, copy):
+ """Apply the NumPy `__array__(dtype, copy)` protocol to an already-materialised array.
+
+ NumPy 2.* passes `dtype` positionally and `copy` by keyword to `__array__`, so every
+ implementation must accept both. A dtype conversion is unavoidably a copy, so an explicit
+ `copy=False` alongside a differing dtype is refused (matching `numpy.ndarray`).
+ """
+ if dtype is not None and dtype != arr.dtype:
+ if copy is False:
+ raise ValueError('Unable to avoid copy while creating an array as requested.')
+ return arr.astype(dtype)
+ if copy:
+ return arr.copy()
+ return arr
+
+
# HACK: This gets overwritten by the toq module to avoid a circular import error.
def toq(*args, **kwargs): # nocov
raise NotImplementedError
@@ -598,8 +614,8 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
res = res._unlicensed_getitem(0)
return res
- def __array__(self, dtype=None):
- return np.asarray(self.np(), dtype=dtype)
+ def __array__(self, dtype=None, copy=None):
+ return _as_array(np.asarray(self.np()), dtype, copy)
class EnumAtom(Atom):
@@ -657,9 +673,11 @@ def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: boo
as_np_timedelta = self.np()
if pd.isnull(as_np_timedelta):
return pd.NaT
- return timedelta(
- microseconds=int(as_np_timedelta.astype('timedelta64[ns]').astype(np.int64))//1000
- )
+ if self._np_type == 'ns':
+ microseconds = int(as_np_timedelta.astype(np.int64)) // 1000
+ else:
+ microseconds = int(as_np_timedelta.astype('timedelta64[us]').astype(np.int64))
+ return timedelta(microseconds=microseconds)
def np(self,
*,
@@ -1356,11 +1374,11 @@ def is_inf(self) -> bool:
@property
def is_pos_inf(self) -> bool:
- return np.isposinf(self.py())
+ return bool(np.isposinf(self.py()))
@property
def is_neg_inf(self) -> bool:
- return np.isneginf(self.py())
+ return bool(np.isneginf(self.py()))
def __round__(self, ndigits=None):
return round(self.py(), ndigits)
@@ -1689,7 +1707,7 @@ def inf_neg(cls): # noqa: B902
@property
def is_null(self) -> bool:
- return self.py(raw=True) == 0j
+ return bool(self.py(raw=True) == 0j)
@property
def is_inf(self) -> bool:
@@ -1873,13 +1891,13 @@ def __setitem__(self, key, val):
raise QError(str(err))
self.__dict__.update(update_assign.__dict__)
- def __array__(self):
+ def __array__(self, dtype=None, copy=None):
# The `__array__` method must return a `np.ndarray`, not a `np.ma.masked_array`. As a
# result, the null check we currently perform (by default) is a waste of time and memory,
# since what will be returned in the end will ultimately expose the underlying values of
# the nulls anyway. We can’t stop `__array__` from returning the raw null values, but we
# can save that time and memory by using `has_nulls=False`.
- return self.np(has_nulls=False)
+ return _as_array(self.np(has_nulls=False), dtype, copy)
def __arrow_array__(self, type=None):
if pa is None:
@@ -2420,12 +2438,12 @@ class IntegralNumericVector(NumericVector):
"""Base type for all q integral numeric vectors."""
@property
def has_nulls(self) -> bool:
- return (self._base_null_value == self.np(raw=True)).any()
+ return bool((self._base_null_value == self.np(raw=True)).any())
@property
def has_infs(self) -> bool:
a = self.np(raw=True)
- return (a == self._base_inf_value).any() or (a == -self._base_inf_value).any()
+ return bool((a == self._base_inf_value).any() or (a == -self._base_inf_value).any())
def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: bool = True):
if raw:
@@ -2560,8 +2578,8 @@ def __len__(self):
def __eq__(self, other):
return self.array == other
- def __array__(self):
- return self.array
+ def __array__(self, dtype=None, copy=None):
+ return _as_array(self.array, dtype, copy)
def reshape(self, *args, **kwargs):
return self
@@ -2619,7 +2637,7 @@ def _unlicensed_getitem(self, index):
@property
def has_nulls(self) -> bool:
- return (0j == self.np(raw=True)).any()
+ return bool((0j == self.np(raw=True)).any())
@property
def has_infs(self) -> bool:
@@ -2718,11 +2736,12 @@ class NonIntegralNumericVector(NumericVector):
@property
def has_nulls(self) -> bool:
a = self.np()
- return np.isnan(np.dot(a, a)) # `np.dot` can be used as a high-performance NaN check
+ # `np.dot` can be used as a high-performance NaN check
+ return bool(np.isnan(np.dot(a, a)))
@property
def has_infs(self) -> bool:
- return (self.np() == np.inf).any()
+ return bool(np.isinf(self.np()).any())
def __pow__(self, other, mod=None):
if mod is None:
@@ -2825,12 +2844,12 @@ class TemporalVector(Vector):
"""Base type for all q temporal vectors."""
@property
def has_nulls(self) -> bool:
- return (self._base_null_value == self.np(raw=True)).any()
+ return bool((self._base_null_value == self.np(raw=True)).any())
@property
def has_infs(self) -> bool:
a = self.np(raw=True)
- return (a == self._base_inf_value).any() or (a == -self._base_inf_value).any()
+ return bool((a == self._base_inf_value).any() or (a == -self._base_inf_value).any())
def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: bool = True):
if raw:
@@ -2936,19 +2955,17 @@ def py(self,
if raw:
return self.np(raw=True, has_nulls=has_nulls).tolist()
if tzinfo is not None:
+ converted = self.np().astype('datetime64[us]').astype(datetime).tolist()
if tzshift:
- return [x.replace(tzinfo=pytz.utc).astimezone(tzinfo)
- for x in self.np().astype('datetime64[us]').astype(datetime).tolist()]
+ return [pd.NaT if x is None else x.replace(tzinfo=pytz.utc).astimezone(tzinfo)
+ for x in converted]
else:
- return [x.replace(tzinfo=tzinfo)
- for x in self.np().astype('datetime64[us]').astype(datetime).tolist()]
- converted_vector=self.np().astype('datetime64[us]').astype(datetime).tolist()
- null_pos=[]
- for x in converted_vector:
+ return [pd.NaT if x is None else x.replace(tzinfo=tzinfo)
+ for x in converted]
+ converted_vector = self.np().astype('datetime64[us]').astype(datetime).tolist()
+ for i, x in enumerate(converted_vector):
if x is None:
- null_pos.append(converted_vector.index(x))
- for i in null_pos:
- converted_vector[i]=pd.NaT
+ converted_vector[i] = pd.NaT
return converted_vector
@@ -3748,7 +3765,7 @@ def delete(self, columns=None, where=None, inplace=False):
```python
>>> qtab.delete(kx.Column('age'))
- >>> qtab.delete([kx.Column('age'), kx.Column('eye')])
+ >>> qtab.delete(kx.Column('age') & kx.Column('eye'))
```
Delete rows of the dataset based on where condition
@@ -3991,11 +4008,11 @@ def window_join(self, table, windows, cols, aggs):
... {'ask_max': [lambda x: max(x), 'ask'],
... 'ask_minus_bid': [lambda x, y: x - y, 'ask', 'bid']})
pykx.Table(pykx.q('
- sym time price ask_minus_bid ask_max
+ sym time price ask_max ask_minus_bid
----------------------------------------
- ibm 10:01:01 100 3 4 103
- ibm 10:01:04 101 4 1 1 1 104
- ibm 10:01:08 105 3 2 1 1 108
+ ibm 10:01:01 100 103 3 4
+ ibm 10:01:04 101 104 4 1 1 1
+ ibm 10:01:08 105 108 3 2 1 1
'))
```
"""
@@ -4302,14 +4319,28 @@ def xbar(self, values):
raise AttributeError("Operation 'xbar' not supported for PartitionedTable type")
-class VirtualTable(Atom):
+class VirtualTable(K):
"""Wrapper for q virtual tables."""
t = 112
+ is_atom = False
select = Table.select
exec = Table.exec
+_VT_base_error = "pykx.VirtualTable objects cannot be operated on directly, you must .select() or .exec() data from them first" # noqa: E501
+_VT_include = {'__module__', '__firstlineno__', '__doc__', '__init__', '__new__', '__del__', '_from_addr', '__repr__', '__str__'} # noqa: E501
+
+
+def _VT_base_error_method(self, *args, **kwargs):
+ raise QError(_VT_base_error)
+
+
+for _name in vars(K):
+ if callable(getattr(K, _name)) and _name not in vars(VirtualTable)and _name not in _VT_include:
+ setattr(VirtualTable, _name, _VT_base_error_method)
+
+
class Dictionary(Mapping):
"""Wrapper for q dictionaries, including regular dictionaries, and keyed tables."""
t = 99
@@ -5200,19 +5231,25 @@ def scan(self):
def each_prior(self):
return q("{x':}", self)
- prior = each_prior
+ @property
+ def prior(self):
+ return self.each_prior
@cached_property
def each_right(self):
return q('{x/:}', self)
- sv = each_right
+ @property
+ def sv(self):
+ return self.each_right
@cached_property
def each_left(self):
return q('{x\\:}', self)
- vs = each_left
+ @property
+ def vs(self):
+ return self.each_left
class Lambda(Function):
@@ -6673,7 +6710,7 @@ def count(self, iterator=None):
... 'a': [1, -1, 0],
... 'b': [[-1, 2, 1], [0, 2], 1]
... })
- >>> tab.exec(kx.Column('b').count(iterator='each')))
+ >>> tab.exec(kx.Column('b').count(iterator='each'))
pykx.LongVector(pykx.q('3 2 1'))
```
"""
@@ -8122,7 +8159,7 @@ def min(self, iterator=None):
... 'a': kx.random.random(100, 5),
... 'b': kx.random.random([100, 3], 5)
... })
- >>> tab.exec(kx.Column('a').max())
+ >>> tab.exec(kx.Column('a').min())
pykx.LongAtom(pykx.q('0'))
```
@@ -8607,7 +8644,6 @@ def ratios(self, iterator=None):
```python
>>> import pykx as kx
>>> tab = kx.Table(data={
- >>> tab = kx.Table(data={
... 'a': kx.random.random(100, 1000),
... 'b': kx.random.random([100, 3], 5)
... })
@@ -8632,7 +8668,6 @@ def reciprocal(self, iterator=None):
```python
>>> import pykx as kx
>>> tab = kx.Table(data={
- >>> tab = kx.Table(data={
... 'a': kx.random.random(100, 1000),
... 'b': kx.random.random([100, 3], 5)
... })
@@ -8657,7 +8692,6 @@ def reverse(self, iterator=None):
```python
>>> import pykx as kx
>>> tab = kx.Table(data={
- >>> tab = kx.Table(data={
... 'a': kx.q.til(100),
... 'b': kx.random.random([100, 3], 5)
... })
@@ -8759,13 +8793,22 @@ def rtrim(self, iterator=None):
"""
return self.call('rtrim', iterator=iterator)
- def scov(self, iterator=None):
+ def scov(self, other, iterator=None, col_arg_ind=0, project_args=None):
"""
- Calculate the sample covariance for items in a column or rows in a column
+ Calculate the sample covariance between a column and one of:
+
+ - Another column
+ - A Python list/numpy array
+ - A `pykx` variable in q memory
Parameters:
+ other: The second column or variable (Python/q) to be used.
iterator: What iterator to use when operating on the column
for example, to execute per row, use `each`
+ col_arg_ind: Determines the index within the multivariate function
+ where the column parameter will be used. Default 0.
+ project_args: The argument indices of a multivariate function which will be
+ projected on the function before evocation with use of an iterator.
Examples:
@@ -8775,10 +8818,10 @@ def scov(self, iterator=None):
>>> import pykx as kx
>>> tab = kx.Table(data={
... 'a': kx.random.random(100, 10.0),
- ... 'b': kx.random.random([100, 3], 10.0)
+ ... 'b': kx.random.random(100, 10.0)
... })
- >>> tab.exec(kx.Column('a').scov())
- pykx.FloatAtom(pykx.q('8.983196'))
+ >>> tab.exec(kx.Column('a').scov(kx.Column('b')))
+ pykx.FloatAtom(pykx.q('1.442801'))
```
Calculate the sample covariance for each row in a column:
@@ -8786,22 +8829,23 @@ def scov(self, iterator=None):
```python
>>> import pykx as kx
>>> tab = kx.Table(data={
- ... 'a': kx.random.random(100, 10.0),
- ... 'b': kx.random.random([100, 3], 10.0)
+ ... 'b': kx.random.random([100, 3], 10.0),
+ ... 'c': kx.random.random([100, 3], 10.0)
... })
- >>> tab.select(kx.Column('b').scov(iterator='each'))
+ >>> tab.select(kx.Column('b').scov(kx.Column('c'), iterator="'"))
pykx.Table(pykx.q('
b
- ---------
- 0.3333333
- 0.3333333
- 5.333333
- 1.333333
+ -----------
+ -0.08110428
+ -1.384917
+ 0.7203654
+ 0.3846634
..
'))
```
"""
- return self.call('scov', iterator=iterator)
+ return self.call('scov', other, iterator=iterator, col_arg_ind=col_arg_ind,
+ project_args=project_args)
def sdev(self, iterator=None):
"""
@@ -9458,7 +9502,15 @@ def within(self, lower, upper, iterator=None, col_arg_ind=0, project_args=None):
... 'b': kx.q.desc(kx.random.random(100, 10))
... })
>>> tab.select(where=kx.Column('a').within(today - 5, today - 3))
- pykx.FloatAtom(pykx.q('2.431111'))
+ pykx.Table(pykx.q('
+ a b
+ ------------
+ 2026.07.11 9
+ 2026.07.12 9
+ 2026.07.11 8
+ 2026.07.12 8
+ ..
+ '))
```
"""
return self.call('within', [lower, upper], iterator=iterator, col_arg_ind=col_arg_ind,
@@ -10111,21 +10163,21 @@ def cast(self, other, iterator=None, col_arg_ind=1, project_args=None):
... })
>>> tab.dtypes
pykx.Table(pykx.q('
- columns datatypes type
- -----------------------------------
- a "kx.LongAtom" "kx.LongAtom"
- b "kx.LongAtom" "kx.LongAtom"
+ columns datatypes
+ ---------------------
+ a "kx.LongAtom"
+ b "kx.LongAtom"
'))
>>> tab.select(
... kx.Column('a') &
... kx.Column('a').cast('float').name('a_float') &
... kx.Column('b')).dtypes
pykx.Table(pykx.q('
- columns datatypes type
- -------------------------------------
- a "kx.LongAtom" "kx.LongAtom"
- a_float "kx.FloatAtom" "kx.FloatAtom"
- b "kx.LongAtom" "kx.LongAtom"
+ columns datatypes
+ ----------------------
+ a "kx.LongAtom"
+ a_float "kx.FloatAtom"
+ b "kx.LongAtom"
'))
```
"""
@@ -10565,8 +10617,8 @@ def len(self, iterator=None):
... 'a': [1, -1, 0],
... 'b': [[-1, 2, 1], [0, 2], 1]
... })
- >>> tab.exec(kx.Column('b').len(iterator='each')))
- pykx.LongVector(pykx.q('3 3 3'))
+ >>> tab.exec(kx.Column('b').len(iterator='each'))
+ pykx.LongVector(pykx.q('3 2 1'))
```
"""
return self.call('count', iterator=iterator)
@@ -11369,10 +11421,8 @@ def key(self, iterator=None):
>>> d1 = kx.q('`a`b`c ! 1 2 3')
>>> d2 = kx.q('`d`e`f ! 4 5 6')
>>> t = kx.Table(data={'n':[d1, d2]})
- >>> t.exec(kx.Column('n').type(iterator='each'))
- pykx.Table(pykx.q('
- n
- -----
+ >>> t.exec(kx.Column('n').key(iterator='each'))
+ pykx.List(pykx.q('
a b c
d e f
'))
@@ -11487,7 +11537,7 @@ def next(self, iterator=None):
```python
>>> import pykx as kx
>>> t = kx.q('([] n:3 6 9 12)')
- >>> t.exec(kx.Column('n').next()))
+ >>> t.exec(kx.Column('n').next())
pykx.LongVector(pykx.q('6 9 12 0N'))
```
"""
diff --git a/tests/data/types.parquet b/tests/data/types.parquet
new file mode 100644
index 0000000..2df098f
Binary files /dev/null and b/tests/data/types.parquet differ
diff --git a/tests/test_cloud_edition.py b/tests/test_cloud_edition.py
index a98b14e..1d78bd9 100644
--- a/tests/test_cloud_edition.py
+++ b/tests/test_cloud_edition.py
@@ -6,7 +6,8 @@
import pytest
-if platform.system() != 'Linux' or platform.machine() == 'aarch64':
+if platform.system() != 'Linux' or platform.machine() == 'aarch64'\
+ or os.getenv('SKIP_LIC_TESTS') is not None:
pytest.skip(
'KDB-X Cloud Edition functionality only supports'
' x86_64 Linux',
diff --git a/tests/test_ipc.py b/tests/test_ipc.py
index 5f8157d..214e7a2 100644
--- a/tests/test_ipc.py
+++ b/tests/test_ipc.py
@@ -87,6 +87,14 @@ def test_ipc_messaging_tcp(kx, q_port):
_generic_ipc_test(q)
+@pytest.mark.unlicensed
+@pytest.mark.parametrize('cls_name', ['QConnection', 'SyncQConnection'])
+def test_single_handle_no_double_init(kx, q_port, cls_name):
+ cls = getattr(kx, cls_name)
+ with cls('localhost', q_port) as q:
+ assert int(q('count .z.W')) == 1
+
+
@pytest.mark.unlicensed
@pytest.mark.parametrize('q_init', [[b'.z.pw:{(x~`username)&y~"password"}\n']])
def test_ipc_messaging_tcp_auth(kx, q_port):
@@ -330,7 +338,7 @@ def test_py_file_execution(kx):
proc = subprocess.Popen(
[q_exe_path, '-p', '15005'],
stdout=subprocess.DEVNULL,
- stderr=subprocess.STDOUT
+ stderr=subprocess.STDOUT,
)
time.sleep(2)
with kx.SyncQConnection(port=15005) as q:
@@ -719,6 +727,39 @@ def test_sync_helpful_error_for_closed_conn(kx, q_port):
q('hclose .z.w; til 10')
+@pytest.mark.unlicensed
+def test_sync_close_finalizes_when_shutdown_fails(kx, q_port):
+ conn = kx.SyncQConnection(port=q_port)
+ real_sock = conn._sock
+ state = {'sock_closed': False, 'finalized': False}
+
+ class _ShutdownFails:
+ def shutdown(self, *args, **kwargs):
+ raise OSError('simulated dead peer')
+
+ def close(self, *args, **kwargs):
+ state['sock_closed'] = True
+ return real_sock.close(*args, **kwargs)
+
+ def __getattr__(self, name):
+ return getattr(real_sock, name)
+
+ real_finalizer = conn._finalizer
+
+ def _tracked_finalizer():
+ state['finalized'] = True
+ return real_finalizer()
+
+ object.__setattr__(conn, '_sock', _ShutdownFails())
+ object.__setattr__(conn, '_finalizer', _tracked_finalizer)
+
+ conn.close()
+
+ assert conn.closed
+ assert state['sock_closed'], 'socket was not closed after shutdown failure'
+ assert state['finalized'], 'finalizer did not run after shutdown failure'
+
+
def check_enough_memory(GiB):
import psutil
minimum_memory = GiB
diff --git a/tests/test_license.py b/tests/test_license.py
index 86a9060..dcfdcee 100644
--- a/tests/test_license.py
+++ b/tests/test_license.py
@@ -2,6 +2,7 @@
from io import StringIO
import os
import re
+import sys
# Do not import pykx here - use the `kx` fixture instead!
import pytest
@@ -498,15 +499,20 @@ def test_expired_license(monkeypatch):
os.environ['QLIC'] = os.environ['QHOME'] = lic_folder
inputs = iter(['n', 'n'])
monkeypatch.setattr('builtins.input', lambda _: next(inputs))
+ monkeypatch.setattr(sys, 'ps1', '>>> ', raising=False)
import pykx as kx
assert not kx.licensed
+ with pytest.raises(Exception) as e:
+ kx.q('1+1')
+ assert "license is expired" in str(e)
@pytest.mark.skipif(
os.getenv('SKIP_LIC_TESTS') is not None,
reason='License tests are being skipped'
)
+@pytest.mark.xfail(reason='Flaky test', strict=False)
def test_mode_licensed_true():
os.environ['PYKX_LICENSED'] = 'True'
lic_folder = '/tmp/license'
@@ -520,6 +526,7 @@ def test_mode_licensed_true():
kx.licensed
+@pytest.mark.xfail(reason='Flaky license initialization ordering', strict=False)
@pytest.mark.skipif(
os.getenv('SKIP_LIC_TESTS') is not None,
reason='License tests are being skipped'
@@ -539,6 +546,7 @@ def test_mode_monkeypatch_licensed_true(monkeypatch):
kx.licensed
+@pytest.mark.xfail(reason='Flaky license initialization ordering', strict=False)
@pytest.mark.skipif(
os.getenv('PYKX_THREADING') is not None,
reason='KXI-63218 PYKX_THREADING licence path differs'
@@ -556,10 +564,16 @@ def test_mode_monkeypatch_licensed_false(monkeypatch):
os.environ['QLIC'] = os.environ['QHOME'] = lic_folder
inputs = iter(['n', 'n'])
+
monkeypatch.setattr('builtins.input', lambda _: next(inputs))
+ monkeypatch.setattr(sys, 'ps1', '>>> ', raising=False)
import pykx as kx
assert not kx.licensed
+ with pytest.raises(Exception) as err:
+ kx.q('1+1')
+ assert "corrupt" in str(err.value)
+ print(str(err.value))
@pytest.mark.skipif(
@@ -570,6 +584,7 @@ def test_mode_monkeypatch_licensed_false(monkeypatch):
os.getenv('SKIP_LIC_TESTS') is not None,
reason='License tests are being skipped'
)
+@pytest.mark.xfail(reason='Flaky test', strict=False)
def test_bad_license():
lic_folder = '/tmp/license'
os.makedirs(lic_folder, exist_ok=True)
@@ -579,3 +594,76 @@ def test_bad_license():
import pykx as kx
assert not kx.licensed
+
+
+@pytest.mark.skipif(
+ os.getenv('PYKX_THREADING') is not None,
+ reason='KXI-63218 PYKX_THREADING licence path differs'
+)
+@pytest.mark.skipif(
+ os.getenv('SKIP_LIC_TESTS') is not None,
+ reason='License tests are being skipped'
+)
+def test_unlicensed_reason_env_var():
+ os.environ['PYKX_UNLICENSED'] = 'True'
+ import pykx as kx
+ assert not kx.licensed
+ with pytest.raises(Exception) as err:
+ kx.q('1+1')
+ assert "PYKX_UNLICENSED" in str(err.value)
+
+
+@pytest.mark.skipif(
+ os.getenv('PYKX_THREADING') is not None,
+ reason='KXI-63218 PYKX_THREADING licence path differs'
+)
+@pytest.mark.skipif(
+ os.getenv('SKIP_LIC_TESTS') is not None,
+ reason='License tests are being skipped'
+)
+def test_unlicensed_reason_qargs():
+ os.environ['QARGS'] = '--unlicensed'
+ import pykx as kx
+ assert not kx.licensed
+ with pytest.raises(Exception) as err:
+ kx.q('1+1')
+ assert "--unlicensed" in str(err.value)
+
+
+@pytest.mark.skipif(
+ os.getenv('PYKX_THREADING') is not None,
+ reason='KXI-63218 PYKX_THREADING licence path differs'
+)
+@pytest.mark.skipif(
+ os.getenv('SKIP_LIC_TESTS') is not None,
+ reason='License tests are being skipped'
+)
+def test_unlicensed_reason_light_load():
+ os.environ['PYKX_LIGHT_LOAD'] = 'true'
+ import pykx as kx
+ assert not kx.licensed
+ with pytest.raises(Exception) as err:
+ kx.q('1+1')
+ assert "light_load" in str(err.value)
+
+
+@pytest.mark.skipif(
+ os.getenv('PYKX_THREADING') is not None,
+ reason='KXI-63218 PYKX_THREADING licence path differs'
+)
+@pytest.mark.skipif(
+ os.getenv('SKIP_LIC_TESTS') is not None,
+ reason='License tests are being skipped'
+)
+def test_unlicensed_reason_no_license_found(tmp_path, monkeypatch):
+ os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute())
+
+ inputs = iter(['N', 'N'])
+ monkeypatch.setattr('builtins.input', lambda _: next(inputs))
+ monkeypatch.setattr(sys, 'ps1', '>>> ', raising=False)
+
+ import pykx as kx
+ assert not kx.licensed
+ with pytest.raises(Exception) as e:
+ kx.q('1+1')
+ assert "No license was found" in str(e)
diff --git a/tests/test_pandas_api.py b/tests/test_pandas_api.py
index 30c9250..4a805b6 100644
--- a/tests/test_pandas_api.py
+++ b/tests/test_pandas_api.py
@@ -1464,6 +1464,11 @@ def test_df_rename(kx, q):
rez = t.rename({'Poliddcy': 'PolicyID', 'Policy': 'p_id'}, axis=1)
assert all(t.pd().rename({'Poliddcy': 'PolicyID', 'Policy': 'p_id'}, axis=1).eq(rez.pd()))
+ t = kx.q('([] Policy: 1 2 3)')
+ cols = {0: 'ignored', 'Policy': 'PolicyID'}
+ rez = t.rename(cols, axis=1)
+ assert all(t.pd().rename(cols, axis=1).eq(rez.pd()))
+
mixed_index_table = kx.q('([] id:(`a;2;1); Policy: 3 4 5; name:`a`b`c)').pd()
df=kx.toq(mixed_index_table.set_index(['id']))
rez = df.rename({1: 'x', 2: 'j'})
diff --git a/tests/test_reset_index.py b/tests/test_pandas_reset_index.py
similarity index 87%
rename from tests/test_reset_index.py
rename to tests/test_pandas_reset_index.py
index 4d166d1..65372a9 100644
--- a/tests/test_reset_index.py
+++ b/tests/test_pandas_reset_index.py
@@ -37,6 +37,14 @@ def test_reset_drop(q):
assert q('~', df.reset_index(['x', 'y'], drop=True), pddf.reset_index(['x', 'y'], drop=True))
+def test_reset_int_drop(q):
+ df = q('([x: til 10; y: 10 - til 10]z: 10?`a`b`c)')
+ pddf = df.pd()
+ assert q('~', df.reset_index(0, drop=True), pddf.reset_index(0, drop=True))
+ assert q('~', df.reset_index(1, drop=True), pddf.reset_index(1, drop=True))
+ assert q('~', df.reset_index([0, 1], drop=True), pddf.reset_index([0, 1], drop=True))
+
+
def test_reset_duplicates(kx, q):
df = q('([til 10;10?1f];10?1f;10?1f)')
assert q('~', df.reset_index(allow_duplicates=True), q('0!', df))
diff --git a/tests/test_pykx.py b/tests/test_pykx.py
index 5b1e542..0c6580e 100644
--- a/tests/test_pykx.py
+++ b/tests/test_pykx.py
@@ -439,26 +439,34 @@ def test_kx_versions(kx):
test_K = str(kx.q.z.K.py())
test_vars = (test_os, test_arch, test_K)
if test_vars == ('Linux', 'x86_64', '5.0'):
- assert kx.q.z.k == kx.q('2026.05.01')
+ assert kx.q.z.k == kx.q('2026.07.23')
elif test_vars == ('Linux', 'aarch64', '5.0'):
- assert kx.q.z.k == kx.q('2026.05.01')
+ assert kx.q.z.k == kx.q('2026.07.23')
elif test_vars == ('Darwin', 'x86_64', '5.0'):
- assert kx.q.z.k == kx.q('2026.05.01')
+ assert kx.q.z.k == kx.q('2026.07.23')
elif test_vars == ('Darwin', 'arm64', '5.0'):
- assert kx.q.z.k == kx.q('2026.05.01')
+ assert kx.q.z.k == kx.q('2026.07.23')
elif test_vars == ('Windows', 'AMD64', '5.0'):
- assert kx.q.z.k == kx.q('2026.05.01')
+ assert kx.q.z.k == kx.q('2026.07.23')
else:
raise AssertionError(f"Unexpected env: {test_vars}")
@pytest.mark.isolate
+@pytest.mark.skipif(
+ os.getenv('SKIP_LIC_TESTS') is not None,
+ reason='License tests are being skipped'
+)
def test_default_qce():
import pykx as kx
assert not kx.q('@[{2 `datetime$`date$x}', dt).py()
+
+
+@pytest.fixture
+def tmp_date_csv_path(tmp_path):
+ p = tmp_path/'date.csv'
+ p.write_text(
+ 'd,n\n'
+ '2023.01.01,0\n'
+ '2023.06.15,1\n'
+ '2023.12.31,2\n'
+ )
+ yield p
+ p.unlink()
+
+
+@pytest.mark.ipc
+def test_read_csv_date_type(kx, q, tmp_date_csv_path):
+ tab = q.read.csv(tmp_date_csv_path, [kx.DateAtom, kx.LongAtom])
+ assert isinstance(tab, kx.Table)
+
+ d = q('{x`d}', tab)
+ assert isinstance(d, kx.DateVector)
+ assert q('{type x}', d).py() == 14
+ assert q('{all x = 2023.01.01 2023.06.15 2023.12.31}', d).py()
+
+
@pytest.fixture
def tmp_fixed_path(tmp_path):
p = tmp_path/'tmp.fixed'
diff --git a/tests/test_reimport.py b/tests/test_reimport.py
index 08e1d46..a7641e1 100644
--- a/tests/test_reimport.py
+++ b/tests/test_reimport.py
@@ -1,15 +1,82 @@
# Do not import pykx here - use the `kx` fixture instead!
+import os
import subprocess
+import sys
+# --- Reimport no longer requires the PyKXReimport helper (Python child) -----
+
def test_reimport(kx):
- with kx.PyKXReimport():
- ret = subprocess.run(["python", "-c", "import pykx"])
+ ret = subprocess.run([sys.executable, "-c", "import pykx"])
assert 0 == ret.returncode
def test_reimport_kdefault(kx):
+ ret = subprocess.run(
+ [sys.executable, "-c", "import os;os.environ['PYKX_DEFAULT_CONVERSION']='k';import pykx"])
+ assert 0 == ret.returncode
+
+
+# --- PyKXReimport remains available and functional --------------------------
+# (Still required e.g. for spawning kdb+ 4.0/4.1 q subprocesses from Python -
+# see tests/manual_reimport_matrix.py for the full cross-process matrix.)
+
+def test_reimport_old(kx):
+ with kx.PyKXReimport():
+ ret = subprocess.run([sys.executable, "-c", "import pykx"])
+ assert 0 == ret.returncode
+
+
+def test_reimport_kdefault_old(kx):
with kx.PyKXReimport():
ret = subprocess.run(
- ["python", "-c", "import os;os.environ['PYKX_DEFAULT_CONVERSION']='k';import pykx"])
+ [sys.executable, "-c",
+ "import os;os.environ['PYKX_DEFAULT_CONVERSION']='k';import pykx"])
assert 0 == ret.returncode
+
+
+def test_pykxreimport_qhome_reset_restore(kx):
+ # Inside the context QHOME is reset to the user's original QHOME; on exit the
+ # pykx-internal QHOME is restored.
+ qhome_before = os.environ.get('QHOME')
+ with kx.PyKXReimport():
+ assert os.environ['QHOME'] == os.environ['PYKX_OLD_QHOME']
+ assert os.environ.get('QHOME') == qhome_before
+
+
+# --- Environment-variable regressions ---------------------------------------
+
+def test_pykx_old_qpath_populated(kx):
+ # Regression: PYKX_OLD_QHOME/QPATH must hold real values so a child process
+ # can restore them. PYKX_OLD_QPATH used to always be empty.
+ assert os.environ.get('PYKX_OLD_QHOME', '') != ''
+ assert os.environ.get('PYKX_OLD_QPATH', '') != ''
+
+
+def test_qpath_no_duplicate_on_nested_import(kx, tmp_path):
+ # Regression: nested KDB-X Python imports must not keep appending
+ # `$QHOME/mod` to QPATH (it used to grow one entry per nesting level).
+ script = tmp_path / "nested_import.py"
+ script.write_text(
+ "import os, sys, subprocess\n"
+ "import pykx # noqa: F401\n"
+ "qp = os.environ.get('QPATH', '')\n"
+ "parts = [p for p in qp.split(os.pathsep) if p]\n"
+ "assert len(parts) == len(set(parts)), 'duplicate QPATH entries: ' + qp\n"
+ "depth = int(sys.argv[1]) if len(sys.argv) > 1 else 0\n"
+ "if depth < 3:\n"
+ " sys.exit(subprocess.run([sys.executable, sys.argv[0], str(depth + 1)]).returncode)\n"
+ )
+ ret = subprocess.run([sys.executable, str(script), "0"])
+ assert 0 == ret.returncode
+
+
+def test_setdefault_does_not_export_env(kx):
+ # Regression: `.pykx.setdefault` must only change the current process, it must
+ # NOT export PYKX_DEFAULT_CONVERSION (which would leak to child processes).
+ before = os.environ.get('PYKX_DEFAULT_CONVERSION')
+ try:
+ kx.q('.pykx.setdefault["pd"]')
+ assert os.environ.get('PYKX_DEFAULT_CONVERSION') == before
+ finally:
+ kx.q('.pykx.setdefault["default"]')
diff --git a/tests/test_schema.py b/tests/test_schema.py
index eb13958..df4839a 100644
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -8,10 +8,11 @@ def test_simple_schema(q, kx):
'col1': kx.GUIDAtom,
'col2': kx.TimeAtom,
'col3': kx.BooleanAtom,
- 'col4': kx.FloatVector})
+ 'col4': kx.FloatVector,
+ 'col5': kx.SecondVector})
assert isinstance(qtab, kx.Table)
- assert kx.q.cols(qtab).py() == ['col1', 'col2', 'col3', 'col4']
- assert kx.q('{exec t from 0!meta x}', qtab).py() == b'gtbf'
+ assert kx.q.cols(qtab).py() == ['col1', 'col2', 'col3', 'col4', 'col5']
+ assert kx.q('{exec t from 0!meta x}', qtab).py() == b'gtbfv'
def test_single_key_schema(q, kx):
diff --git a/tests/test_sql.py b/tests/test_sql.py
index 5dda2a4..6036bc1 100644
--- a/tests/test_sql.py
+++ b/tests/test_sql.py
@@ -170,7 +170,8 @@ def test_sql_get_input_values(q, kx):
)
@pytest.mark.embedded
def test_sql_string_col(q):
- q('t:([] optid:1 2 3;Market:`a`b`CBOE;date:3#2023.11.14;Symbol:("a";"b";"odMP=20"))')
+ q('''t:([] optid:1 2 3;Market:`a`b`CBOE;date:3#2023.11.14;
+ Symbol:(enlist "a";enlist "b";"odMP=20"))''')
qres = q('''select optid,Market,Symbol from t
where date = 2023.11.14,Market=`CBOE,Symbol like "odMP=20"''')
sqlres = q("""s)select optid,Market,Symbol from t
diff --git a/tests/test_toq.py b/tests/test_toq.py
index 3a229e3..1cd39f4 100644
--- a/tests/test_toq.py
+++ b/tests/test_toq.py
@@ -5,6 +5,7 @@
from pathlib import Path
from platform import system
from sys import getrefcount
+from threading import Thread
from uuid import UUID, uuid4
# Do not import Pandas, PyArrow, or KDB-X Python here - use the pd/pa/kx fixtures instead!
@@ -1008,6 +1009,68 @@ def test_from_numpy_ndarray_3(kx):
assert all(minute_array.np() == minute_array_np)
+@pytest.mark.nep49
+def test_from_numpy_recarray(kx):
+ # A structured/record array (as produced by ``Table.np``) converts to a q Table with one
+ # column per named field, rather than raising a TypeError (KXI-71774). This holds both for a
+ # ``numpy.recarray`` and for a plain structured ``numpy.ndarray``.
+ rec = np.array([(1, b'a'), (2, b'b')], dtype=[('x', 'i8'), ('y', 'S1')])
+ for arr in (rec, rec.view(np.recarray)):
+ tab = kx.toq(arr)
+ assert isinstance(tab, kx.Table)
+ assert kx.K(arr).py() == tab.py()
+ # The 'S1' field 'y' becomes a q char column (one char per row).
+ assert tab.py() == {'x': [1, 2], 'y': b'ab'}
+
+ # The reported round-trip must work out of the box: a single row char column.
+ single_char = kx.q('([] c:enlist " ")')
+ assert kx.toq(single_char.np()).py() == single_char.py()
+
+ # A range of column types should round-trip through ``Table.np`` -> ``toq`` for both the
+ # recarray and the plain structured ndarray forms.
+ tab = kx.q('([] a:1 2 3; b:`x`y`z; c:"abc"; d:1.1 2.2 3.3; e:0x010203; f:3?0Ng)')
+ assert kx.q('~', kx.toq(tab.np()), tab).py()
+ assert kx.q('~', kx.toq(np.asarray(tab.np())), tab).py()
+
+ # Empty tables preserve their (char) column typing rather than degrading to a list.
+ empty = kx.q('([] c:0#" ")')
+ assert kx.q('~', kx.toq(empty.np()), empty).py()
+
+ # 'U1' (str) fields become symbol columns, 'S1' (bytes) fields become char columns.
+ mixed = np.array([('a', b'z')], dtype=[('s', 'U1'), ('b', 'S1')])
+ mtab = kx.toq(mixed)
+ assert kx.q('type', mtab['s']).py() == 11 # symbol vector
+ assert kx.q('type', mtab['b']).py() == 10 # char vector
+
+ # ktype=KeyedTable is supported (keyed on a generated index).
+ assert isinstance(kx.toq(tab.np(), ktype=kx.KeyedTable), kx.KeyedTable)
+ assert isinstance(kx.toq(tab.np(), ktype=kx.Table), kx.Table)
+
+ # strings_as_char converts symbol fields to lists of char vectors.
+ sac = kx.toq(tab.np(), strings_as_char=True)
+ assert kx.q('type', sac['b']).py() == 0 # general list of char vectors
+
+ # handle_nulls preserves q nulls through the round-trip.
+ with_nulls = kx.q('([] a:1 2 0N 4; d:1.1 2.2 0n 4.4)')
+ assert kx.q('~', kx.toq(with_nulls.np(), handle_nulls=True), with_nulls).py()
+
+ # Unsupported ktypes and multi-dimensional structured arrays raise TypeError.
+ with pytest.raises(TypeError):
+ kx.toq(rec, kx.LongVector)
+ with pytest.raises(TypeError):
+ kx.toq(np.zeros((2, 2), dtype=[('a', 'i8')]))
+
+
+@pytest.mark.nep49
+def test_from_numpy_single_char(kx):
+ # The default conversion of a length-one 'S1' (bytes) array remains a CharAtom, matching the
+ # existing behaviour where a single byte becomes an atom (KXI-71774).
+ assert isinstance(kx.toq(np.array([b' '], dtype='S1')), kx.CharAtom)
+ # A CharVector can still be forced explicitly from a single byte.
+ assert isinstance(kx.CharVector(b' '), kx.CharVector)
+ assert kx.CharVector(b' ').py() == b' '
+
+
@pytest.mark.unlicensed
@pytest.mark.nep49
def test_from_numpy_incompatible_types(kx):
@@ -1373,6 +1436,49 @@ def test_from_arrow(kx, pa, pd):
assert a.combine_chunks() == kx.K(a).pa()
+@pytest.mark.licensed
+@pytest.mark.nep49
+def test_from_arrow_empty_string_column(kx, pd, pa):
+ schema = pa.schema([('s', pa.large_string()), ('i', pa.int64())])
+ via_string = isinstance(schema.empty_table().to_pandas()['s'].dtype, pd.StringDtype)
+ empty_str_type = 11 if via_string else 0
+
+ empty = kx.toq(schema.empty_table())
+ assert kx.q('type', empty['s']).py() == empty_str_type
+ assert kx.q('type', empty['i']).py() == 7
+
+ all_null = kx.toq(pa.table({'s': pa.array([None, None], type=pa.large_string())}))
+ assert kx.q('type', all_null['s']).py() == empty_str_type
+ if via_string:
+ assert kx.q('~', all_null['s'], kx.q('2#`')).py()
+
+ populated = kx.toq(pa.table({'s': pa.array(['a', None], type=pa.large_string())}))
+ assert kx.q('type', populated['s']).py() == 11
+ assert kx.q('~', populated['s'], kx.q('`a`')).py()
+
+
+@pytest.mark.licensed
+@pytest.mark.nep49
+def test_from_pandas_empty_string_column(kx, pd):
+ for data in ([], [None], [None, None]):
+ df = pd.DataFrame({'s': pd.Series(data, dtype='string')})
+ assert kx.q('type', kx.toq(df)['s']).py() == 11
+ assert kx.q('type', kx.toq(df, strings_as_char=True)['s']).py() == 0
+
+
+@pytest.mark.licensed
+@pytest.mark.nep49
+def test_from_pandas_arrow_string_column(kx, pd, pa):
+ if not hasattr(pd, 'ArrowDtype'):
+ pytest.skip('pd.ArrowDtype requires pandas>=1.5')
+ dtype = pd.ArrowDtype(pa.large_string())
+ for data in ([], [pd.NA], [pd.NA, pd.NA], ['a', pd.NA], ['a', 'b']):
+ col = kx.toq(pd.DataFrame({'s': pd.Series(data, dtype=dtype)}))['s']
+ assert kx.q('type', col).py() == 11
+ mixed = kx.toq(pd.DataFrame({'s': pd.Series(['a', pd.NA], dtype=dtype)}))['s']
+ assert kx.q('~', mixed, kx.q('`a`')).py()
+
+
@pytest.mark.licensed
@pytest.mark.nep49
def test_null_roundtrip(kx):
@@ -1760,3 +1866,61 @@ def test_2d_array_from_file_no_allocator(kx):
def test_embedding_segfault(kx):
df=pd.DataFrame(dict(embeddings=list(np.random.ranf((500, 10)).astype(np.float32))))
kx.toq(df)
+
+
+def test_table_in_thread(kx):
+
+ def generate_data():
+ now = datetime.utcnow()
+ trade_id = uuid4()
+ df = pd.DataFrame(
+ [[now, "VOD", "LSE", "buy", 75.90, 100, trade_id]],
+ columns=["time", "sym", "exch", "side", "price", "size", "tradeID"]
+ )
+ return kx.Table(df)
+
+ def generate_data_in_thread():
+ res = []
+
+ def thread_body():
+ d = generate_data()
+ res.append(d)
+ t = Thread(target=thread_body)
+ t.start()
+ t.join()
+ return res[0]
+
+ assert isinstance(generate_data_in_thread(), kx.Table)
+
+
+@pytest.mark.licensed_only
+def test_keep_local_times_off(kx):
+ a = pd.Timestamp('2026-07-28 15:41:22.734077-0400', tz='America/New_York')
+ b = pd.Series([a, a])
+
+ k = kx.q('2026.07.28D15:41:22.734077000')
+ kk = kx.q('2026.07.28D15:41:22.734077000 2026.07.28D15:41:22.734077000')
+ k2 = kx.q('2026.07.28D19:41:22.734077000')
+ kk2 = kx.q('2026.07.28D19:41:22.734077000 2026.07.28D19:41:22.734077000')
+
+ assert kx.toq(a) == k2
+ assert (kx.toq(b) == kk2).all()
+ assert kx.toq(a) != k
+ assert not (kx.toq(b) == kk).any()
+
+
+@pytest.mark.isolate
+def test_keep_local_times_on():
+ import os
+ os.environ['PYKX_KEEP_LOCAL_TIMES'] = '1'
+ import pykx as kx
+ import pandas as pd
+
+ a = pd.Timestamp('2026-07-28 15:41:22.734077-0400', tz='America/New_York')
+ b = pd.Series([a, a])
+
+ k = kx.q('2026.07.28D15:41:22.734077000')
+ kk = kx.q('2026.07.28D15:41:22.734077000 2026.07.28D15:41:22.734077000')
+
+ assert kx.toq(a) == k
+ assert (kx.toq(b) == kk).all()
diff --git a/tests/test_util.py b/tests/test_util.py
index a76df66..76de4f6 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -106,22 +106,41 @@ def time_when_first_run():
@pytest.mark.ipc
-def test_pickle_pykx_df_block_manager(kx, q):
+def test_pickle_pykx_df_block_manager(q):
"""Test that a Pandas DataFrame created by KDB-X Python can be deserialized without KDB-X Python.
- DataFrames that originate from KDB-X Python have a custom block manager. We have to take care to
- serialize it as a regular block manager so that it can be deserialized without
- KDB-X Python installed.
+ On Pandas <3.0, DataFrames that originate from KDB-X Python use a custom block manager
+ (`pykx.util.BlockManagerUnconsolidated`). We have to take care to serialize it as a regular
+ block manager so that it can be deserialized without KDB-X Python installed. On Pandas 3.0+
+ `.pd()` already builds a standard `BlockManager`, so nothing KDB-X specific is present.
"""
df = q('([] date:9?.z.D; id:9?9; time:9?.z.N; bs:9?9; bp:9?9f; ap:9?9f; as:9?9)').pd()
serialized = pickle.dumps(df)
assert b'pykx' not in serialized
- if not kx.config.pandas_gt2:
- # `df._data` is the block manager
- assert type(df._data).__name__.encode() not in serialized
+ assert b'BlockManagerUnconsolidated' not in serialized
+
+
+def test_pykx_df_equals_both_directions(kx, q):
+ """`DataFrame.equals` comparing a KDB-X Python DataFrame with a native Pandas DataFrame.
+
+ Pandas 2.2.0 started comparing the `_mgr` type in `DataFrame.equals`. That makes the reverse
+ direction (KDB-X Python DataFrame as the caller) return False only while `.pd()` still uses
+ its custom `_mgr` - i.e. Pandas 2.2 up to (but not including) 3.0 (a documented known issue,
+ see docs/help/issues.md). Before 2.2 there is no `_mgr` check, and on 3.0+ `.pd()` builds a
+ standard `BlockManager`, so both directions are symmetric.
+ """
+ import pandas as pd
+ df1 = pd.DataFrame({'cl': ['foo']})
+ df2 = q('([] cl:enlist `foo)').pd()
+ # Native Pandas DataFrame as caller is True on every supported Pandas version.
+ assert df1.equals(df2)
+ pandas_version = tuple(int(x) for x in pd.__version__.split('.')[:2])
+ reverse_asymmetric = pandas_version >= (2, 2) and not kx.config.pandas_gt2
+ if reverse_asymmetric:
+ # Known limitation on Pandas 2.2 to 2.x: custom `_mgr` makes this direction False.
+ assert not df2.equals(df1)
else:
- # `df._mgr` is the block manager in Pandas 3
- assert type(df._mgr).__name__.encode() not in serialized
+ assert df2.equals(df1)
@pytest.mark.unlicensed
diff --git a/tests/test_virtual_table.py b/tests/test_virtual_table.py
new file mode 100644
index 0000000..2151190
--- /dev/null
+++ b/tests/test_virtual_table.py
@@ -0,0 +1,201 @@
+from pathlib import Path
+
+import pytest
+
+
+class TestVirtualTable:
+
+ def make_tab(self, kx):
+ pq = kx.module.use('kx.pq')
+ pq.t = kx.module.use('kx.pq.t')
+ tab = pq.pq(Path('tests/data/types.parquet'))
+ return tab
+
+ def test_select(self, kx):
+ tab = self.make_tab(kx)
+ assert kx.q('~', tab.select(kx.Column('col0') & kx.Column('col1')),
+ kx.q('flip `col0`col1!(110100100b;1 2 3 4 5 6 7 8 9)')).py()
+
+ def test_exec(self, kx):
+ tab = self.make_tab(kx)
+ assert kx.q('~', tab.exec(kx.Column('col1')), kx.q('1 2 3 4 5 6 7 8 9')).py()
+
+ errStr = "pykx.VirtualTable objects cannot be operated on directly, you must .select() or .exec() data from them first" # noqa: E501
+
+ # Conversion methods
+ def test_cast(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).cast(kx.FloatAtom)
+ assert self.errStr in str(err.value)
+
+ def test_py(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).py()
+ assert self.errStr in str(err.value)
+
+ def test_np(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).np()
+ assert self.errStr in str(err.value)
+
+ def test_pd(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).pd()
+ assert self.errStr in str(err.value)
+
+ def test_pa(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).pa()
+ assert self.errStr in str(err.value)
+
+ # Utility methods
+ def test_any(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).any()
+ assert self.errStr in str(err.value)
+
+ def test_all(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).all()
+ assert self.errStr in str(err.value)
+
+ def test_copy(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).copy()
+ assert self.errStr in str(err.value)
+
+ def test_is_atom(self, kx):
+ assert self.make_tab(kx).is_atom is False
+
+ # Comparison operators
+ def test_lt(self, kx):
+ with pytest.raises(kx.QError) as err:
+ assert self.make_tab(kx) < self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_le(self, kx):
+ with pytest.raises(kx.QError) as err:
+ assert self.make_tab(kx) <= self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_eq(self, kx):
+ with pytest.raises(kx.QError) as err:
+ assert self.make_tab(kx) == self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_ne(self, kx):
+ with pytest.raises(kx.QError) as err:
+ assert self.make_tab(kx) != self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_gt(self, kx):
+ with pytest.raises(kx.QError) as err:
+ assert self.make_tab(kx) > self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_ge(self, kx):
+ with pytest.raises(kx.QError) as err:
+ assert self.make_tab(kx) >= self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ # Arithmetic operators
+ def test_add(self, kx):
+ with pytest.raises(kx.QError) as err:
+ assert self.make_tab(kx) + self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_radd(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__radd__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_sub(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx) - self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_rsub(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__rsub__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_mul(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx) * self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_rmul(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__rmul__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_truediv(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx) / self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_rtruediv(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__rtruediv__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_floordiv(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx) // self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_rfloordiv(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__rfloordiv__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_mod(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx) % self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_rmod(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__rmod__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_divmod(self, kx):
+ with pytest.raises(kx.QError) as err:
+ divmod(self.make_tab(kx), self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_rdivmod(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__rdivmod__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_pow(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx) ** self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_rpow(self, kx):
+ with pytest.raises(kx.QError) as err:
+ self.make_tab(kx).__rpow__(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ # Unary operators
+ def test_neg(self, kx):
+ with pytest.raises(kx.QError) as err:
+ -self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_pos(self, kx):
+ with pytest.raises(kx.QError) as err:
+ +self.make_tab(kx)
+ assert self.errStr in str(err.value)
+
+ def test_abs(self, kx):
+ with pytest.raises(kx.QError) as err:
+ abs(self.make_tab(kx))
+ assert self.errStr in str(err.value)
+
+ def test_bool(self, kx):
+ with pytest.raises(kx.QError) as err:
+ bool(self.make_tab(kx))
+ assert self.errStr in str(err.value)
diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py
index 16d0c67..4968833 100644
--- a/tests/test_wrappers.py
+++ b/tests/test_wrappers.py
@@ -15,6 +15,7 @@
from uuid import UUID
import itertools
import sys
+import warnings
# Do not import Pandas, PyArrow, or pykx here - use the pd/pa/kx fixtures instead!
import numpy as np
@@ -724,6 +725,15 @@ def test_is_null_and_is_inf(self, q):
assert not q('{x*y+z}').is_null
assert not q('{x*y+z}').is_inf
+ for expr in ('0Ng', '0Nh', '0Ne', '0Nf', '0Np', '0Nn', '0Nu', '0Nv', '0Nt', '" "', '`'):
+ atom = q(expr)
+ for prop in ('is_null', 'is_inf', 'is_pos_inf', 'is_neg_inf'):
+ assert type(getattr(atom, prop)) is bool, f'{expr}.{prop}'
+ for expr in ('0We', '-0We', '0Wf', '-0Wf', '0Wh', '-0Wh'):
+ atom = q(expr)
+ for prop in ('is_inf', 'is_pos_inf', 'is_neg_inf'):
+ assert type(getattr(atom, prop)) is bool, f'{expr}.{prop}'
+
@pytest.mark.nep49
def test_null_np(self, q, kx):
for type_char in 'hij':
@@ -1084,6 +1094,25 @@ def test_arr(self, kx, q):
assert np.asarray(kx.DateAtom(datetime(2003, 4, 5))).dtype == "= version.parse('2.0')
+ for obj in (kx.LongAtom(42), kx.q('til 5')):
+ assert np.asarray(obj, dtype=object).dtype == object
+ with warnings.catch_warnings():
+ warnings.simplefilter('error', DeprecationWarning)
+ assert np.array(obj) is not None
+ assert np.array(obj, copy=True) is not None
+ if numpy_gt2:
+ # A dtype conversion is unavoidably a copy, so an explicit copy=False must
+ # be refused. NumPy 1.x has lenient copy semantics and does not raise here.
+ with pytest.raises(ValueError):
+ np.array(obj, dtype=object, copy=False)
+
@pytest.mark.unlicensed
def test_arr_nulls(self, kx, q):
assert np.asarray(kx.ShortAtom(None)).dtype == "int16"
@@ -1817,6 +1846,10 @@ def f(type_code, zero):
assert q(f'@[v;-3?50;:;0N{type_code}]').has_nulls
assert q(f'@[v;-3?50;:;0W{type_code}]').has_infs
+ for expr in (v, q(f'@[v;-3?50;:;0N{type_code}]'), q(f'@[v;-3?50;:;0W{type_code}]')):
+ assert type(expr.has_nulls) is bool, f'{type_code}.has_nulls'
+ assert type(expr.has_infs) is bool, f'{type_code}.has_infs'
+
types = (
('h', '0h'), ('i', '0i'), ('j', '0j'), ('e', '10000e'), ('f', '10000f'), ('p', '0p'),
('m', '2000.01m'), ('d', '2000.01.01d'), ('n', '00:00:00.000000000'), ('u', '00:00'),
@@ -1825,6 +1858,21 @@ def f(type_code, zero):
for type_code, zero in types:
f(type_code, zero)
+ for expr in ('0Ng,3?0Ng', 'v where not null v:10?0Ng', '011011100b', '0xdeadbeef'):
+ vec = q(expr)
+ assert type(vec.has_nulls) is bool, f'{expr}.has_nulls'
+ assert type(vec.has_infs) is bool, f'{expr}.has_infs'
+
+ def test_has_infs_negative_infinity(self, q, kx):
+ assert isinstance(q('-0w -0w -0w'), kx.FloatVector)
+ assert q('-0w -0w -0w').has_infs
+ assert isinstance(q('-0we, -0we, -0we'), kx.RealVector)
+ assert q('-0we, -0we, -0we').has_infs
+ assert q('1 2 -0w').has_infs
+ assert q('1 2 0w').has_infs
+ assert not q('1 2 3f').has_infs
+ assert type(q('-0w -0w').has_infs) is bool
+
def test_np_timestampvector_nulls(self, kx):
assert pd.isna(kx.q('0Np').py())
assert kx.q('enlist 0Np').py() == [kx.TimestampAtom(kx.q('0Np'))]
@@ -2430,6 +2478,27 @@ def test_empty_vector(self, q):
assert q('"p"$()').np().dtype == np.dtype('datetime64[ns]')
assert q('"p"$()').np(raw=True).dtype == np.int64
+ @pytest.mark.nep49
+ def test_py_multiple_nulls(self, q, kx):
+ vec = kx.q('(2023.01.01D12:00:00.000000000; 0Np; 2023.01.03D06:30:00.000000000; 0Np; 0Np)')
+ assert isinstance(vec, kx.TimestampVector)
+ result = vec.py()
+ assert result[0] == datetime(2023, 1, 1, 12, 0, 0)
+ assert result[2] == datetime(2023, 1, 3, 6, 30, 0)
+ assert result[1] is pd.NaT
+ assert result[3] is pd.NaT
+ assert result[4] is pd.NaT
+ assert None not in result
+
+ @pytest.mark.nep49
+ def test_py_tzinfo_nulls(self, kx):
+ vec = kx.q('(2023.01.01D12:00:00.000000000; 0Np; 2023.01.03D06:30:00.000000000)')
+ result = vec.py(tzinfo=pytz.timezone('GMT'))
+ assert result[0] == datetime(2023, 1, 1, 12, 0, 0, tzinfo=pytz.utc)
+ assert result[0].tzinfo is not None
+ assert result[2] == datetime(2023, 1, 3, 6, 30, 0, tzinfo=pytz.utc)
+ assert result[1] is pd.NaT
+
class Test_MonthVector:
q_vec_str = '2006.04 1947.10 1876.04 2170.01m'
@@ -2901,10 +2970,20 @@ def test_pa(self, q, pa):
t = q(self.q_table_str)
assert all(t.pa().to_pandas() == t.pd(raw_guids=True))
+ def test_pd_raw_guids_repr(self, q):
+ # A raw_guids=True DataFrame must be renderable: pandas/numpy call
+ # PandasUUIDArray.__array__(dtype) when formatting, which previously
+ # raised `TypeError: __array__() takes 1 positional argument but 2 were given`.
+ df = q(self.q_table_str).pd(raw_guids=True)
+ assert repr(df)
+ assert np.asarray(df['c'].values, dtype=object).shape == (3,)
+
def test_has_null_and_has_inf(self, q):
table = q('([]0w,9?1f;0n,9?1f)')
assert table.has_nulls
assert table.has_infs
+ assert type(table.has_nulls) is bool
+ assert type(table.has_infs) is bool
def test_null_to_pandas(self, q, pd):
q('ty:2 5 6 7 8 9 10 11 12 13 14 16 17 18 19h')
@@ -3430,6 +3509,8 @@ def test_has_null_and_has_inf(self, q):
dic = q('flip ([]0w,9?1f;0n,9?1f)')
assert dic.has_nulls
assert dic.has_infs
+ assert type(dic.has_nulls) is bool
+ assert type(dic.has_infs) is bool
def test_nested_dict(self, q):
single_nested = {
@@ -4222,6 +4303,9 @@ def compare_nulls(q_null, py_null):
def test_infinites(kx, q, pa):
import datetime
+ # Pandas >= 2.0 supports non-nanosecond resolution, so month/date/minute
+ # infinities (out of the nanosecond range) convert instead of overflowing.
+ pandas_gt1 = not pd.__version__.split('.')[0] == '1'
def compare_infinites(q_infinite, py_infinite):
assert type(q_infinite) == type(py_infinite)
@@ -4248,7 +4332,7 @@ def compare_all_infinites(q_infinites, py_infinites, conv, ind=None, skip=None):
py_positive_infinites = [math.inf, math.inf, math.inf, float('inf'), float('inf'),
datetime.datetime(2262, 4, 11, 23, 47, 16, 854775), 2147484007,
2147494604, datetime.timedelta(106751, 16, 854775, 0, 47, 23),
- datetime.timedelta(-3220, 4, 33138, 0, 5, 5),
+ datetime.timedelta(minutes=2147483647),
datetime.timedelta(24855, 7, 0, 0, 14, 3),
datetime.timedelta(24, 23, 647000, 0, 31, 20)]
compare_all_infinites(positive_infinites, py_positive_infinites, 'py')
@@ -4256,7 +4340,7 @@ def compare_all_infinites(q_infinites, py_infinites, conv, ind=None, skip=None):
py_negative_infinites = [-math.inf, -math.inf, -math.inf, float('-inf'), float('-inf'),
datetime.datetime(1707, 9, 22, 0, 12, 43, 145224), -2147483287,
-2147472690, datetime.timedelta(-106752, 43, 145224, 0, 12),
- datetime.timedelta(3219, 55, 966861, 0, 54, 18),
+ datetime.timedelta(minutes=-2147483647),
datetime.timedelta(-24856, 53, 0, 0, 45, 20),
datetime.timedelta(-25, 36, 353000, 0, 28, 3)]
compare_all_infinites(negative_infinites, py_negative_infinites, 'py')
@@ -4309,18 +4393,33 @@ def compare_all_infinites(q_infinites, py_infinites, conv, ind=None, skip=None):
np.float64('-inf'), pd.Timestamp('1707-09-22 00:12:43.145224193'),
None, None, pd.Timedelta(-9223372036854775807, 'ns'), None,
pd.Timedelta(-2147483647, 's'), pd.Timedelta(-2147483647, 'ms')]
+ # month (6), date (7) and minute (9) exceed pandas' nanosecond range. On
+ # pandas >= 2.0 they convert (coarser resolution) and are verified below:
+ # month/date pinned via their (already verified) .np() datetime64 value, minute
+ # directly. On pandas 1.x these overflow, so they remain skipped.
skip = [6, 7, 9]
- # 6 7 9 Values out of range - Pandas constructors block them
compare_all_infinites(positive_infinites, pd_positive_infinites, 'pd', skip=skip)
compare_all_infinites(negative_infinites, pd_negative_infinites, 'pd', skip=skip)
+ if pandas_gt1:
+ for i in (6, 7):
+ assert positive_infinites[i].pd().to_datetime64() == np_positive_infinites[i]
+ assert negative_infinites[i].pd().to_datetime64() == np_negative_infinites[i]
+ assert positive_infinites[9].pd().to_numpy() == np_positive_infinites[9]
+ assert negative_infinites[9].pd().to_numpy() == np_negative_infinites[9]
# infinites in Vectors .pd()
- skip = [6, 7, 9] # 6 7 9 Values out of range - Pandas constructors block them
+ skip = [6, 7, 9] # verified below on pandas >= 2.0 (overflow on pandas 1.x)
pd_infinites = [[x, y] for x, y in zip(pd_positive_infinites, pd_negative_infinites)]
pd_infinites[5][0] = pd.Timestamp('1707-09-22T00:12:43.145224191')
compare_all_infinites(infinites, pd_infinites, 'pd', ind=0, skip=skip)
pd_infinites[5][1] = pd.Timestamp('1707-09-22 00:12:43.145224193')
compare_all_infinites(infinites, pd_infinites, 'pd', ind=1, skip=skip)
+ if pandas_gt1:
+ for i in (6, 7):
+ assert infinites[i].pd()[0].to_datetime64() == np_infinites[i][0]
+ assert infinites[i].pd()[1].to_datetime64() == np_infinites[i][1]
+ assert infinites[9].pd()[0].to_numpy() == np_infinites[9][0]
+ assert infinites[9].pd()[1].to_numpy() == np_infinites[9][1]
# Atom infinites .pa()
pa_positive_infinites = [np.int16(32767), np.int32(2147483647), np.int64(9223372036854775807),
@@ -4335,16 +4434,23 @@ def compare_all_infinites(q_infinites, py_infinites, conv, ind=None, skip=None):
None, None, pd.Timedelta(-9223372036854775807, 'ns'), None,
pd.Timedelta(-2147483647, 's'),
pd.Timedelta(-2147483647, 'ms')]
+ # month (6), date (7) and minute (9) exceed pandas' nanosecond range; verified
+ # below on pandas >= 2.0 (month/date pinned via .np(), minute directly). On
+ # pandas 1.x these overflow, so they remain skipped.
skip = [6, 7, 9]
- # 6, 7, 9 .pa runs but creates out of bounds objects
compare_all_infinites(positive_infinites, pa_positive_infinites, 'pa', skip=skip)
compare_all_infinites(negative_infinites, pa_negative_infinites, 'pa', skip=skip)
+ if pandas_gt1:
+ for i in (6, 7):
+ assert positive_infinites[i].pa().to_datetime64() == np_positive_infinites[i]
+ assert negative_infinites[i].pa().to_datetime64() == np_negative_infinites[i]
+ assert positive_infinites[9].pa().to_numpy() == np_positive_infinites[9]
+ assert negative_infinites[9].pa().to_numpy() == np_negative_infinites[9]
# infinites in Vectors .pa()
- skip = [5, 6, 7, 9]
- # 5, 6 pyarrow.lib.ArrowNotImplementedError: Unsupported datetime64 time unit
- # 7 OverflowError: days=-2147472692; must have magnitude <= 999999999
- # 9 pyarrow.lib.ArrowNotImplementedError: Unsupported timedelta64 time unit
+ # timestamp (5) is verified below; month (6), date (7) and minute (9) raise and
+ # are asserted with pytest.raises after the value comparisons.
+ skip = [6, 7, 9]
pa_infinites = [[x, y] for x, y in zip(pa_positive_infinites, pa_negative_infinites)]
pa_infinites[0][0] = pa.array([32767], pa.int16())[0]
pa_infinites[1][0] = pa.array([2147483647], pa.int32())[0]
@@ -4361,12 +4467,22 @@ def compare_all_infinites(q_infinites, py_infinites, conv, ind=None, skip=None):
pa_infinites[2][1] = pa.array([-9223372036854775807], pa.int64())[0]
pa_infinites[3][1] = pa.array([np.float32('-inf')], pa.float32())[0]
pa_infinites[4][1] = pa.array([np.float64('-inf')], pa.float64())[0]
- pa_infinites[5][1] = pa.array([-9223372036854775809+946684800000000000], pa.timestamp('ns'))[0]
+ pa_infinites[5][1] = pa.array([np.datetime64('1707-09-22T00:12:43.145224193')],
+ pa.timestamp('ns'))[0]
pa_infinites[8][1] = pa.array([-9223372036854775807], pa.duration('ns'))[0]
pa_infinites[10][1] = pa.array([-2147483647], pa.duration('s'))[0]
pa_infinites[11][1] = pa.array([-2147483647], pa.duration('ms'))[0]
compare_all_infinites(infinites, pa_infinites, 'pa', ind=1, skip=skip)
+ # month (6) and minute (9) vector .pa() raise on an unsupported pyarrow unit;
+ # date (7) constructs but its out-of-range value raises when materialised.
+ with pytest.raises(pa.lib.ArrowNotImplementedError):
+ infinites[6].pa()
+ with pytest.raises(pa.lib.ArrowNotImplementedError):
+ infinites[9].pa()
+ with pytest.raises(OverflowError):
+ infinites[7].pa()[0].as_py()
+
# Conversions of nested K lists requires a license, We need to be able to call
# __getitem__ on the list to get correctly typed Numpy arrays to use.