Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
get_unpickle_hmac_key,
set_unpickle_hmac_key,
)
from pyrosetta.utility import has_cereal


pyrosetta.init(extra_options="-constant_seed", set_logging_handler="logging")
Expand Down Expand Up @@ -220,9 +221,11 @@ def test_score_serialization(self):
with self.assertRaises(Exception):
test_pose.cache[str(obj_type)] = value_input
continue
else:
test_pose.cache[str(obj_type)] = value_input
value_output = test_pose.cache[str(obj_type)]
if obj_type == pyrosetta.Pose and not has_cereal():
# `Pose` instances require cereal support for round-trip serialization
continue
test_pose.cache[str(obj_type)] = value_input
value_output = test_pose.cache[str(obj_type)]

# Test instance types
self.assertIsInstance(value_input, obj_type)
Expand Down Expand Up @@ -517,7 +520,7 @@ def test_pose_cache(self):
self.assertEqual(self.pose.cache["bytes_to_str"], "ASCII binary")

with self.assertWarns(UserWarning):
setPoseExtraScore(self.pose, "invalid_bytes", pickle.dumps("raw binary")) # Manually set; warns since it's raw binary
setPoseExtraScore(self.pose, "invalid_bytes", pickle.dumps("raw binary", protocol=pickle.DEFAULT_PROTOCOL)) # Manually set; warns since it's raw binary
with self.assertRaises(UnicodeDecodeError):
self.pose.cache["invalid_bytes"]
with self.assertRaises(UnicodeDecodeError):
Expand All @@ -528,7 +531,7 @@ def test_pose_cache(self):
with self.assertRaises(KeyError):
self.pose.cache["invalid_bytes"]

for bytestring in (b'ASCII binary', pickle.dumps("raw binary")):
for bytestring in (b'ASCII binary', pickle.dumps("raw binary", protocol=pickle.DEFAULT_PROTOCOL)):
self.pose.cache.extra["bytes"] = bytestring # Automatically gets serialized
self.assertIn("bytes", self.pose.cache.extra.string)
with self.assertRaises(KeyError):
Expand Down Expand Up @@ -780,9 +783,10 @@ def test_pose_cache(self):
with self.assertWarns(UserWarning):
self.pose.cache.metrics.string["float"] = 1e5
self.assertIn("float", self.pose.cache.metrics.real.keys())
with self.assertWarns(UserWarning):
self.pose.cache.metrics.real["pose"] = pyrosetta.pose_from_sequence("DATA")
self.assertIn("pose", self.pose.cache.metrics.string.keys())
if has_cereal():
with self.assertWarns(UserWarning):
self.pose.cache.metrics.real["pose"] = pyrosetta.pose_from_sequence("DATA")
self.assertIn("pose", self.pose.cache.metrics.string.keys())

with self.assertWarns(UserWarning):
self.pose.cache.extra.real["str"] = "String"
Expand Down Expand Up @@ -870,27 +874,28 @@ def test_pose_cache(self):
with self.assertWarns(ClobberWarning):
dict(self.pose.cache)

# Test that `CacheableDataType.SIMPLE_METRIC_DATA` pose data is not automatically set by accessing `Pose.cache`
pose = pyrosetta.io.pose_from_sequence("TEST/SIMPLE/METRIC/DATA")
pickler = partial(pickle.dumps, protocol=pickle.DEFAULT_PROTOCOL)
bytes_start_1 = pickler(pose)
self.assertFalse(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertFalse(pose.cache.metrics._has_sm_data())
_ = dict(pose.cache) # Access `Pose.cache`
self.assertFalse(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertFalse(pose.cache.metrics._has_sm_data())
bytes_final_1 = pickler(pose)
self.assertEqual(bytes_start_1, bytes_final_1, msg="Pose is not bitwise identical after accessing `Pose.cache`.")
pose.cache.metrics["pi"] = math.pi # Add SimpleMetrics data
bytes_start_2 = pickler(pose)
self.assertTrue(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertTrue(pose.cache.metrics._has_sm_data())
_ = dict(pose.cache) # Access `Pose.cache`
self.assertTrue(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertTrue(pose.cache.metrics._has_sm_data())
bytes_final_2 = pickler(pose)
self.assertEqual(bytes_start_2, bytes_final_2, msg="Pose is not bitwise identical after accessing `Pose.cache`.")
self.assertNotEqual(bytes_final_1, bytes_final_2, msg="Pose is bitwise identical after adding SimpleMetrics data to `Pose.cache`.")
if has_cereal():
# Test that `CacheableDataType.SIMPLE_METRIC_DATA` pose data is not automatically set by accessing `Pose.cache`
pose = pyrosetta.io.pose_from_sequence("TEST/SIMPLE/METRIC/DATA")
pickler = partial(pickle.dumps, protocol=pickle.DEFAULT_PROTOCOL)
bytes_start_1 = pickler(pose)
self.assertFalse(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertFalse(pose.cache.metrics._has_sm_data())
_ = dict(pose.cache) # Access `Pose.cache`
self.assertFalse(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertFalse(pose.cache.metrics._has_sm_data())
bytes_final_1 = pickler(pose)
self.assertEqual(bytes_start_1, bytes_final_1, msg="Pose is not bitwise identical after accessing `Pose.cache`.")
pose.cache.metrics["pi"] = math.pi # Add SimpleMetrics data
bytes_start_2 = pickler(pose)
self.assertTrue(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertTrue(pose.cache.metrics._has_sm_data())
_ = dict(pose.cache) # Access `Pose.cache`
self.assertTrue(pose.data().has(CacheableDataType.SIMPLE_METRIC_DATA))
self.assertTrue(pose.cache.metrics._has_sm_data())
bytes_final_2 = pickler(pose)
self.assertEqual(bytes_start_2, bytes_final_2, msg="Pose is not bitwise identical after accessing `Pose.cache`.")
self.assertNotEqual(bytes_final_1, bytes_final_2, msg="Pose is bitwise identical after adding SimpleMetrics data to `Pose.cache`.")

# Test `Pose.cache` access to SimpleMetrics data when `CacheableDataType.SIMPLE_METRIC_DATA` is not yet set up
pose = pyrosetta.io.pose_from_sequence("EMPTY/DATA/CACHE")
Expand Down Expand Up @@ -1236,7 +1241,7 @@ def test_secure_unpickler(self):

# Test secure serialization round-trip
data = {"foo": [1, 2, 3], "bar": ("String", b"Bytes"), "baz": complex(1, -2)}
obj = pickle.dumps(data, protocol=5)
obj = pickle.dumps(data, protocol=pickle.DEFAULT_PROTOCOL)
key = get_unpickle_hmac_key()
if key is not None:
obj = SecureSerializerBase._prepend_hmac_tag(obj, key)
Expand All @@ -1263,17 +1268,19 @@ def __reduce__(self):
data = {
"foo": list(range(10)),
"bar": dict(enumerate([complex(1, i) for i in range(10)])),
"baz": pyrosetta.pose_from_sequence("TEST"),
}
if has_cereal():
data["baz"] = pyrosetta.pose_from_sequence("TEST")
_hmac_size = 32
set_unpickle_hmac_key(os.urandom(_hmac_size))
self.assertEqual(len(get_unpickle_hmac_key()), _hmac_size)
string = SecureSerializerBase.secure_to_base64_pickle(data)
out = SecureSerializerBase.secure_from_base64_pickle(string)
self.assertEqual(out["foo"], data["foo"])
self.assertEqual(out["bar"], data["bar"])
self.assertIsInstance(out["baz"], pyrosetta.Pose)
self.assertEqual(out["baz"].sequence(), data["baz"].sequence())
if has_cereal():
self.assertIsInstance(out["baz"], pyrosetta.Pose)
self.assertEqual(out["baz"].sequence(), data["baz"].sequence())
# Test tampering with HMAC tag
arr = bytearray(base64.b64decode(string.encode(SecureSerializerBase._encoder)))
_hmac_tag_idx = 3
Expand Down Expand Up @@ -1324,11 +1331,13 @@ def __reduce__(self):
else:
instance = module()
test_pose.cache[builtin] = instance
_test_pose = SecureSerializerBase.secure_loads(
pickle.dumps(test_pose, protocol=pickle.HIGHEST_PROTOCOL)
)
_instance = _test_pose.cache[builtin]
self.assertIsInstance(_instance, type(instance))
self.assertIsInstance(test_pose.cache[builtin], type(instance))
if has_cereal():
_test_pose = SecureSerializerBase.secure_loads(
pickle.dumps(test_pose, protocol=pickle.DEFAULT_PROTOCOL)
)
_instance = _test_pose.cache[builtin]
self.assertIsInstance(_instance, type(instance))

# Test disallowed packages
test_pose = pyrosetta.pose_from_sequence("TEST/CACHE")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,22 @@ def setUp(self, local_dir=workdir):
if not self._dask_scheduler:
n_workers = 2
threads_per_worker = 2
diagnostics_port = None
dashboard_address = None
local_cluster_kwargs = dict(
n_workers=n_workers,
threads_per_worker=threads_per_worker,
dashboard_address=dashboard_address,
local_directory=local_dir,
)
if __dask_version__ <= (2, 1, 0):
self.local_cluster = dask.distributed.LocalCluster(
n_workers=n_workers,
threads_per_worker=threads_per_worker,
diagnostics_port=diagnostics_port,
local_dir=local_dir,
local_cluster_kwargs["local_dir"] = local_cluster_kwargs.pop(
"local_directory", local_dir
)
else:
self.local_cluster = dask.distributed.LocalCluster(
n_workers=n_workers,
threads_per_worker=threads_per_worker,
diagnostics_port=diagnostics_port,
local_directory=local_dir,
if __dask_version__ < (2026, 6, 0):
local_cluster_kwargs["diagnostics_port"] = local_cluster_kwargs.pop(
"dashboard_address", dashboard_address
)
self.local_cluster = dask.distributed.LocalCluster(**local_cluster_kwargs)
cluster = self.local_cluster
else:
self.local_cluster = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
import time
import unittest

from pyrosetta.utility import get_package_version


__dask_version__ = get_package_version("dask")

class TestDaskArgs(unittest.TestCase):

Expand All @@ -36,8 +40,14 @@ def test_worker_extra(self):
import pyrosetta.distributed.tasks.score as score

# Setup cluster scheduler and working directory
dashboard_address = None
local_cluster_kwargs = dict(n_workers=0, dashboard_address=dashboard_address)
if __dask_version__ < (2026, 6, 0):
local_cluster_kwargs["diagnostics_port"] = local_cluster_kwargs.pop(
"dashboard_address", dashboard_address
)
with tempfile.TemporaryDirectory() as workdir, LocalCluster(
n_workers=0, diagnostics_port=None
**local_cluster_kwargs
) as cluster:

# Context manager controls launch & teardown of test worker
Expand Down Expand Up @@ -102,8 +112,8 @@ def one_worker(init_flags):
cluster_result = delayed(protocol)(pose).compute()

self.assertEqual(
cluster_result.scores["total_score"],
local_result.scores["total_score"]
cluster_result.pose.cache["total_score"],
local_result.pose.cache["total_score"]
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,26 @@ class TestAlignment(unittest.TestCase):

@staticmethod
def _rotation_matrix(axis, theta):
from numpy import cross, eye
from scipy.linalg import expm, norm
return expm(cross(eye(3), axis/norm(axis)*theta))
"""
NumPy-only equivalent of the SciPy implementation:
from numpy import cross, eye
from scipy.linalg import expm, norm
return expm(cross(eye(3), axis/norm(axis)*theta))
"""
from numpy import array, cos, eye, sin
from numpy.linalg import norm

axis = array(axis, dtype=float)
axis /= norm(axis)
x, y, z = axis

K = array([
[0, -z, y],
[z, 0, -x],
[-y, x, 0],
])

return eye(3) + sin(theta) * K + (1 - cos(theta)) * (K @ K)

def test_superimpose(self):
numpy.random.seed(1663)
Expand Down
13 changes: 13 additions & 0 deletions source/src/python/PyRosetta/src/test/T021_Pose_Methods.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# :noTabs=true:
# (c) Copyright Rosetta Commons Member Institutions.
# (c) This file is part of the Rosetta software suite and is made available under license.
# (c) The Rosetta software is developed by the contributing members of the Rosetta Commons.
# (c) For more information, see http://www.rosettacommons.org.
# (c) Questions about this can be addressed to University of Washington CoMotion, email: license@uw.edu.

__author__ = "Jason C. Klima"

from utils.distributed import run_unittest

if __name__ == "__main__":
run_unittest("pyrosetta.tests.bindings.core.test_pose", timeout=60)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

-- i guess this is one way to take care of this issue but the issue is that test will get silently "green" if no numpy was installed. Have you considered updating get_required_pyrosetta_python_packages_for_release_package so it returns at least numpy for non-distributed platform? That way we should always be able to run this test. Thoughts?

https://github.com/RosettaCommons/rosetta/blob/main/tests/benchmark/tests/__init__.py#L520

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lyskov Thanks for reviewing, and good point - I have now updated both get_required_pyrosetta_python_packages_for_release_package and get_required_pyrosetta_python_packages_for_testing to directly install numpy and removed the silent green mechanism (for reference, this is the same mechanism we currently use to skip T900-T921 tests on non-serialization builds, which are not fully silent since they print warnings when skipping due to missing packages). I agree, let's not exacerbate the issue by adding more skipping behavior. The T021 and T022 tests now pass without being skipped and will error out if numpy is missing.

13 changes: 13 additions & 0 deletions source/src/python/PyRosetta/src/test/T022_Pose_Alignment.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# :noTabs=true:
# (c) Copyright Rosetta Commons Member Institutions.
# (c) This file is part of the Rosetta software suite and is made available under license.
# (c) The Rosetta software is developed by the contributing members of the Rosetta Commons.
# (c) For more information, see http://www.rosettacommons.org.
# (c) Questions about this can be addressed to University of Washington CoMotion, email: license@uw.edu.

__author__ = "Jason C. Klima"

from utils.distributed import run_unittest

if __name__ == "__main__":
run_unittest("pyrosetta.tests.numeric.test_alignment", timeout=30)
2 changes: 0 additions & 2 deletions source/src/python/PyRosetta/src/test/T900_distributed.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,11 @@
def main(wait: bool, streaming: bool, timeout: int) -> None:
run_test_cases(
"pyrosetta.tests.bindings.init.test_init_files",
"pyrosetta.tests.bindings.core.test_pose",
"pyrosetta.tests.distributed.test_concurrency",
"pyrosetta.tests.distributed.test_dask",
"pyrosetta.tests.distributed.test_gil",
"pyrosetta.tests.distributed.test_smoke",
"pyrosetta.tests.distributed.test_viewer",
"pyrosetta.tests.numeric.test_alignment",
wait=wait,
streaming=streaming,
timeout=timeout,
Expand Down
22 changes: 19 additions & 3 deletions source/src/python/PyRosetta/src/test/utils/distributed.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@
Any,
Callable,
List,
NoReturn,
Optional,
TypeVar,
cast,
)
Expand Down Expand Up @@ -62,12 +60,22 @@ def has_pyrosetta_distributed_package_requirements() -> bool:
return False


def has_numpy_installed() -> bool:
"""Test if `numpy` is installed in the virtual environment."""
try:
import numpy
return True
except ImportError as ex:
print(f"{type(ex).__name__}: {ex}")
return False


def has_python_version(major: int, minor: int) -> bool:
"""Test if the Python version is greater than or equal to the provided `major` and `minor` values."""
return tuple(sys.version_info) >= (major, minor)


def exit_if_missing_pyrosetta_distributed_requirements(returncode: int = 0) -> Optional[NoReturn]:
def exit_if_missing_pyrosetta_distributed_requirements(returncode: int = 0) -> None:
"""Exit the Python process if the `pyrosetta.distributed` framework requirements are missing from the virtual environment."""

if not has_python_version(3, 6):
Expand All @@ -81,6 +89,14 @@ def exit_if_missing_pyrosetta_distributed_requirements(returncode: int = 0) -> O
sys.exit(returncode)


def exit_if_missing_numpy_requirement(returncode: int = 0) -> None:
"""Exit the Python process if `numpy` is missing from the virtual environment."""

if not has_numpy_installed():
print(f"The `numpy` package required for the tests is missing. Skipping the tests...")
sys.exit(returncode)


def print_environment_export() -> None:
"""Print the active virtual environment export."""

Expand Down
14 changes: 11 additions & 3 deletions tests/benchmark/tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,7 +514,11 @@ def get_required_pyrosetta_python_packages_for_testing(platform, conda=False, st
if static_versions:
packages = set_static_versions_for_python_packages(packages)

return get_packages_str_for_python_packages(packages) if 'serialization' in platform['extras'] and platform.get('python', DEFAULT_PYTHON_VERSION)[:2] != '2.' else ''
return (
get_packages_str_for_python_packages(packages)
if 'serialization' in platform['extras'] and platform.get('python', DEFAULT_PYTHON_VERSION)[:2] != '2.'
else get_packages_str_for_python_packages({'numpy': packages.get('numpy', DEFAULT_PACKAGE_VERSIONS_FOR_PYROSETTA_DISTRIBUTED['numpy'])})
)


def get_required_pyrosetta_python_packages_for_release_package(platform, conda=True, static_versions=True, distributed_packages=False):
Expand All @@ -525,7 +529,7 @@ def get_required_pyrosetta_python_packages_for_release_package(platform, conda=T

IMPORTANT: there should be no spaces between package name and version number

IMPORTANT: if PyRosetta build without serialization support or on Python-2 platform then we DO NOT REQUIRE any packages
IMPORTANT: if PyRosetta build without serialization support or on Python-2 platform then we require only `numpy`
'''

python_version = tuple( map(int, platform.get('python', DEFAULT_PYTHON_VERSION).split('.') ) )
Expand All @@ -542,7 +546,11 @@ def get_required_pyrosetta_python_packages_for_release_package(platform, conda=T
if static_versions:
packages = set_static_versions_for_python_packages(packages)

return get_packages_list_for_python_packages(packages) if 'serialization' in platform['extras'] and platform.get('python', DEFAULT_PYTHON_VERSION)[:2] != '2.' else []
return (
get_packages_list_for_python_packages(packages)
if 'serialization' in platform['extras'] and platform.get('python', DEFAULT_PYTHON_VERSION)[:2] != '2.'
else get_packages_list_for_python_packages({'numpy': packages.get('numpy', DEFAULT_PACKAGE_VERSIONS_FOR_PYROSETTA_DISTRIBUTED['numpy'])})
)


def build_pyrosetta(rosetta_dir, platform, jobs, config, mode='MinSizeRel', options='', conda=None, verbose=False, skip_compile=False, version=None):
Expand Down