ai-content-maker/.venv/Lib/site-packages/numba/tests/test_hashing.py

500 lines
18 KiB
Python
Raw Normal View History

2024-05-03 04:18:51 +03:00
# -*- coding: utf-8 -*-
"""
Test hashing of various supported types.
"""
import unittest
import os
import sys
import subprocess
from collections import defaultdict
from textwrap import dedent
import numpy as np
from numba import jit, config, typed, typeof
from numba.core import types, utils
import unittest
from numba.tests.support import (TestCase, skip_unless_py10_or_later,
run_in_subprocess)
from numba.cpython.unicode import compile_time_get_string_data
from numba.cpython import hashing
def hash_usecase(x):
return hash(x)
class TestHashingSetup(TestCase):
def test_warn_on_fnv(self):
# FNV hash alg variant is not supported, check Numba warns
work = """
import sys
import warnings
from collections import namedtuple
# hash_info is a StructSequence, mock as a named tuple
fields = ["width", "modulus", "inf", "nan", "imag", "algorithm",
"hash_bits", "seed_bits", "cutoff"]
hinfo = sys.hash_info
FAKE_HASHINFO = namedtuple('FAKE_HASHINFO', fields)
fd = dict()
for f in fields:
fd[f] = getattr(hinfo, f)
fd['algorithm'] = 'fnv'
fake_hashinfo = FAKE_HASHINFO(**fd)
# replace the hashinfo with the fnv version
sys.hash_info = fake_hashinfo
with warnings.catch_warnings(record=True) as warns:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
from numba import njit
@njit
def foo():
hash(1)
foo()
assert len(warns) > 0
expect = "FNV hashing is not implemented in Numba. See PEP 456"
for w in warns:
if expect in str(w.message):
break
else:
raise RuntimeError("Expected warning not found")
"""
subprocess.check_call([sys.executable, '-c', dedent(work)])
class TestHashAlgs(TestCase):
# This tests Numba hashing replication against cPython "gold", i.e. the
# actual hash values for given inputs, algs and PYTHONHASHSEEDs
# Test adapted from:
# https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Lib/test/test_hash.py#L197-L264
# and
# https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Lib/test/test_hash.py#L174-L189
# 32bit little, 64bit little, 32bit big, 64bit big
known_hashes = {
'djba33x': [ # only used for small strings
# seed 0, 'abc'
[193485960, 193485960, 193485960, 193485960],
# seed 42, 'abc'
[-678966196, 573763426263223372, -820489388, -4282905804826039665],
],
'siphash13': [
# NOTE: PyUCS2 layout depends on endianness
# seed 0, 'abc'
[69611762, -4594863902769663758, 69611762, -4594863902769663758],
# seed 42, 'abc'
[-975800855, 3869580338025362921, -975800855, 3869580338025362921],
# seed 42, 'abcdefghijk'
[-595844228, 7764564197781545852, -595844228, 7764564197781545852],
# seed 0, 'äú∑ℇ'
[-1093288643, -2810468059467891395, -1041341092, 4925090034378237276],
# seed 42, 'äú∑ℇ'
[-585999602, -2845126246016066802, -817336969, -2219421378907968137],
],
'siphash24': [
# NOTE: PyUCS2 layout depends on endianness
# seed 0, 'abc'
[1198583518, 4596069200710135518, 1198583518, 4596069200710135518],
# seed 42, 'abc'
[273876886, -4501618152524544106, 273876886, -4501618152524544106],
# seed 42, 'abcdefghijk'
[-1745215313, 4436719588892876975, -1745215313, 4436719588892876975],
# seed 0, 'äú∑ℇ'
[493570806, 5749986484189612790, -1006381564, -5915111450199468540],
# seed 42, 'äú∑ℇ'
[-1677110816, -2947981342227738144, -1860207793, -4296699217652516017],
],
}
def get_expected_hash(self, position, length):
if length < sys.hash_info.cutoff:
algorithm = "djba33x"
else:
algorithm = sys.hash_info.algorithm
IS_64BIT = not config.IS_32BITS
if sys.byteorder == 'little':
platform = 1 if IS_64BIT else 0
else:
assert(sys.byteorder == 'big')
platform = 3 if IS_64BIT else 2
return self.known_hashes[algorithm][position][platform]
def get_hash_command(self, repr_):
return 'print(hash(eval(%a)))' % repr_
def get_hash(self, repr_, seed=None):
env = os.environ.copy()
if seed is not None:
env['PYTHONHASHSEED'] = str(seed)
else:
env.pop('PYTHONHASHSEED', None)
out, _ = run_in_subprocess(code=self.get_hash_command(repr_),
env=env)
stdout = out.decode().strip()
return int(stdout)
def test_against_cpython_gold(self):
args = (('abc', 0, 0), ('abc', 42, 1), ('abcdefghijk', 42, 2),
('äú∑ℇ', 0, 3), ('äú∑ℇ', 42, 4),)
for input_str, seed, position in args:
with self.subTest(input_str=input_str, seed=seed):
got = self.get_hash(repr(input_str), seed=seed)
expected = self.get_expected_hash(position, len(input_str))
self.assertEqual(got, expected)
class BaseTest(TestCase):
def setUp(self):
self.cfunc = jit(nopython=True)(hash_usecase)
def check_hash_values(self, values):
cfunc = self.cfunc
for val in list(values):
nb_hash = cfunc(val)
self.assertIsInstance(nb_hash, int)
try:
self.assertEqual(nb_hash, hash(val))
except AssertionError as e:
print("val, nb_hash, hash(val)")
print(val, nb_hash, hash(val))
print("abs(val), hashing._PyHASH_MODULUS - 1")
print(abs(val), hashing._PyHASH_MODULUS - 1)
raise e
def int_samples(self, typ=np.int64):
for start in (0, -50, 60000, 1 << 32):
info = np.iinfo(typ)
if not info.min <= start <= info.max:
continue
n = 100
yield range(start, start + n)
yield range(start, start + 100 * n, 100)
yield range(start, start + 128 * n, 128)
yield [-1]
def float_samples(self, typ):
info = np.finfo(typ)
for start in (0, 10, info.max ** 0.5, info.max / 1000.0):
n = 100
min_step = max(info.tiny, start * info.resolution)
for step in (1.2, min_step ** 0.5, min_step):
if step < min_step:
continue
a = np.linspace(start, start + n * step, n)
a = a.astype(typ)
yield a
yield -a
yield a + a.mean()
# Infs, nans, zeros, magic -1
a = [0.0, 0.5, -0.0, -1.0, float('inf'), -float('inf'),]
# Python 3.10 has a hash for nan based on the pointer to the PyObject
# containing the nan, skip this input and use explicit test instead.
if utils.PYVERSION < (3, 10):
a.append(float('nan'))
yield typ(a)
def complex_samples(self, typ, float_ty):
for real in self.float_samples(float_ty):
for imag in self.float_samples(float_ty):
# Ensure equal sizes
real = real[:len(imag)]
imag = imag[:len(real)]
a = real + typ(1j) * imag
# Python 3.10 has a hash for nan based on the pointer to the
# PyObject containing the nan, skip input that ends up as nan
if utils.PYVERSION >= (3, 10):
if not np.any(np.isnan(a)):
yield a
else:
yield a
class TestNumberHashing(BaseTest):
"""
Test hashing of number types.
"""
def check_floats(self, typ):
for a in self.float_samples(typ):
self.assertEqual(a.dtype, np.dtype(typ))
self.check_hash_values(a)
def check_complex(self, typ, float_ty):
for a in self.complex_samples(typ, float_ty):
self.assertEqual(a.dtype, np.dtype(typ))
self.check_hash_values(a)
def test_floats(self):
self.check_floats(np.float32)
self.check_floats(np.float64)
def test_complex(self):
self.check_complex(np.complex64, np.float32)
self.check_complex(np.complex128, np.float64)
def test_bool(self):
self.check_hash_values([False, True])
def test_ints(self):
minmax = []
for ty in [np.int8, np.uint8, np.int16, np.uint16,
np.int32, np.uint32, np.int64, np.uint64]:
for a in self.int_samples(ty):
self.check_hash_values(a)
info = np.iinfo(ty)
# check hash(-1) = -2
# check hash(0) = 0
self.check_hash_values([ty(-1)])
self.check_hash_values([ty(0)])
signed = 'uint' not in str(ty)
# check bit shifting patterns from min through to max
sz = ty().itemsize
for x in [info.min, info.max]:
shifts = 8 * sz
# x is a python int, do shifts etc as a python int and init
# numpy type from that to avoid numpy type rules
y = x
for i in range(shifts):
twiddle1 = 0xaaaaaaaaaaaaaaaa
twiddle2 = 0x5555555555555555
vals = [y]
for tw in [twiddle1, twiddle2]:
val = y & twiddle1
if val < sys.maxsize:
vals.append(val)
for v in vals:
self.check_hash_values([ty(v)])
if signed: # try the same with flipped signs
# negated signed INT_MIN will overflow
for v in vals:
if v != info.min:
self.check_hash_values([ty(-v)])
if x == 0: # unsigned min is 0, shift up
y = (y | 1) << 1
else: # everything else shift down
y = y >> 1
# these straddle the branch between returning the int as the hash and
# doing the PyLong hash alg
self.check_hash_values([np.int64(0x1ffffffffffffffe)])
self.check_hash_values([np.int64(0x1fffffffffffffff)])
self.check_hash_values([np.uint64(0x1ffffffffffffffe)])
self.check_hash_values([np.uint64(0x1fffffffffffffff)])
# check some values near sys int mins
self.check_hash_values([np.int64(-0x7fffffffffffffff)])
self.check_hash_values([np.int64(-0x7ffffffffffffff6)])
self.check_hash_values([np.int64(-0x7fffffffffffff9c)])
self.check_hash_values([np.int32(-0x7fffffff)])
self.check_hash_values([np.int32(-0x7ffffff6)])
self.check_hash_values([np.int32(-0x7fffff9c)])
@skip_unless_py10_or_later
def test_py310_nan_hash(self):
# On Python 3.10+ nan's hash to a value which is based on the pointer to
# the PyObject containing the nan. Numba cannot replicate as there's no
# object, it instead produces equivalent behaviour, i.e. hashes to
# something "unique".
# Run 10 hashes, make sure that the "uniqueness" is sufficient that
# there's more than one hash value. Not much more can be done!
x = [float('nan') for i in range(10)]
out = set([self.cfunc(z) for z in x])
self.assertGreater(len(out), 1)
class TestTupleHashing(BaseTest):
"""
Test hashing of tuples.
"""
def check_tuples(self, value_generator, split):
for values in value_generator:
tuples = [split(a) for a in values]
self.check_hash_values(tuples)
def test_homogeneous_tuples(self):
typ = np.uint64
def split2(i):
"""
Split i's bits into 2 integers.
"""
i = typ(i)
return (i & typ(0x5555555555555555),
i & typ(0xaaaaaaaaaaaaaaaa),
)
def split3(i):
"""
Split i's bits into 3 integers.
"""
i = typ(i)
return (i & typ(0x2492492492492492),
i & typ(0x4924924924924924),
i & typ(0x9249249249249249),
)
self.check_tuples(self.int_samples(), split2)
self.check_tuples(self.int_samples(), split3)
# Check exact. Sample values from:
# https://github.com/python/cpython/blob/b738237d6792acba85b1f6e6c8993a812c7fd815/Lib/test/test_tuple.py#L80-L93
# Untypable empty tuples are replaced with (7,).
self.check_hash_values([(7,), (0,), (0, 0), (0.5,),
(0.5, (7,), (-2, 3, (4, 6)))])
def test_heterogeneous_tuples(self):
modulo = 2**63
def split(i):
a = i & 0x5555555555555555
b = (i & 0xaaaaaaaa) ^ ((i >> 32) & 0xaaaaaaaa)
return np.int64(a), np.float64(b * 0.0001)
self.check_tuples(self.int_samples(), split)
class TestUnicodeHashing(BaseTest):
def test_basic_unicode(self):
kind1_string = "abcdefghijklmnopqrstuvwxyz"
for i in range(len(kind1_string)):
self.check_hash_values([kind1_string[:i]])
sep = ""
kind2_string = sep.join(list(kind1_string))
for i in range(len(kind2_string)):
self.check_hash_values([kind2_string[:i]])
sep = "🐍⚡"
kind4_string = sep.join(list(kind1_string))
for i in range(len(kind4_string)):
self.check_hash_values([kind4_string[:i]])
empty_string = ""
self.check_hash_values(empty_string)
def test_hash_passthrough(self):
# no `hash` call made, this just checks that `._hash` is correctly
# passed through from an already existing string
kind1_string = "abcdefghijklmnopqrstuvwxyz"
@jit(nopython=True)
def fn(x):
return x._hash
hash_value = compile_time_get_string_data(kind1_string)[-1]
self.assertTrue(hash_value != -1)
self.assertEqual(fn(kind1_string), hash_value)
def test_hash_passthrough_call(self):
# check `x._hash` and hash(x) are the same
kind1_string = "abcdefghijklmnopqrstuvwxyz"
@jit(nopython=True)
def fn(x):
return x._hash, hash(x)
hash_value = compile_time_get_string_data(kind1_string)[-1]
self.assertTrue(hash_value != -1)
self.assertEqual(fn(kind1_string), (hash_value, hash_value))
@unittest.skip("Needs hash computation at const unpickling time")
def test_hash_literal(self):
# a strconst always seem to have an associated hash value so the hash
# member of the returned value should contain the correct hash
@jit(nopython=True)
def fn():
x = "abcdefghijklmnopqrstuvwxyz"
return x
val = fn()
tmp = hash("abcdefghijklmnopqrstuvwxyz")
self.assertEqual(tmp, (compile_time_get_string_data(val)[-1]))
def test_hash_on_str_creation(self):
# In cPython some? new strings do not have a cached hash until hash() is
# called
def impl(do_hash):
const1 = "aaaa"
const2 = "眼眼眼眼"
new = const1 + const2
if do_hash:
hash(new)
return new
jitted = jit(nopython=True)(impl)
# do not compute the hash, cPython will have no cached hash, but Numba
# will
compute_hash = False
expected = impl(compute_hash)
got = jitted(compute_hash)
a = (compile_time_get_string_data(expected))
b = (compile_time_get_string_data(got))
self.assertEqual(a[:-1], b[:-1])
self.assertTrue(a[-1] != b[-1])
# now with compute hash enabled, cPython will have a cached hash as will
# Numba
compute_hash = True
expected = impl(compute_hash)
got = jitted(compute_hash)
a = (compile_time_get_string_data(expected))
b = (compile_time_get_string_data(got))
self.assertEqual(a, b)
class TestUnhashable(TestCase):
# Tests that unhashable types behave correctly and raise a TypeError at
# runtime.
def test_hash_unhashable(self):
unhashables = (typed.Dict().empty(types.int64, types.int64),
typed.List().empty_list(types.int64),
np.ones(4))
cfunc = jit(nopython=True)(hash_usecase)
for ty in unhashables:
with self.assertRaises(TypeError) as raises:
cfunc(ty)
expected = f"unhashable type: '{str(typeof(ty))}'"
self.assertIn(expected, str(raises.exception))
def test_no_generic_hash(self):
# In CPython, if there's no attr `__hash__` on an object, a hash of the
# object's pointer is returned (see: _Py_HashPointer in the CPython
# source). Numba has no access to such objects and can't create them
# either, so it catches this case and raises an exception.
@jit(nopython=True)
def foo():
hash(np.cos)
with self.assertRaises(TypeError) as raises:
foo()
expected = ("No __hash__ is defined for object ")
self.assertIn(expected, str(raises.exception))
if __name__ == "__main__":
unittest.main()