Skip to content

Commit

Permalink
Merge pull request #12 from static-frame/11/np2
Browse files Browse the repository at this point in the history 
Numpy2 updates
  • Loading branch information
@flexatone
flexatone authored Jul 13, 2024
2 parents 821973d + c892266 commit 7f828a5
Show file tree
Hide file tree
Showing 7 changed files with 109 additions and 54 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
- {version: '3.10', requirements: 'requirements-test-3_11.txt'}
- {version: '3.11', requirements: 'requirements-test-3_11.txt'}
- {version: '3.12', requirements: 'requirements-test-3_11.txt'}
# - {version: '3.12', requirements: 'requirements-test-3_12-np2.txt'}
# - {version: '3.12', requirements: 'requirements-test-3_11-np2.txt'}

runs-on: ${{ matrix.os.name }}

Expand Down
8 changes: 8 additions & 0 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,14 @@ FrameFixtures support defining features unique to StaticFrame, such as specifyin



What is New in FramFixtures
------------------------------

1.1.0
............

Now compatible with NumPy 2.0.


Grammar
------------------------------
Expand Down
2 changes: 1 addition & 1 deletion frame_fixtures/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from frame_fixtures.core import Fixture as Fixture #pylint: disable=W0611
from frame_fixtures.core import parse as parse #pylint: disable=W0611

__version__ = '1.0.0'
__version__ = '1.1.0'


87 changes: 43 additions & 44 deletions frame_fixtures/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,33 +19,26 @@
from static_frame import TypeBlocks #pragma: no cover

TNDArrayAny = np.ndarray[tp.Any, tp.Any]
# TNDArrayBool = np.ndarray[tp.Any, np.dtype[np.bool_]]
# TNDArrayObject = np.ndarray[tp.Any, np.dtype[np.object_]]
# TNDArrayIntDefault = np.ndarray[tp.Any, np.dtype[np.int64]]

TDtypeAny = np.dtype[tp.Any]
# TDtypeObject = np.dtype[np.object_] #pragma: no cover



StrToType = tp.Dict[str, tp.Type[tp.Any]]
StrConstructorArg = tp.Union[str, tp.Tuple[str, ...]]
StrConstructorType = tp.Tuple[StrConstructorArg, ...]
StrConstructorsType = tp.Dict[str, StrConstructorType]
TStrToType = tp.Dict[str, tp.Type[tp.Any]]
TStrConstructorArg = tp.Union[str, tp.Tuple[str, ...]]
TStrConstructorType = tp.Tuple[TStrConstructorArg, ...]
TStrConstructorsType = tp.Dict[str, TStrConstructorType]


ConstructorOrConstructors = tp.Union[
TConstructorOrConstructors = tp.Union[
tp.Type['ContainerOperand'],
tp.Tuple[tp.Type['ContainerOperand'], ...]
]
DtypeSpecOrSpecs = tp.Union['TDtypeSpecifier', tp.Tuple['TDtypeSpecifier', ...]]
TDtypeSpecOrSpecs = tp.Union['TDtypeSpecifier', tp.Tuple['TDtypeSpecifier', ...]]

BuildElement = tp.Union[tp.Type['ContainerOperand'], 'TDtypeSpecifier']
BuildArg = tp.Union[BuildElement, tp.Tuple[BuildElement]]
TBuildType = tp.Tuple[BuildArg, ...]
TBuildElement = tp.Union[tp.Type['ContainerOperand'], 'TDtypeSpecifier']
TBuildArg = tp.Union[TBuildElement, tp.Tuple[TBuildElement]]
TBuildType = tp.Tuple[TBuildArg, ...]

ShapeType = tp.Tuple[int, int]
IndexTypes = tp.Union['Index', 'IndexHierarchy']
TShapeType = tp.Tuple[int, int]
TIndexTypes = tp.Union['Index', 'IndexHierarchy']


DTYPE_OBJECT = np.dtype(object)
Expand Down Expand Up @@ -108,7 +101,7 @@ def repeat_count(iter: tp.Iterable[T],
#-------------------------------------------------------------------------------
def get_str_to_constructor(
module_sf: tp.Optional[ModuleType],
) -> StrToType:
) -> TStrToType:
if module_sf is None:
import static_frame as sf
module_sf = sf
Expand Down Expand Up @@ -155,7 +148,7 @@ def get_str_to_constructor(
return ref


def get_str_to_dtype() -> StrToType:
def get_str_to_dtype() -> TStrToType:
'''Get a mapping from a string representation to a dtype specifier (not always a dtype)
'''
ref = {}
Expand Down Expand Up @@ -198,7 +191,7 @@ def get_str_to_dtype() -> StrToType:


class StrToTypeInterface:
'''Wrapper around StrToType mapping that provides informative key-errors.
'''Wrapper around TStrToType mapping that provides informative key-errors.
'''
def __init__(self,
module_sf: tp.Optional[ModuleType] = None,
Expand All @@ -223,13 +216,13 @@ def __getitem__(self, key: str) -> tp.Type[tp.Any]:
except KeyError:
raise FrameFixtureSyntaxError(f'{key!r} is not a valid specifier. Choose a constructor specifier ({", ".join(self._constructor_specifiers.keys())}) or a dtype specifier ({", ".join(self._dtype_specifiers.keys())})') from None


#-------------------------------------------------------------------------------
class SourceValues:
_SEED = 22
_COUNT = 0 # current count; this values is mutated

_INTS = EMPTY_ARRAY
_INTS = EMPTY_ARRAY # will be a numpy array of int64
_INTS_DTYPE = np.dtype(np.int64)
_CHARS = EMPTY_ARRAY
_BYTES = EMPTY_ARRAY

Expand Down Expand Up @@ -271,14 +264,14 @@ def update_primitives(cls, count: int = COUNT_INIT) -> None:
cls._COUNT = count * 2

if not len(cls._INTS):
values_int = np.arange(cls._COUNT, dtype=np.int64)
values_int = np.arange(cls._COUNT, dtype=cls._INTS_DTYPE)
cls.shuffle(values_int)
cls._INTS = values_int
cls._CHARS = cls._ints_to_chars(cls._INTS)
cls._BYTES = cls._CHARS.astype(DTYPE_BYTES)
else:
offset = len(cls._INTS)
values_ext = np.arange(offset, cls._COUNT, dtype=np.int64)
values_ext = np.arange(offset, cls._COUNT, dtype=cls._INTS_DTYPE)
cls.shuffle(values_ext)
cls._INTS = np.concatenate((cls._INTS, values_ext))
cls._CHARS = np.concatenate((
Expand All @@ -297,9 +290,15 @@ def dtype_to_element_iter(cls,
cls.update_primitives(count)

if dtype.kind == 'i': # int
def gen() -> tp.Iterator[tp.Any]:
for v in cls._INTS:
yield v * (-1 if v % 3 == 0 else 1)
if dtype == cls._INTS_DTYPE:
def gen() -> tp.Iterator[tp.Any]:
for v in cls._INTS:
yield v * (-1 if v % 3 == 0 else 1)
else:
def gen() -> tp.Iterator[tp.Any]:
for v in cls._INTS:
# NOTE: have to astype here with NumPy2, as fromiter rejects casting
yield v.astype(dtype) * (-1 if v % 3 == 0 else 1)

elif dtype.kind == 'u': # int unsigned
def gen() -> tp.Iterator[tp.Any]:
Expand Down Expand Up @@ -424,7 +423,7 @@ def dtype_to_array(cls,
@classmethod
@lru_cache(maxsize=128)
def dtype_spec_to_array(cls,
dtype_spec: DtypeSpecOrSpecs,
dtype_spec: TDtypeSpecOrSpecs,
count: int = COUNT_INIT,
shift: int = 0,
) -> TNDArrayAny:
Expand Down Expand Up @@ -484,7 +483,7 @@ class Grammar:

@classmethod
def validate(cls,
constructors: StrConstructorsType,
constructors: TStrConstructorsType,
) -> None:
if cls.SHAPE not in constructors:
raise FrameFixtureSyntaxError(f'missing required label: {cls.SHAPE}')
Expand All @@ -502,7 +501,7 @@ def validate(cls,
@classmethod
def dsl_to_str_constructors(cls,
dsl: str,
) -> StrConstructorsType:
) -> TStrConstructorsType:

body = ast.parse(dsl).body
if len(body) != 1:
Expand Down Expand Up @@ -530,12 +529,12 @@ def parts() -> tp.Iterator[ast.Call]:
else:
raise FrameFixtureSyntaxError(f'no support for token {root.value}')

constructors: StrConstructorsType = {}
constructors: TStrConstructorsType = {}

for p in parts(): # each is a Call object
key = p.func.id #type: ignore

args: tp.List[StrConstructorArg] = []
args: tp.List[TStrConstructorArg] = []
for arg in p.args:
if isinstance(arg, ast.Tuple):
args.append(tuple(sub.id for sub in arg.elts)) #type: ignore
Expand Down Expand Up @@ -624,10 +623,10 @@ class Fixture:
@staticmethod
def _build_index(
count: int,
constructor: ConstructorOrConstructors,
dtype_spec: DtypeSpecOrSpecs,
constructor: TConstructorOrConstructors,
dtype_spec: TDtypeSpecOrSpecs,
str_to_type: StrToTypeInterface,
) -> IndexTypes:
) -> TIndexTypes:

constructor_is_tuple = isinstance(constructor, tuple)

Expand Down Expand Up @@ -668,8 +667,8 @@ def labels() -> tp.Iterator[tp.Tuple[tp.Any, ...]]:

@staticmethod
def _build_type_blocks(
shape: ShapeType,
dtype_specs: tp.Sequence[DtypeSpecOrSpecs],
shape: TShapeType,
dtype_specs: tp.Sequence[TDtypeSpecOrSpecs],
str_to_type: StrToTypeInterface,
) -> 'TypeBlocks':

Expand All @@ -691,12 +690,12 @@ def gen() -> tp.Iterator[TNDArrayAny]:
#---------------------------------------------------------------------------
@staticmethod
def _str_to_build(
constructor: StrConstructorType, # typle of elements or tuples
constructor: TStrConstructorType, # typle of elements or tuples
str_to_type: StrToTypeInterface,
) -> TBuildType:
'''Convert strings to types or SF classes.
'''
def gen() -> tp.Iterator[BuildArg]:
def gen() -> tp.Iterator[TBuildArg]:
for v in constructor:
if isinstance(v, tuple):
yield tuple(str_to_type[part] for part in v) # type: ignore
Expand All @@ -706,14 +705,14 @@ def gen() -> tp.Iterator[BuildArg]:

@classmethod
def _to_containers(cls,
constructors: StrConstructorsType,
constructors: TStrConstructorsType,
str_to_type: StrToTypeInterface,
) -> tp.Tuple['TypeBlocks',
tp.Optional[IndexTypes],
tp.Optional[IndexTypes],
tp.Optional[TIndexTypes],
tp.Optional[TIndexTypes],
]:

shape: ShapeType = tp.cast(ShapeType, constructors['s'])
shape: TShapeType = tp.cast(TShapeType, constructors['s'])

if Grammar.VALUES not in constructors:
values_constructor = ('float',)
Expand Down
51 changes: 51 additions & 0 deletions frame_fixtures/test/test_core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,3 +221,54 @@ def test_index_dt64_a() -> None:
for u in DT64_UNITS:
f = Fixture.parse(f's(3,2)|i(I{u}, dt{u})')
assert f.index.dtype == np.dtype(f'datetime64[{u}]') # type: ignore


#-------------------------------------------------------------------------------
def test_i8() -> None:
dt64 = np.datetime64
f1 = Fixture.parse('s(2,4)|v(int8,str)|i(ID,dtD)|c(ID,dtD)').rename('x', index='y', columns='z')
assert(f1.to_pairs() ==
((dt64('2065-01-17'), ((dt64('2065-01-17'), np.int8(47)), (dt64('1979-12-28'), np.int8(-61)))), (dt64('1979-12-28'), ((dt64('2065-01-17'), np.str_('zaji')), (dt64('1979-12-28'), np.str_('zJnC')))), (dt64('2219-12-23'), ((dt64('2065-01-17'), np.int8(-64)), (dt64('1979-12-28'), np.int8(-91)))), (dt64('2052-09-12'), ((dt64('2065-01-17'), np.str_('z2Oo')), (dt64('1979-12-28'), np.str_('z5l6'))))))
# <Frame: x>
# <IndexDate: z> 2065-01-17 1979-12-28 2219-12-23 2052-09-12 <datetime64[D]>
# <IndexDate: y>
# 2065-01-17 47 zaji -64 z2Oo
# 1979-12-28 -61 zJnC -91 z5l6
# <datetime64[D]> <int8> <<U4> <int8> <<U4>

def test_i16() -> None:
f1 = Fixture.parse('s(2,4)|v(int16,str)')
assert (f1.to_pairs() ==
((np.int64(0), ((np.int64(0), np.int16(-22481)), (np.int64(1), np.int16(27331)))), (np.int64(1), ((np.int64(0), np.str_('zaji')), (np.int64(1), np.str_('zJnC')))), (np.int64(2), ((np.int64(0), np.int16(-3648)), (np.int64(1), np.int16(25765)))), (np.int64(3), ((np.int64(0), np.str_('z2Oo')), (np.int64(1), np.str_('z5l6'))))))

def test_i32() -> None:
f1 = Fixture.parse('s(2,4)|v(int32,str)')
assert (f1.to_pairs() ==
((np.int64(0), ((np.int64(0), np.int32(-88017)), (np.int64(1), np.int32(92867)))), (np.int64(1), ((np.int64(0), np.str_('zaji')), (np.int64(1), np.str_('zJnC')))), (np.int64(2), ((np.int64(0), np.int32(-3648)), (np.int64(1), np.int32(91301)))), (np.int64(3), ((np.int64(0), np.str_('z2Oo')), (np.int64(1), np.str_('z5l6'))))))

def test_u8() -> None:
f1 = Fixture.parse('s(4,10)|v(uint8,uint8)')
assert (f1.to_pairs() ==
((np.int64(0), ((np.int64(0), np.uint8(146)), (np.int64(1), np.uint8(61)), (np.int64(2), np.uint8(67)), (np.int64(3), np.uint8(56)))), (np.int64(1), ((np.int64(0), np.uint8(150)), (np.int64(1), np.uint8(250)), (np.int64(2), np.uint8(100)), (np.int64(3), np.uint8(254)))), (np.int64(2), ((np.int64(0), np.uint8(94)), (np.int64(1), np.uint8(182)), (np.int64(2), np.uint8(201)), (np.int64(3), np.uint8(87)))), (np.int64(3), ((np.int64(0), np.uint8(101)), (np.int64(1), np.uint8(1)), (np.int64(2), np.uint8(96)), (np.int64(3), np.uint8(212)))), (np.int64(4), ((np.int64(0), np.uint8(245)), (np.int64(1), np.uint8(246)), (np.int64(2), np.uint8(204)), (np.int64(3), np.uint8(140)))), (np.int64(5), ((np.int64(0), np.uint8(67)), (np.int64(1), np.uint8(56)), (np.int64(2), np.uint8(73)), (np.int64(3), np.uint8(245)))), (np.int64(6), ((np.int64(0), np.uint8(246)), (np.int64(1), np.uint8(204)), (np.int64(2), np.uint8(140)), (np.int64(3), np.uint8(69)))), (np.int64(7), ((np.int64(0), np.uint8(69)), (np.int64(1), np.uint8(129)), (np.int64(2), np.uint8(82)), (np.int64(3), np.uint8(115)))), (np.int64(8), ((np.int64(0), np.uint8(202)), (np.int64(1), np.uint8(228)), (np.int64(2), np.uint8(94)), (np.int64(3), np.uint8(190)))), (np.int64(9), ((np.int64(0), np.uint8(179)), (np.int64(1), np.uint8(147)), (np.int64(2), np.uint8(142)), (np.int64(3), np.uint8(14))))))

def test_u16() -> None:
f1 = Fixture.parse('s(4,10)|v(uint16,uint16)')
assert (f1.to_pairs() ==
((np.int64(0), ((np.int64(0), np.uint16(57746)), (np.int64(1), np.uint16(3389)), (np.int64(2), np.uint16(62787)), (np.int64(3), np.uint16(35128)))), (np.int64(1), ((np.int64(0), np.uint16(55958)), (np.int64(1), np.uint16(31994)), (np.int64(2), np.uint16(31332)), (np.int64(3), np.uint16(58622)))), (np.int64(2), ((np.int64(0), np.uint16(57438)), (np.int64(1), np.uint16(61878)), (np.int64(2), np.uint16(63177)), (np.int64(3), np.uint16(32343)))), (np.int64(3), ((np.int64(0), np.uint16(15717)), (np.int64(1), np.uint16(13057)), (np.int64(2), np.uint16(53344)), (np.int64(3), np.uint16(63188)))), (np.int64(4), ((np.int64(0), np.uint16(52469)), (np.int64(1), np.uint16(1526)), (np.int64(2), np.uint16(39628)), (np.int64(3), np.uint16(40588)))), (np.int64(5), ((np.int64(0), np.uint16(62787)), (np.int64(1), np.uint16(35128)), (np.int64(2), np.uint16(47433)), (np.int64(3), np.uint16(52469)))), (np.int64(6), ((np.int64(0), np.uint16(1526)), (np.int64(1), np.uint16(39628)), (np.int64(2), np.uint16(40588)), (np.int64(3), np.uint16(31045)))), (np.int64(7), ((np.int64(0), np.uint16(31045)), (np.int64(1), np.uint16(63361)), (np.int64(2), np.uint16(35154)), (np.int64(3), np.uint16(29043)))), (np.int64(8), ((np.int64(0), np.uint16(4042)), (np.int64(1), np.uint16(49636)), (np.int64(2), np.uint16(12894)), (np.int64(3), np.uint16(64446)))), (np.int64(9), ((np.int64(0), np.uint16(64691)), (np.int64(1), np.uint16(9619)), (np.int64(2), np.uint16(910)), (np.int64(3), np.uint16(8206))))))

def test_u32() -> None:
f1 = Fixture.parse('s(4,10)|v(uint32,uint32)')
assert (f1.to_pairs() ==
((np.int64(0), ((np.int64(0), np.uint32(188818)), (np.int64(1), np.uint32(68925)), (np.int64(2), np.uint32(62787)), (np.int64(3), np.uint32(166200)))), (np.int64(1), ((np.int64(0), np.uint32(187030)), (np.int64(1), np.uint32(163066)), (np.int64(2), np.uint32(31332)), (np.int64(3), np.uint32(189694)))), (np.int64(2), ((np.int64(0), np.uint32(188510)), (np.int64(1), np.uint32(61878)), (np.int64(2), np.uint32(194249)), (np.int64(3), np.uint32(32343)))), (np.int64(3), ((np.int64(0), np.uint32(15717)), (np.int64(1), np.uint32(144129)), (np.int64(2), np.uint32(118880)), (np.int64(3), np.uint32(128724)))), (np.int64(4), ((np.int64(0), np.uint32(183541)), (np.int64(1), np.uint32(67062)), (np.int64(2), np.uint32(170700)), (np.int64(3), np.uint32(40588)))), (np.int64(5), ((np.int64(0), np.uint32(62787)), (np.int64(1), np.uint32(166200)), (np.int64(2), np.uint32(178505)), (np.int64(3), np.uint32(183541)))), (np.int64(6), ((np.int64(0), np.uint32(67062)), (np.int64(1), np.uint32(170700)), (np.int64(2), np.uint32(40588)), (np.int64(3), np.uint32(162117)))), (np.int64(7), ((np.int64(0), np.uint32(162117)), (np.int64(1), np.uint32(128897)), (np.int64(2), np.uint32(100690)), (np.int64(3), np.uint32(29043)))), (np.int64(8), ((np.int64(0), np.uint32(69578)), (np.int64(1), np.uint32(180708)), (np.int64(2), np.uint32(143966)), (np.int64(3), np.uint32(129982)))), (np.int64(9), ((np.int64(0), np.uint32(195763)), (np.int64(1), np.uint32(9619)), (np.int64(2), np.uint32(910)), (np.int64(3), np.uint32(73742))))))

def test_f16() -> None:
f1 = Fixture.parse('s(4,3)|v(float16,float16)')
assert (f1.to_pairs() ==
((np.int64(0), ((np.int64(0), np.float16(1930.0)), (np.int64(1), np.float16(-1760.0)), (np.int64(2), np.float16(1857.0)), (np.int64(3), np.float16(1699.0)))), (np.int64(1), ((np.int64(0), np.float16(-611.0)), (np.int64(1), np.float16(3244.0)), (np.int64(2), np.float16(-823.0)), (np.int64(3), np.float16(114.56)))), (np.int64(2), ((np.int64(0), np.float16(694.5)), (np.int64(1), np.float16(-72.94)), (np.int64(2), np.float16(1826.0)), (np.int64(3), np.float16(604.0))))))

def test_f32() -> None:
f1 = Fixture.parse('s(4,3)|v(float32,float32)')
assert (f1.to_pairs() ==
((np.int64(0), ((np.int64(0), np.float32(1930.4)), (np.int64(1), np.float32(-1760.34)), (np.int64(2), np.float32(1857.34)), (np.int64(3), np.float32(1699.34)))), (np.int64(1), ((np.int64(0), np.float32(-610.8)), (np.int64(1), np.float32(3243.94)), (np.int64(2), np.float32(-823.14)), (np.int64(3), np.float32(114.58)))), (np.int64(2), ((np.int64(0), np.float32(694.3)), (np.int64(1), np.float32(-72.96)), (np.int64(2), np.float32(1826.02)), (np.int64(3), np.float32(604.1))))))


5 changes: 5 additions & 0 deletions requirements-test-3_11-np2.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
invoke==2.2.0
numpy==2.0.0
pytest==8.2.2
pytest-cov==4.0.0
static-frame==2.10.0
8 changes: 0 additions & 8 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,6 @@
from os import path
import typing as tp

# https://packaging.python.org/distributing/
# to deploy:
# pip install wheel, twine
# python setup.py sdist
# python setup.py bdist_wheel
# twine upload dist/*
# rm -r build; rm -r dist; rm -r *.egg-info

ROOT_DIR_FP = path.abspath(path.dirname(__file__))

def get_long_description() -> str:
Expand Down

0 comments on commit 7f828a5

Please sign in to comment.