diff -pruN 2.9.2-1/.github/workflows/build_wheels.yml 3.6.1-1/.github/workflows/build_wheels.yml --- 2.9.2-1/.github/workflows/build_wheels.yml 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/.github/workflows/build_wheels.yml 2025-08-12 08:35:42.000000000 +0000 @@ -8,56 +8,62 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-11] + # macos-13 is an intel runner, macos-14 is apple silicon + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v1 + if: runner.os == 'Linux' && runner.arch == 'X64' + uses: docker/setup-qemu-action@v3.2.0 with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.16.1 + uses: pypa/cibuildwheel@v2.22.0 # to supply options, put them in 'env', like: env: # configure cibuildwheel to build native archs ('auto'), and some # emulated ones - CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x - CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_ARCHS_LINUX: ${{ runner.arch == 'X64' && 'auto ppc64le s390x' || 'auto' }} CIBW_TEST_COMMAND: python -c "import bitarray; assert bitarray.test().wasSuccessful()" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-sdist path: dist/*.tar.gz upload_pypi: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # upload to PyPI on every tag if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: - name: artifact + pattern: cibw-* path: dist + merge-multiple: true - - uses: pypa/gh-action-pypi-publish@v1.4.2 + - uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff -pruN 2.9.2-1/CHANGE_LOG 3.6.1-1/CHANGE_LOG --- 2.9.2-1/CHANGE_LOG 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/CHANGE_LOG 2025-08-12 08:35:42.000000000 +0000 @@ -1,3 +1,163 @@ +2025-08-12 3.6.1: +------------------- + * add development files for statistical tests in `devel/random/` + * optimize `util.sum_indices()` + * fix RecursionError in `util.random_k()`, see #239 + * add `devel/test_sum_indices.py` + + +2025-07-29 3.6.0: +------------------- + * add `util.random_k()`, see #237 + * add `util.sum_indices()` + * optimize `util.xor_indices()` + * move development files from `examples/` to `devel/` + + +2025-07-21 3.5.2: +------------------- + * optimize `util.random_p()` by also using bitwise AND in final step + * fix DeprecationWarning regarding `u` type code + * add [verification tests](../devel/test_random.py) for internals + of `util.random_p()` + + +2025-07-14 3.5.1: +------------------- + * optimize `util.random_p()` for `n < 100` + * add [Random Bitarrays](random_p.rst) documentation + * add [statistical tests for random functions](../devel/test_random.py) + + +2025-07-06 3.5.0: +------------------- + * add `util.random_p()` + * improve sparse compression testing + + +2025-06-23 3.4.3: +------------------- + * minor updates to documentation + * C-level: + - simplify and speedup `extend_unicode01()` + - customize `resize_lite()` - avoid unused code + - use `PyTypeObject` for bitarray type object in `_util.c` to + be consistent with `_bitarray.c` + - add and improve comments to implementation of sparse bitarray + compression + - simplify `sc_count()` + + +2025-05-21 3.4.2: +------------------- + * extend documentation of + [compression of sparse bitarrays](sparse_compression.rst) + * `util.sc_decode()` and `util.vl_decode()` now raise `StopIteration` + instead of `ValueError` when unexpected end of stream is encountered + * add debug mode tests for `read_n()`, `write_n()` and `count_from_word()` + + +2025-05-15 3.4.1: +------------------- + * add `pyproject.toml`, see #233 + * implement `bits2bytes()` in C + * optimize `delslice()` when `step` is larger than about 5 + * consistently name `*_span()` and `*_range()` in C for + invert, set and count + * organize and add tests (including debug mode tests for `zlw()`) + + +2025-05-06 3.4.0: +------------------- + * remove `.endian()` method in favor of data descriptor `.endian` + * allow bitarray initializer `bytes` or `bytearray` to set buffer directly + * allow calling `.extend()` with `bytes` object (although the only + valid bytes are 0x00 and 0x01) + * add `util.byteswap()` + * add `util.correspond_all()` + * fix `.reverse()` for imported buffer + * drop Python 3.5 support + * add tests + + +2025-05-02 3.3.2: +------------------- + * fix off-by-one-error in check for length of count argument + in `util.canonical_decode()` + * simplify `util.int2ba()` + * add tests + * add [masked indexing example](../examples/masked.py) + * add [tricks example](../devel/tricks.py) + + +2025-04-04 3.3.1: +------------------- + * remove `License` classifier in favor of a SPDX license expression, #231 + * reorganize and cleanup many tests + + +2025-03-30 3.3.0: +------------------- + * add optional `group` and `sep` arguments' to `.to01()`, #230 - + as well as `util.ba2hex()` and `util.ba2base()` + * ignore whitespace in `util.base2ba()` and `util.hex2ba()` + * check for embedded nul characters when extending (and initializing) + bitarray from string + * improve testing + * add [double precision floating point number example](../examples/double.py) + + +2025-03-19 3.2.0: +------------------- + * add `util.xor_indices()`, #229 + * add [Hamming code example](../examples/hamming.py) + + +2025-03-06 3.1.1: +------------------- + * updated `pythoncapi_compat.h` for pypy3.11 support, see #227 + * use `__builtin_parityll()` when available in `util.parity()` + * add `parity_64()` to header + * simplify some tests + * add [LFSR example](../examples/lfsr.py) + + +2025-02-19 3.1.0: +------------------- + * allow mask assignment to bitarrays, see #225 + * add missing masked operations to pyi-file + * refactor `resize()` and avoid overallocation when downsizing buffer + * update `build_wheels.yml` + * fix some typos + * minor simplifications + * rename `growth/` example to `resize/` and add tests for `resize()` + * update gene example + * add comments + + +2024-10-15 3.0.0: +------------------- + * see [Bitarray 3 transition](bitarray3.rst) + * remove Python 2.7 support + * `.decode()` now returns iterator (equivalent to past `.iterdecode()`) + * `.search()` now returns iterator (equivalent to past `.itersearch()`) + * remove `.iterdecode()` and `.itersearch()` + * remove `util.rindex()`, use `.index(..., right=1)` instead, + deprecated since 2.9 + * remove `util.make_endian()`, use `bitarray(..., endian=...)` instead, + deprecated since 2.9 + * remove hackish support for `bitarray()` handling unpickling, + see detailed explaination in #207 - closes #206 + + +2024-10-10 2.9.3: +------------------- + * add official Python 3.13 support + * update cibuildwheel to 2.21.3 + * minor simplifications + * fix some typos + + 2024-01-01 2.9.2: ------------------- * optimize initialization from strings by not constantly resizing buffer @@ -45,7 +205,7 @@ 2023-12-04 2.8.4: ------------------- * simplify `copy_n()` (remove special cases), see #d2d6fd53 - * add [word shift example C program](../examples/shift_r8.c), + * add [word shift example C program](../devel/shift_r8.c), and simplify `shift_r8()` * improve documentation and testing * add [roadmap](https://github.com/ilanschnell/bitarray#roadmap) @@ -165,10 +325,10 @@ 2022-07-19 2.6.0: ------------------- - * add data descriptions: `.nbytes`, `.padbits`, `.readonly` + * add data descriptors: `.nbytes`, `.padbits`, `.readonly` * allow optional `endian` argument to be `None` when creating bitarrays * fix type annotation for `canonical_decode()`, #178 - * frozenbitarray's padbits are now guaranteed to be zero + * frozenbitarray's pad bits are now guaranteed to be zero * add tests @@ -193,7 +353,7 @@ * optimize `delslice()` for cases like `del a[1:17:2]` when `a` is large * fix `examples/huffman/compress.py` to handle files with 0 or 1 characters, see also #172 - * add `skipIF` decorator for skipping tests + * add `skipIf` decorator for skipping tests * add tests @@ -281,7 +441,7 @@ a speedup for `.find()`, `.index()`, `.search()` and `util.rindex()` * add optional start and stop arguments to `.bytereverse()` * add example to illustrate how - [unaligned copying](../examples/copy_n.py) works internally + [unaligned copying](../devel/copy_n.py) works internally * add documentation * add tests diff -pruN 2.9.2-1/LICENSE 3.6.1-1/LICENSE --- 2.9.2-1/LICENSE 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/LICENSE 2025-08-12 08:35:42.000000000 +0000 @@ -11,7 +11,7 @@ license to reproduce, analyze, test, per prepare derivative works, distribute, and otherwise use bitarray alone or in any derivative version, provided, however, that Ilan Schnell's License Agreement and Ilan Schnell's notice of copyright, i.e., "Copyright (c) -2008 - 2024 Ilan Schnell; All Rights Reserved" are retained in bitarray +2008 - 2025 Ilan Schnell; All Rights Reserved" are retained in bitarray alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on diff -pruN 2.9.2-1/Makefile 3.6.1-1/Makefile --- 2.9.2-1/Makefile 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/Makefile 2025-08-12 08:35:42.000000000 +0000 @@ -10,7 +10,7 @@ test: bitarray/_bitarray.so install: - $(PYTHON) setup.py install + $(PYTHON) -m pip install -vv . doc: bitarray/_bitarray.so diff -pruN 2.9.2-1/README.rst 3.6.1-1/README.rst --- 2.9.2-1/README.rst 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/README.rst 2025-08-12 08:35:42.000000000 +0000 @@ -11,21 +11,6 @@ ability to import and export buffers. T are mapped to other objects, including memory-mapped files. -Roadmap -------- - -In 2024 (probably around July), we are planning the release of bitarray 3.0. -The 3.0 release will: - -* Remove Python 2.7 support. -* Rename ``.itersearch()`` to ``.search()`` and ``.iterdecode()`` - to ``.decode()`` (and remove their non-iterator counterpart). -* Remove ``util.rindex()``, use ``.index(..., right=1)`` instead -* Remove ``util.make_endian()``, use ``bitarray(..., endian=...)`` instead -* Remove hackish support for ``bitarray()`` handling unpickling, - see detailed explaination in `#207 `__. This will close `#206 `__. - - Key features ------------ @@ -42,15 +27,16 @@ Key features * Immutable ``frozenbitarray`` objects which are hashable * Sequential search * Type hinting -* Extensive test suite with about 500 unittests. +* Extensive test suite with over 500 unittests * Utility module ``bitarray.util``: * conversion to and from hexadecimal strings - * (de-) serialization + * generating random bitarrays * pretty printing * conversion to and from integers * creating Huffman codes * compression of sparse bitarrays + * (de-) serialization * various count functions * other helpful functions @@ -65,22 +51,15 @@ versions. Which means you can simply: $ pip install bitarray -In addition, conda packages are available (both the default Anaconda -repository as well as conda-forge support bitarray): - -.. code-block:: shell-session - - $ conda install bitarray - Once you have installed the package, you may want to test it: .. code-block:: shell-session $ python -c 'import bitarray; bitarray.test()' bitarray is installed in: /Users/ilan/bitarray/bitarray - bitarray version: 2.9.2 - sys.version: 3.11.0 (main, Oct 25 2022) [Clang 14.0.4] - sys.prefix: /Users/ilan/Mini3/envs/py311 + bitarray version: 3.6.1 + sys.version: 3.13.5 (main, Jun 16 2025) [Clang 18.1.8] + sys.prefix: /Users/ilan/miniforge pointer size: 64 bit sizeof(size_t): 8 sizeof(bitarrayobject): 80 @@ -92,7 +71,7 @@ Once you have installed the package, you ......................................................................... ................................................................ ---------------------------------------------------------------------- - Ran 502 tests in 0.416s + Ran 591 tests in 0.163s OK @@ -214,15 +193,15 @@ Bitarray objects support the bitwise ope >>> ~a # invert bitarray('010001110') >>> b = bitarray('111001011') - >>> a ^ b + >>> a ^ b # bitwise XOR bitarray('010111010') - >>> a &= b + >>> a &= b # inplace AND >>> a bitarray('101000001') - >>> a <<= 2 # in-place left shift by 2 + >>> a <<= 2 # in-place left-shift by 2 >>> a bitarray('100000100') - >>> b >> 1 + >>> b >> 1 # return b right-shifted by 1 bitarray('011100101') The C language does not specify the behavior of negative shifts and @@ -244,10 +223,10 @@ shift (``>>``) always shifts towards hig Bit-endianness -------------- -Unless explicitly converting to machine representation, using -the ``.tobytes()``, ``.frombytes()``, ``.tofile()`` and ``.fromfile()`` -methods, as well as using ``memoryview``, the bit-endianness will have no -effect on any computation, and one can skip this section. +Unless explicitly converting to machine representation, i.e. initializing +the buffer directly, using ``.tobytes()``, ``.frombytes()``, ``.tofile()`` +or ``.fromfile()``, as well as using ``memoryview()``, the bit-endianness +will have no effect on any computation, and one can skip this section. Since bitarrays allows addressing individual bits, where the machine represents 8 bits in one byte, there are two obvious choices for this @@ -260,10 +239,9 @@ By default, bitarrays use big-endian rep .. code-block:: python - >>> a = bitarray() - >>> a.endian() + >>> a = bitarray(b'A') + >>> a.endian 'big' - >>> a.frombytes(b'A') >>> a bitarray('01000001') >>> a[6] = 1 @@ -279,11 +257,10 @@ specified explicitly: .. code-block:: python - >>> a = bitarray(endian='little') - >>> a.frombytes(b'A') + >>> a = bitarray(b'A', endian='little') >>> a bitarray('10000010') - >>> a.endian() + >>> a.endian 'little' Here, the low-bit comes first because little-endian means that increasing @@ -292,7 +269,7 @@ So ``a[0]`` is the lowest address and le and ``a[7]`` is the highest address and most significant bit. The bit-endianness is a property of the bitarray object. -The endianness cannot be changed once a bitarray object is created. +The endianness cannot be changed once a bitarray object has been created. When comparing bitarray objects, the endianness (and hence the machine representation) is irrelevant; what matters is the mapping from indices to bits: @@ -301,6 +278,12 @@ to bits: >>> bitarray('11001', endian='big') == bitarray('11001', endian='little') True + >>> a = bitarray(b'\x01', endian='little') + >>> b = bitarray(b'\x80', endian='big') + >>> a == b + True + >>> a.tobytes() == b.tobytes() + False Bitwise operations (``|``, ``^``, ``&=``, ``|=``, ``^=``, ``~``) are implemented efficiently using the corresponding byte operations in C, i.e. the @@ -308,25 +291,6 @@ operators act on the machine representat Therefore, it is not possible to perform bitwise operators on bitarrays with different endianness. -When converting to and from machine representation, using -the ``.tobytes()``, ``.frombytes()``, ``.tofile()`` and ``.fromfile()`` -methods, the endianness matters: - -.. code-block:: python - - >>> a = bitarray(endian='little') - >>> a.frombytes(b'\x01') - >>> a - bitarray('10000000') - >>> b = bitarray(endian='big') - >>> b.frombytes(b'\x80') - >>> b - bitarray('10000000') - >>> a == b - True - >>> a.tobytes() == b.tobytes() - False - As mentioned above, the endianness can not be changed once an object is created. However, you can create a new bitarray with different endianness: @@ -369,17 +333,16 @@ found while iterating. For example: Note that the string ``'Hello'`` is an iterable, but the symbols are not limited to characters, in fact any immutable Python object can be a symbol. Taking the same dictionary, we can apply the ``.decode()`` method which will -return a list of the symbols: +return an iterable of the symbols: .. code-block:: python - >>> a.decode(d) + >>> list(a.decode(d)) ['H', 'e', 'l', 'l', 'o'] >>> ''.join(a.decode(d)) 'Hello' -Since symbols are not limited to being characters, it is necessary to return -them as elements of a list, rather than simply returning the joined string. +Symbols are not limited to being characters. The above dictionary ``d`` can be efficiently constructed using the function ``bitarray.util.huffman_code()``. I also wrote `Huffman coding in Python using bitarray `__ for more @@ -388,21 +351,19 @@ background information. When the codes are large, and you have many decode calls, most time will be spent creating the (same) internal decode tree objects. In this case, it will be much faster to create a ``decodetree`` object, which can be -passed to bitarray's ``.decode()`` and ``.iterdecode()`` methods, instead -of passing the prefix code dictionary to those methods itself: +passed to bitarray's ``.decode()`` method, instead of passing the prefix +code dictionary to those methods itself: .. code-block:: python >>> from bitarray import bitarray, decodetree >>> t = decodetree({'a': bitarray('0'), 'b': bitarray('1')}) >>> a = bitarray('0110') - >>> a.decode(t) + >>> list(a.decode(t)) ['a', 'b', 'b', 'a'] - >>> ''.join(a.iterdecode(t)) - 'abba' The sole purpose of the immutable ``decodetree`` object is to be passed -to bitarray's ``.decode()`` and ``.iterdecode()`` methods. +to bitarray's ``.decode()`` method. Frozenbitarrays @@ -427,7 +388,7 @@ and can therefore be used as a dictionar Reference ========= -bitarray version: 2.9.2 -- `change log `__ +bitarray version: 3.6.1 -- `change log `__ In the following, ``item`` and ``value`` are usually a single bit - an integer 0 or 1. @@ -440,15 +401,12 @@ The bitarray object: ``bitarray(initializer=0, /, endian='big', buffer=None)`` -> bitarray Return a new bitarray object whose items are bits initialized from - the optional initial object, and endianness. - The initializer may be of the following types: - - ``int``: Create a bitarray of given integer length. The initial values are - all ``0``. - - ``str``: Create bitarray from a string of ``0`` and ``1``. - - ``iterable``: Create bitarray from iterable or sequence of integers 0 or 1. + the optional initializer, and bit-endianness. + The initializer may be one of the following types: + a.) ``int`` bitarray, initialized to zeros, of given length + b.) ``bytes`` or ``bytearray`` to initialize buffer directly + c.) ``str`` of 0s and 1s, ignoring whitespace and "_" + d.) iterable of integers 0 or 1. Optional keyword arguments: @@ -460,7 +418,9 @@ The bitarray object: cannot be present (or has to be ``None``). The imported buffer may be read-only or writable, depending on the object type. - New in version 2.3: optional ``buffer`` argument. + New in version 2.3: optional ``buffer`` argument + + New in version 3.4: allow initializer ``bytes`` or ``bytearray`` to set buffer directly bitarray methods: @@ -485,7 +445,7 @@ bitarray methods: 0. memory address of buffer 1. buffer size (in bytes) - 2. bit-endianness as a string + 2. bit-endianness as a Unicode string 3. number of pad bits 4. allocated memory for the buffer (in bytes) 5. memory is read-only @@ -497,16 +457,16 @@ bitarray methods: For each byte in byte-range(start, stop) reverse bits in-place. The start and stop indices are given in terms of bytes (not bits). Also note that this method only changes the buffer; it does not change the - endianness of the bitarray object. Padbits are left unchanged such that - two consecutive calls will always leave the bitarray unchanged. + bit-endianness of the bitarray object. Pad bits are left unchanged such + that two consecutive calls will always leave the bitarray unchanged. - New in version 2.2.5: optional start and stop arguments. + New in version 2.2.5: optional start and stop arguments ``clear()`` Remove all items from the bitarray. - New in version 1.4. + New in version 1.4 ``copy()`` -> bitarray @@ -521,16 +481,21 @@ bitarray methods: The ``value`` may also be a sub-bitarray. In this case non-overlapping occurrences are counted within ``[start:stop]`` (``step`` must be 1). - New in version 1.1.0: optional start and stop arguments. + New in version 1.1.0: optional start and stop arguments - New in version 2.3.7: optional step argument. + New in version 2.3.7: optional step argument - New in version 2.9: add non-overlapping sub-bitarray count. + New in version 2.9: add non-overlapping sub-bitarray count -``decode(code, /)`` -> list +``decode(code, /)`` -> iterator Given a prefix code (a dict mapping symbols to bitarrays, or ``decodetree`` - object), decode content of bitarray and return it as a list of symbols. + object), decode content of bitarray and return an iterator over + corresponding symbols. + + See also: `Bitarray 3 transition `__ + + New in version 3.0: returns iterator (equivalent to past ``.iterdecode()``) ``encode(code, iterable, /)`` @@ -539,15 +504,13 @@ bitarray methods: with corresponding bitarray for each symbol. -``endian()`` -> str - Return the bit-endianness of the bitarray as a string (``little`` or ``big``). - - ``extend(iterable, /)`` - Append all items from ``iterable`` to the end of the bitarray. - If the iterable is a string, each ``0`` and ``1`` are appended as + Append items from to the end of the bitarray. + If ``iterable`` is a Unicode string, each ``0`` and ``1`` are appended as bits (ignoring whitespace and underscore). + New in version 3.4: allow ``bytes`` object + ``fill()`` -> int Add zeros to the end of the bitarray, such that the length will be @@ -559,25 +522,25 @@ bitarray methods: is found, such that sub_bitarray is contained within ``[start:stop]``. Return -1 when sub_bitarray is not found. - New in version 2.1. + New in version 2.1 - New in version 2.9: add optional keyword argument ``right``. + New in version 2.9: add optional keyword argument ``right`` ``frombytes(bytes, /)`` Extend bitarray with raw bytes from a bytes-like object. Each added byte will add eight bits to the bitarray. - New in version 2.5.0: allow bytes-like argument. + New in version 2.5.0: allow bytes-like argument ``fromfile(f, n=-1, /)`` Extend bitarray with up to ``n`` bytes read from file object ``f`` (or any other binary stream what supports a ``.read()`` method, e.g. ``io.BytesIO``). - Each read byte will add eight bits to the bitarray. When ``n`` is omitted or - negative, all bytes until EOF are read. When ``n`` is non-negative but - exceeds the data available, ``EOFError`` is raised (but the available data - is still read and appended). + Each read byte will add eight bits to the bitarray. When ``n`` is omitted + or negative, reads and extends all data until EOF. + When ``n`` is non-negative but exceeds the available data, ``EOFError`` is + raised. However, the available data is still read and extended. ``index(sub_bitarray, start=0, stop=, /, right=False)`` -> int @@ -585,7 +548,7 @@ bitarray methods: is found, such that sub_bitarray is contained within ``[start:stop]``. Raises ``ValueError`` when the sub_bitarray is not present. - New in version 2.9: add optional keyword argument ``right``. + New in version 2.9: add optional keyword argument ``right`` ``insert(index, value, /)`` @@ -596,23 +559,7 @@ bitarray methods: Invert all bits in bitarray (in-place). When the optional ``index`` is given, only invert the single bit at index. - New in version 1.5.3: optional index argument. - - -``iterdecode(code, /)`` -> iterator - Given a prefix code (a dict mapping symbols to bitarrays, or ``decodetree`` - object), decode content of bitarray and return an iterator over - the symbols. - - -``itersearch(sub_bitarray, start=0, stop=, /, right=False)`` -> iterator - Return iterator over indices where sub_bitarray is found, such that - sub_bitarray is contained within ``[start:stop]``. - The indices are iterated in ascending order (from lowest to highest), - unless ``right=True``, which will iterate in descending oder (starting with - rightmost match). - - New in version 2.9: optional start and stop arguments - add optional keyword argument ``right``. + New in version 1.5.3: optional index argument ``pack(bytes, /)`` @@ -624,7 +571,7 @@ bitarray methods: transfer of data between bitarray objects to other Python objects (for example NumPy's ndarray object) which have a different memory view. - New in version 2.5.0: allow bytes-like argument. + New in version 2.5.0: allow bytes-like argument ``pop(index=-1, /)`` -> item @@ -641,11 +588,18 @@ bitarray methods: Reverse all bits in bitarray (in-place). -``search(sub_bitarray, limit=, /)`` -> list - Searches for given sub_bitarray in self, and return list of start - positions. - The optional argument limits the number of search results to the integer - specified. By default, all search results are returned. +``search(sub_bitarray, start=0, stop=, /, right=False)`` -> iterator + Return iterator over indices where sub_bitarray is found, such that + sub_bitarray is contained within ``[start:stop]``. + The indices are iterated in ascending order (from lowest to highest), + unless ``right=True``, which will iterate in descending order (starting with + rightmost match). + + See also: `Bitarray 3 transition `__ + + New in version 2.9: optional start and stop arguments - add optional keyword argument ``right`` + + New in version 3.0: returns iterator (equivalent to past ``.itersearch()``) ``setall(value, /)`` @@ -657,9 +611,13 @@ bitarray methods: Sort all bits in bitarray (in-place). -``to01()`` -> str - Return a string containing '0's and '1's, representing the bits in the - bitarray. +``to01(group=0, sep=' ')`` -> str + Return bitarray as Unicode string of '0's and '1's. + The bits are grouped into ``group`` bits (default is no grouping). + When grouped, the string ``sep`` is inserted between groups + of ``group`` characters, default is a space. + + New in version 3.3: optional ``group`` and ``sep`` arguments ``tobytes()`` -> bytes @@ -671,8 +629,8 @@ bitarray methods: ``tolist()`` -> list - Return bitarray as list of integer items. - ``a.tolist()`` is equal to ``list(a)``. + Return bitarray as list of integers. + ``a.tolist()`` equals ``list(a)``. Note that the list object being created will require 32 or 64 times more memory (depending on the machine architecture) than the bitarray object, @@ -680,7 +638,7 @@ bitarray methods: ``unpack(zero=b'\x00', one=b'\x01')`` -> bytes - Return bytes containing one character for each bit in the bitarray, + Return bytes that contain one byte for each bit in the bitarray, using specified mapping. @@ -689,6 +647,12 @@ bitarray data descriptors: Data descriptors were added in version 2.6. +``endian`` -> str + bit-endianness as Unicode string + + New in version 3.4: replaces former ``.endian()`` method + + ``nbytes`` -> int buffer size in bytes @@ -709,14 +673,14 @@ Other objects: object is initialized. A ``frozenbitarray`` is immutable and hashable, and may therefore be used as a dictionary key. - New in version 1.1. + New in version 1.1 ``decodetree(code, /)`` -> decodetree Given a prefix code (a dict mapping symbols to bitarrays), - create a binary tree object to be passed to ``.decode()`` or ``.iterdecode()``. + create a binary tree object to be passed to ``.decode()``. - New in version 1.6. + New in version 1.6 Functions defined in the `bitarray` module: @@ -727,11 +691,11 @@ Functions defined in the `bitarray` modu ``get_default_endian()`` -> str - Return the default endianness for new bitarray objects being created. - Unless ``_set_default_endian('little')`` was called, the default endianness - is ``big``. + Return the default bit-endianness for new bitarray objects being created. + Unless ``_set_default_endian('little')`` was called, the default + bit-endianness is ``big``. - New in version 1.3. + New in version 1.3 ``test(verbosity=1)`` -> TextTestResult @@ -743,74 +707,95 @@ Functions defined in `bitarray.util` mod This sub-module was added in version 1.2. -``zeros(length, /, endian=None)`` -> bitarray - Create a bitarray of length, with all values 0, and optional - endianness, which may be 'big', 'little'. +``any_and(a, b, /)`` -> bool + Efficient implementation of ``any(a & b)``. + + New in version 2.7 + + +``ba2base(n, bitarray, /, group=0, sep=' ')`` -> str + Return a string containing the base ``n`` ASCII representation of + the bitarray. Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. + The bitarray has to be multiple of length 1, 2, 3, 4, 5 or 6 respectively. + For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the + standard base 64 alphabet is used. + When grouped, the string ``sep`` is inserted between groups + of ``group`` characters, default is a space. + + See also: `Bitarray representations `__ + New in version 1.9 -``ones(length, /, endian=None)`` -> bitarray - Create a bitarray of length, with all values 1, and optional - endianness, which may be 'big', 'little'. + New in version 3.3: optional ``group`` and ``sep`` arguments - New in version 2.9. +``ba2hex(bitarray, /, group=0, sep=' ')`` -> hexstr + Return a string containing the hexadecimal representation of + the bitarray (which has to be multiple of 4 in length). + When grouped, the string ``sep`` is inserted between groups + of ``group`` characters, default is a space. -``urandom(length, /, endian=None)`` -> bitarray - Return a bitarray of ``length`` random bits (uses ``os.urandom``). + New in version 3.3: optional ``group`` and ``sep`` arguments - New in version 1.7. +``ba2int(bitarray, /, signed=False)`` -> int + Convert the given bitarray to an integer. + The bit-endianness of the bitarray is respected. + ``signed`` indicates whether two's complement is used to represent the integer. -``pprint(bitarray, /, stream=None, group=8, indent=4, width=80)`` - Prints the formatted representation of object on ``stream`` (which defaults - to ``sys.stdout``). By default, elements are grouped in bytes (8 elements), - and 8 bytes (64 elements) per line. - Non-bitarray objects are printed by the standard library - function ``pprint.pprint()``. - New in version 1.8. +``base2ba(n, asciistr, /, endian=None)`` -> bitarray + Bitarray of base ``n`` ASCII representation. + Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. + For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the + standard base 64 alphabet is used. Whitespace is ignored. + See also: `Bitarray representations `__ -``make_endian(bitarray, /, endian)`` -> bitarray - When the endianness of the given bitarray is different from ``endian``, - return a new bitarray, with endianness ``endian`` and the same elements - as the original bitarray. - Otherwise (endianness is already ``endian``) the original bitarray is returned - unchanged. + New in version 1.9 - New in version 1.3. + New in version 3.3: ignore whitespace - New in version 2.9: deprecated - use ``bitarray()``. +``byteswap(a, /, n=)`` + Reverse every ``n`` consecutive bytes of ``a`` in-place. + By default, all bytes are reversed. Note that ``n`` is not limited to 2, 4 + or 8, but can be any positive integer. + Also, ``a`` may be any object that exposes a writable buffer. + Nothing about this function is specific to bitarray objects. -``rindex(bitarray, sub_bitarray=1, start=0, stop=, /)`` -> int - Return rightmost (highest) index where sub_bitarray (or item - defaults - to 1) is found in bitarray (``a``), such that sub_bitarray is contained - within ``a[start:stop]``. - Raises ``ValueError`` when the sub_bitarray is not present. + New in version 3.4 - New in version 2.3.0: optional start and stop arguments. - New in version 2.9: deprecated - use ``.index(..., right=1)``. +``canonical_decode(bitarray, count, symbol, /)`` -> iterator + Decode bitarray using canonical Huffman decoding tables + where ``count`` is a sequence containing the number of symbols of each length + and ``symbol`` is a sequence of symbols in canonical order. + See also: `Canonical Huffman Coding `__ -``strip(bitarray, /, mode='right')`` -> bitarray - Return a new bitarray with zeros stripped from left, right or both ends. - Allowed values for mode are the strings: ``left``, ``right``, ``both`` + New in version 2.5 -``count_n(a, n, value=1, /)`` -> int - Return lowest index ``i`` for which ``a[:i].count(value) == n``. - Raises ``ValueError`` when ``n`` exceeds total count (``a.count(value)``). +``canonical_huffman(dict, /)`` -> tuple + Given a frequency map, a dictionary mapping symbols to their frequency, + calculate the canonical Huffman code. Returns a tuple containing: + + 0. the canonical Huffman code as a dict mapping symbols to bitarrays + 1. a list containing the number of symbols of each code length + 2. a list of symbols in canonical order - New in version 2.3.6: optional value argument. + Note: the two lists may be used as input for ``canonical_decode()``. + See also: `Canonical Huffman Coding `__ -``parity(a, /)`` -> int - Return parity of bitarray ``a``. - ``parity(a)`` is equivalent to ``a.count() % 2`` but more efficient. + New in version 2.5 + + +``correspond_all(a, b, /)`` -> tuple + Return tuple with counts of: ~a & ~b, ~a & b, a & ~b, a & b - New in version 1.9. + New in version 3.4 ``count_and(a, b, /)`` -> int @@ -818,6 +803,13 @@ This sub-module was added in version 1.2 as no intermediate bitarray object gets created. +``count_n(a, n, value=1, /)`` -> int + Return lowest index ``i`` for which ``a[:i].count(value) == n``. + Raises ``ValueError`` when ``n`` exceeds total count (``a.count(value)``). + + New in version 2.3.6: optional value argument + + ``count_or(a, b, /)`` -> int Return ``(a | b).count()`` in a memory efficient manner, as no intermediate bitarray object gets created. @@ -830,17 +822,39 @@ This sub-module was added in version 1.2 This is also known as the Hamming distance. -``any_and(a, b, /)`` -> bool - Efficient implementation of ``any(a & b)``. +``deserialize(bytes, /)`` -> bitarray + Return a bitarray given a bytes-like representation such as returned + by ``serialize()``. - New in version 2.7. + See also: `Bitarray representations `__ + New in version 1.8 -``subset(a, b, /)`` -> bool - Return ``True`` if bitarray ``a`` is a subset of bitarray ``b``. - ``subset(a, b)`` is equivalent to ``a | b == b`` (and equally ``a & b == a``) but - more efficient as no intermediate bitarray object is created and the buffer - iteration is stopped as soon as one mismatch is found. + New in version 2.5.0: allow bytes-like argument + + +``hex2ba(hexstr, /, endian=None)`` -> bitarray + Bitarray of hexadecimal representation. hexstr may contain any number + (including odd numbers) of hex digits (upper or lower case). + Whitespace is ignored. + + New in version 3.3: ignore whitespace + + +``huffman_code(dict, /, endian=None)`` -> dict + Given a frequency map, a dictionary mapping symbols to their frequency, + calculate the Huffman code, i.e. a dict mapping those symbols to + bitarrays (with given bit-endianness). Note that the symbols are not limited + to being strings. Symbols may be any hashable object. + + +``int2ba(int, /, length=None, endian=None, signed=False)`` -> bitarray + Convert the given integer to a bitarray (with given bit-endianness, + and no leading (big-endian) / trailing (little-endian) zeros), unless + the ``length`` of the bitarray is provided. An ``OverflowError`` is raised + if the integer is not representable with the given number of bits. + ``signed`` determines whether two's complement is used to represent the integer, + and requires ``length`` to be provided. ``intervals(bitarray, /)`` -> iterator @@ -848,76 +862,72 @@ This sub-module was added in version 1.2 iterator over tuples ``(value, start, stop)``. The intervals are guaranteed to be in order, and their size is always non-zero (``stop - start > 0``). - New in version 2.7. - - -``ba2hex(bitarray, /)`` -> hexstr - Return a string containing the hexadecimal representation of - the bitarray (which has to be multiple of 4 in length). - + New in version 2.7 -``hex2ba(hexstr, /, endian=None)`` -> bitarray - Bitarray of hexadecimal representation. hexstr may contain any number - (including odd numbers) of hex digits (upper or lower case). +``ones(n, /, endian=None)`` -> bitarray + Create a bitarray of length ``n``, with all values ``1``, and optional + bit-endianness (``little`` or ``big``). -``ba2base(n, bitarray, /)`` -> str - Return a string containing the base ``n`` ASCII representation of - the bitarray. Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. - The bitarray has to be multiple of length 1, 2, 3, 4, 5 or 6 respectively. - For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the - standard base 64 alphabet is used. + New in version 2.9 - See also: `Bitarray representations `__ - New in version 1.9. +``parity(a, /)`` -> int + Return parity of bitarray ``a``. + ``parity(a)`` is equivalent to ``a.count() % 2`` but more efficient. + New in version 1.9 -``base2ba(n, asciistr, /, endian=None)`` -> bitarray - Bitarray of base ``n`` ASCII representation. - Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. - For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the - standard base 64 alphabet is used. - See also: `Bitarray representations `__ +``pprint(bitarray, /, stream=None, group=8, indent=4, width=80)`` + Prints the formatted representation of object on ``stream`` (which defaults + to ``sys.stdout``). By default, elements are grouped in bytes (8 elements), + and 8 bytes (64 elements) per line. + Non-bitarray objects are printed by the standard library + function ``pprint.pprint()``. - New in version 1.9. + New in version 1.8 -``ba2int(bitarray, /, signed=False)`` -> int - Convert the given bitarray to an integer. - The bit-endianness of the bitarray is respected. - ``signed`` indicates whether two's complement is used to represent the integer. +``random_k(n, /, k, endian=None)`` -> bitarray + Return (pseudo-) random bitarray of length ``n`` with ``k`` elements + set to one. Mathematically equivalent to setting (in a bitarray of + length ``n``) all bits at indices ``random.sample(range(n), k)`` to one. + The random bitarrays are reproducible when giving Python's ``random.seed()`` + with a specific seed value. + This function requires Python 3.9 or higher, as it depends on the standard + library function ``random.randbytes()``. Raises ``NotImplementedError`` + when Python version is too low. -``int2ba(int, /, length=None, endian=None, signed=False)`` -> bitarray - Convert the given integer to a bitarray (with given endianness, - and no leading (big-endian) / trailing (little-endian) zeros), unless - the ``length`` of the bitarray is provided. An ``OverflowError`` is raised - if the integer is not representable with the given number of bits. - ``signed`` determines whether two's complement is used to represent the integer, - and requires ``length`` to be provided. + New in version 3.6 -``serialize(bitarray, /)`` -> bytes - Return a serialized representation of the bitarray, which may be passed to - ``deserialize()``. It efficiently represents the bitarray object (including - its bit-endianness) and is guaranteed not to change in future releases. +``random_p(n, /, p=0.5, endian=None)`` -> bitarray + Return (pseudo-) random bitarray of length ``n``, where each bit has + probability ``p`` of being one (independent of any other bits). Mathematically + equivalent to ``bitarray((random() < p for _ in range(n)), endian)``, but much + faster for large ``n``. The random bitarrays are reproducible when giving + Python's ``random.seed()`` with a specific seed value. - See also: `Bitarray representations `__ + This function requires Python 3.12 or higher, as it depends on the standard + library function ``random.binomialvariate()``. Raises ``NotImplementedError`` + when Python version is too low. - New in version 1.8. + See also: `Random Bitarrays `__ + New in version 3.5 -``deserialize(bytes, /)`` -> bitarray - Return a bitarray given a bytes-like representation such as returned - by ``serialize()``. - See also: `Bitarray representations `__ +``sc_decode(stream)`` -> bitarray + Decompress binary stream (an integer iterator, or bytes-like object) of a + sparse compressed (``sc``) bitarray, and return the decoded bitarray. + This function consumes only one bitarray and leaves the remaining stream + untouched. Use ``sc_encode()`` for compressing (encoding). - New in version 1.8. + See also: `Compression of sparse bitarrays `__ - New in version 2.5.0: allow bytes-like argument. + New in version 2.7 ``sc_encode(bitarray, /)`` -> bytes @@ -927,28 +937,42 @@ This sub-module was added in version 1.2 See also: `Compression of sparse bitarrays `__ - New in version 2.7. + New in version 2.7 -``sc_decode(stream)`` -> bitarray - Decompress binary stream (an integer iterator, or bytes-like object) of a - sparse compressed (``sc``) bitarray, and return the decoded bitarray. - This function consumes only one bitarray and leaves the remaining stream - untouched. Use ``sc_encode()`` for compressing (encoding). +``serialize(bitarray, /)`` -> bytes + Return a serialized representation of the bitarray, which may be passed to + ``deserialize()``. It efficiently represents the bitarray object (including + its bit-endianness) and is guaranteed not to change in future releases. - See also: `Compression of sparse bitarrays `__ + See also: `Bitarray representations `__ - New in version 2.7. + New in version 1.8 -``vl_encode(bitarray, /)`` -> bytes - Return variable length binary representation of bitarray. - This representation is useful for efficiently storing small bitarray - in a binary stream. Use ``vl_decode()`` for decoding. +``strip(bitarray, /, mode='right')`` -> bitarray + Return a new bitarray with zeros stripped from left, right or both ends. + Allowed values for mode are the strings: ``left``, ``right``, ``both`` - See also: `Variable length bitarray format `__ - New in version 2.2. +``subset(a, b, /)`` -> bool + Return ``True`` if bitarray ``a`` is a subset of bitarray ``b``. + ``subset(a, b)`` is equivalent to ``a | b == b`` (and equally ``a & b == a``) but + more efficient as no intermediate bitarray object is created and the buffer + iteration is stopped as soon as one mismatch is found. + + +``sum_indices(a, /)`` -> int + Return sum of indices of all active bits in bitarray ``a``. + Equivalent to ``sum(i for i, v in enumerate(a) if v)``. + + New in version 3.6 + + +``urandom(n, /, endian=None)`` -> bitarray + Return random bitarray of length ``n`` (uses ``os.urandom()``). + + New in version 1.7 ``vl_decode(stream, /, endian=None)`` -> bitarray @@ -958,38 +982,29 @@ This sub-module was added in version 1.2 See also: `Variable length bitarray format `__ - New in version 2.2. + New in version 2.2 -``huffman_code(dict, /, endian=None)`` -> dict - Given a frequency map, a dictionary mapping symbols to their frequency, - calculate the Huffman code, i.e. a dict mapping those symbols to - bitarrays (with given endianness). Note that the symbols are not limited - to being strings. Symbols may may be any hashable object (such as ``None``). - - -``canonical_huffman(dict, /)`` -> tuple - Given a frequency map, a dictionary mapping symbols to their frequency, - calculate the canonical Huffman code. Returns a tuple containing: - - 0. the canonical Huffman code as a dict mapping symbols to bitarrays - 1. a list containing the number of symbols of each code length - 2. a list of symbols in canonical order +``vl_encode(bitarray, /)`` -> bytes + Return variable length binary representation of bitarray. + This representation is useful for efficiently storing small bitarray + in a binary stream. Use ``vl_decode()`` for decoding. - Note: the two lists may be used as input for ``canonical_decode()``. + See also: `Variable length bitarray format `__ - See also: `Canonical Huffman Coding `__ + New in version 2.2 - New in version 2.5. +``xor_indices(a, /)`` -> int + Return xor reduced indices of all active bits in bitarray ``a``. + This is essentially equivalent to + ``reduce(operator.xor, (i for i, v in enumerate(a) if v))``. -``canonical_decode(bitarray, count, symbol, /)`` -> iterator - Decode bitarray using canonical Huffman decoding tables - where ``count`` is a sequence containing the number of symbols of each length - and ``symbol`` is a sequence of symbols in canonical order. + New in version 3.2 - See also: `Canonical Huffman Coding `__ - New in version 2.5. +``zeros(n, /, endian=None)`` -> bitarray + Create a bitarray of length ``n``, with all values ``0``, and optional + bit-endianness (``little`` or ``big``). diff -pruN 2.9.2-1/bitarray/__init__.py 3.6.1-1/bitarray/__init__.py --- 2.9.2-1/bitarray/__init__.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/__init__.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright (c) 2008 - 2024, Ilan Schnell; All Rights Reserved +# Copyright (c) 2008 - 2025, Ilan Schnell; All Rights Reserved """ This package defines an object type which can efficiently represent a bitarray. Bitarrays are sequence types and behave very much like lists. @@ -12,7 +12,7 @@ Author: Ilan Schnell from __future__ import absolute_import from bitarray._bitarray import (bitarray, decodetree, _sysinfo, - _bitarray_reconstructor, + bits2bytes, _bitarray_reconstructor, get_default_endian, _set_default_endian, __version__) @@ -54,19 +54,6 @@ and may therefore be used as a dictionar __ilshift__ = __irshift__ = __delitem__ -def bits2bytes(__n): - """bits2bytes(n, /) -> int - -Return the number of bytes necessary to store n bits. -""" - import sys - if not isinstance(__n, (int, long) if sys.version_info[0] == 2 else int): - raise TypeError("integer expected") - if __n < 0: - raise ValueError("non-negative integer expected") - return (__n + 7) // 8 - - def test(verbosity=1): """test(verbosity=1) -> TextTestResult diff -pruN 2.9.2-1/bitarray/__init__.pyi 3.6.1-1/bitarray/__init__.pyi --- 2.9.2-1/bitarray/__init__.pyi 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/__init__.pyi 2025-08-12 08:35:42.000000000 +0000 @@ -1,6 +1,6 @@ -# Copyright (c) 2021 - 2024, Ilan Schnell; All Rights Reserved +# Copyright (c) 2021 - 2025, Ilan Schnell; All Rights Reserved # -# This stub, as well as util.pyi, are tested with Python 3.9 and mypy 0.950 +# This stub, as well as util.pyi, are tested with Python 3.10 and mypy 1.11.2 from collections.abc import Iterable, Iterator, Sequence from unittest.runner import TextTestResult @@ -9,7 +9,8 @@ from typing import Any, BinaryIO, Dict, CodeDict = Dict[Any, bitarray] -BytesLike = Union[bytes, Iterable[int]] +# Python 3.12 has abc.Buffer which should be used instead +BytesLike = Union[bytes, bytearray] class decodetree: @@ -41,9 +42,10 @@ class bitarray: stop: int = ..., step: int = ...) -> int: ... - def decode(self, code: Union[CodeDict, decodetree]) -> list: ... def encode(self, code: CodeDict, x: Iterable) -> None: ... - def endian(self) -> str: ... + def decode(self, + code: Union[CodeDict, decodetree]) -> Iterator: ... + def extend(self, x: Union[str, Iterable[int]]) -> None: ... def fill(self) -> int: ... def find(self, @@ -62,25 +64,23 @@ class bitarray: def insert(self, i: int, value: int) -> None: ... def invert(self, i: int = ...) -> None: ... - def iterdecode(self, - code: Union[CodeDict, decodetree]) -> Iterator: ... - def itersearch(self, - sub_bitarray: Union[bitarray, int], - start: int = ..., - stop: int = ..., - right: int = ...) -> Iterator[int]: ... + def search(self, + sub_bitarray: Union[bitarray, int], + start: int = ..., + stop: int = ..., + right: int = ...) -> Iterator[int]: ... def pack(self, b: BytesLike) -> None: ... def pop(self, i: int = ...) -> int: ... def remove(self, value: int) -> None: ... def reverse(self) -> None: ... - def search(self, sub_bitarray: Union[bitarray, int], - limit: int = ...) -> list[int]: ... def setall(self, value: int) -> None: ... def sort(self, reverse: int) -> None: ... - def to01(self) -> str: ... + def to01(self, + group: int = ..., + sep: str = ...) -> str: ... def tobytes(self) -> bytes: ... def tofile(self, f: BinaryIO) -> None: ... def tolist(self) -> list[int]: ... @@ -93,12 +93,18 @@ class bitarray: @overload def __getitem__(self, i: int) -> int: ... @overload - def __getitem__(self, s: Union[slice, Sequence]) -> bitarray: ... + def __getitem__(self, + s: Union[slice, bitarray, Sequence]) -> bitarray: ... @overload - def __setitem__(self, i: Union[int, slice, Sequence], o: int) -> None: ... + def __setitem__(self, + i: Union[int, slice, Sequence], + o: int) -> None: ... @overload - def __setitem__(self, s: Union[slice, Sequence] , o: bitarray) -> None: ... - def __delitem__(self, i: Union[int, slice, Sequence]) -> None: ... + def __setitem__(self, + s: Union[slice, bitarray, Sequence], + o: bitarray) -> None: ... + def __delitem__(self, + i: Union[int, slice, bitarray, Sequence]) -> None: ... def __add__(self, other: bitarray) -> bitarray: ... def __iadd__(self, other: bitarray) -> bitarray: ... @@ -125,6 +131,8 @@ class bitarray: # data descriptors @property + def endian(self) -> str: ... + @property def nbytes(self) -> int: ... @property def padbits(self) -> int: ... diff -pruN 2.9.2-1/bitarray/_bitarray.c 3.6.1-1/bitarray/_bitarray.c --- 2.9.2-1/bitarray/_bitarray.c 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/_bitarray.c 2025-08-12 08:35:42.000000000 +0000 @@ -1,5 +1,5 @@ /* - Copyright (c) 2008 - 2024, Ilan Schnell; All Rights Reserved + Copyright (c) 2008 - 2025, Ilan Schnell; All Rights Reserved bitarray is published under the PSF license. This file is the C part of the bitarray package. @@ -19,19 +19,20 @@ /* default bit-endianness */ static int default_endian = ENDIAN_BIG; -static PyTypeObject Bitarray_Type; - -/* translation table - setup during module initialization */ +/* translation table - setup during module initialization */ static char reverse_trans[256]; +static PyTypeObject Bitarray_Type; + #define bitarray_Check(obj) PyObject_TypeCheck((obj), &Bitarray_Type) static int resize(bitarrayobject *self, Py_ssize_t nbits) { - const Py_ssize_t allocated = self->allocated, size = Py_SIZE(self); - const Py_ssize_t newsize = BYTES(nbits); + const size_t size = Py_SIZE(self); + const size_t allocated = self->allocated; + const size_t newsize = BYTES((size_t) nbits); size_t new_allocated; if (self->ob_exports > 0) { @@ -45,32 +46,19 @@ resize(bitarrayobject *self, Py_ssize_t return -1; } - if (nbits < 0 || newsize < 0) { + if (nbits < 0) { PyErr_Format(PyExc_OverflowError, "bitarray resize %zd", nbits); return -1; } - assert(allocated >= size && size == BYTES(self->nbits)); + assert(allocated >= size && size == BYTES((size_t) self->nbits)); /* ob_item == NULL implies ob_size == allocated == 0 */ assert(self->ob_item != NULL || (size == 0 && allocated == 0)); - /* allocated == 0 implies size == 0 */ - assert(allocated != 0 || size == 0); /* resize() is never called on read-only memory */ assert(self->readonly == 0); + /* bypass everything when buffer size hasn't changed */ if (newsize == size) { - /* buffer size hasn't changed - bypass everything */ - self->nbits = nbits; - return 0; - } - - /* Bypass reallocation when a allocation is large enough to accommodate - the newsize. If the newsize falls lower than half the allocated size, - then proceed with the reallocation to shrink the bitarray. - */ - if (allocated >= newsize && newsize >= (allocated >> 1)) { - assert(self->ob_item != NULL || newsize == 0); - Py_SET_SIZE(self, newsize); self->nbits = nbits; return 0; } @@ -84,21 +72,31 @@ resize(bitarrayobject *self, Py_ssize_t return 0; } - /* Overallocate proportional to the bitarray size. - Add padding to make the allocated size multiple of 4. - The growth pattern is: 0, 4, 8, 16, 24, 32, 40, 48, 56, 64, 76, ... - The pattern starts out the same as for lists but then grows at a - smaller rate so that larger bitarrays only overallocate by 1/16th, - as bitarrays are assumed to be memory critical. */ - new_allocated = ((size_t) newsize + (newsize >> 4) + - (newsize < 8 ? 3 : 7)) & ~(size_t) 3; - - /* Do not overallocate if the new size is closer to overallocated size - than to the old size. */ - if (newsize - size > (Py_ssize_t) new_allocated - newsize) - new_allocated = ((size_t) newsize + 3) & ~(size_t) 3; + if (allocated >= newsize) { + /* current buffer is large enough to host the requested size */ + if (newsize >= allocated / 2) { + /* minor downsize, bypass reallocation */ + Py_SET_SIZE(self, newsize); + self->nbits = nbits; + return 0; + } + /* major downsize, resize down to exact size */ + new_allocated = newsize; + } + else { + /* need to grow buffer */ + new_allocated = newsize; + /* overallocate when previous size isn't zero and when growth + is moderate */ + if (size != 0 && newsize / 2 <= allocated) { + /* overallocate proportional to the bitarray size and + add padding to make the allocated size multiple of 4 */ + new_allocated += (newsize >> 4) + (newsize < 8 ? 3 : 7); + new_allocated &= ~(size_t) 3; + } + } - assert(new_allocated >= (size_t) newsize); + assert(new_allocated >= newsize); self->ob_item = PyMem_Realloc(self->ob_item, new_allocated); if (self->ob_item == NULL) { PyErr_NoMemory(); @@ -114,31 +112,29 @@ resize(bitarrayobject *self, Py_ssize_t static bitarrayobject * newbitarrayobject(PyTypeObject *type, Py_ssize_t nbits, int endian) { - const Py_ssize_t nbytes = BYTES(nbits); + const size_t nbytes = BYTES((size_t) nbits); bitarrayobject *obj; - if (nbits < 0 || nbytes < 0) { - PyErr_Format(PyExc_OverflowError, "new bitarray %zd", nbits); - return NULL; - } + assert(nbits >= 0); obj = (bitarrayobject *) type->tp_alloc(type, 0); if (obj == NULL) return NULL; - Py_SET_SIZE(obj, nbytes); if (nbytes == 0) { obj->ob_item = NULL; } else { - obj->ob_item = (char *) PyMem_Malloc((size_t) nbytes); + /* allocate exact size */ + obj->ob_item = (char *) PyMem_Malloc(nbytes); if (obj->ob_item == NULL) { PyObject_Del(obj); PyErr_NoMemory(); return NULL; } } - obj->allocated = nbytes; + Py_SET_SIZE(obj, nbytes); + obj->allocated = nbytes; /* no overallocation */ obj->nbits = nbits; obj->endian = endian; obj->ob_exports = 0; @@ -148,6 +144,7 @@ newbitarrayobject(PyTypeObject *type, Py return obj; } +/* return new copy of bitarray object self */ static bitarrayobject * bitarray_cp(bitarrayobject *self) { @@ -186,27 +183,12 @@ buffers_overlap(bitarrayobject *self, bi if (Py_SIZE(self) == 0 || Py_SIZE(other) == 0) return 0; -/* is pointer ptr in buffer of bitarray a */ +/* is pointer ptr in buffer of bitarray a ? */ #define PIB(a, ptr) (a->ob_item <= ptr && ptr < a->ob_item + Py_SIZE(a)) return PIB(self, other->ob_item) || PIB(other, self->ob_item); #undef PIB } -/* setup translation table, which maps each byte to it's reversed: - reverse_trans = {0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, ..., 0xff} */ -static void -setup_reverse_trans(void) -{ - int j, k; - - for (k = 0; k < 256; k++) { - reverse_trans[k] = 0x00; - for (j = 0; j < 8; j++) - if (k & 128 >> j) - reverse_trans[k] |= 1 << j; - } -} - /* reverse bits in first n characters of p */ static void bytereverse(char *p, Py_ssize_t n) @@ -224,7 +206,7 @@ bytereverse(char *p, Py_ssize_t n) They operate on little-endian and bit-endian bitarrays respectively. As we shift right, we need to start with the highest address and loop downwards such that lower bytes are still unaltered. - See also examples/shift_r8.c + See also devel/shift_r8.c */ static void shift_r8le(unsigned char *buff, Py_ssize_t n, int k) @@ -241,7 +223,7 @@ shift_r8le(unsigned char *buff, Py_ssize if (n || w) /* add shifted next lower byte */ buff[i] |= buff[i - 1] >> (8 - k); } - assert(w == 0 || to_aligned((void *) buff) == 0); + assert(w == 0 || ((uintptr_t) buff) % 4 == 0); while (w--) { /* shift in word-range(0, w) */ uint64_t *p = ((uint64_t *) buff) + w; #if HAVE_BUILTIN_BSWAP64 && PY_BIG_ENDIAN @@ -271,7 +253,7 @@ shift_r8be(unsigned char *buff, Py_ssize if (n || w) /* add shifted next lower byte */ buff[i] |= buff[i - 1] << (8 - k); } - assert(w == 0 || to_aligned((void *) buff) == 0); + assert(w == 0 || ((uintptr_t) buff) % 4 == 0); while (w--) { /* shift in word-range(0, w) */ uint64_t *p = ((uint64_t *) buff) + w; #if HAVE_BUILTIN_BSWAP64 && PY_LITTLE_ENDIAN @@ -291,7 +273,8 @@ static void shift_r8(bitarrayobject *self, Py_ssize_t a, Py_ssize_t b, int k) { unsigned char *buff = (unsigned char *) self->ob_item + a; - Py_ssize_t s = 0, n = b - a; /* number of bytes to be shifted */ + Py_ssize_t n = b - a; /* number of bytes to be shifted */ + Py_ssize_t s = 0; /* distance to next aligned pointer */ assert(0 <= k && k < 8); assert(0 <= a && a <= Py_SIZE(self)); @@ -306,14 +289,14 @@ shift_r8(bitarrayobject *self, Py_ssize_ n -= s; } - if (IS_LE(self)) { + if (IS_LE(self)) { /* little endian */ shift_r8le(buff, n, k); if (s) { buff[0] |= buff[-1] >> (8 - k); shift_r8le(buff - s, s, k); } } - else { + else { /* big endian */ shift_r8be(buff, n, k); if (s) { buff[0] |= buff[-1] << (8 - k); @@ -323,11 +306,11 @@ shift_r8(bitarrayobject *self, Py_ssize_ } /* Copy n bits from other (starting at b) onto self (starting at a). - Please see examples/copy_n.py for more details. + Please see devel/copy_n.py for more details. Notes: - self and other may have opposite bit-endianness - - other may equal self - copy a section of self onto ifself + - other may equal self - copy a section of self onto itself - when other and self are distinct objects, their buffers may not overlap */ @@ -403,22 +386,6 @@ insert_n(bitarrayobject *self, Py_ssize_ return 0; } -static void -invert(bitarrayobject *self) -{ - const Py_ssize_t nbytes = Py_SIZE(self); - const Py_ssize_t cwords = nbytes / 8; /* complete 64-bit words */ - char *buff = self->ob_item; - uint64_t *wbuff = WBUFF(self); - Py_ssize_t i; - - assert(self->readonly == 0); - for (i = 0; i < cwords; i++) - wbuff[i] = ~wbuff[i]; - for (i = 8 * cwords; i < nbytes; i++) - buff[i] = ~buff[i]; -} - /* repeat self m times (negative m is treated as 0) */ static int repeat(bitarrayobject *self, Py_ssize_t m) @@ -453,42 +420,127 @@ repeat(bitarrayobject *self, Py_ssize_t return 0; } -/* set bits in range(a, b) in self to vi */ +/* the following functions xyz_span, xyz_range operate on bitarray items: + - xyz_span: contiguous bits - self[a:b] (step=1) + - xyz_range: self[start:stop:step] (step > 0 is required) + */ + +/* invert bits self[a:b] in-place */ static void -setrange(bitarrayobject *self, Py_ssize_t a, Py_ssize_t b, int vi) +invert_span(bitarrayobject *self, Py_ssize_t a, Py_ssize_t b) { + const Py_ssize_t n = b - a; /* number of bits to invert */ + Py_ssize_t i; + assert(0 <= a && a <= self->nbits); assert(0 <= b && b <= self->nbits); assert(self->readonly == 0); - if (b >= a + 8) { - const Py_ssize_t byte_a = BYTES(a); /* byte-range(byte_a, byte_b) */ - const Py_ssize_t byte_b = b / 8; + if (n >= 64) { + const Py_ssize_t wa = (a + 63) / 64; /* word-range(wa, wb) */ + const Py_ssize_t wb = b / 64; + uint64_t *wbuff = WBUFF(self); - assert(a + 8 > 8 * byte_a && 8 * byte_b + 8 > b); + invert_span(self, a, 64 * wa); + for (i = wa; i < wb; i++) + wbuff[i] = ~wbuff[i]; + invert_span(self, 64 * wb, b); + } + else if (n >= 8) { + const Py_ssize_t ca = BYTES(a); /* char-range(ca, cb) */ + const Py_ssize_t cb = b / 8; + char *buff = self->ob_item; - setrange(self, a, 8 * byte_a, vi); - memset(self->ob_item + byte_a, vi ? 0xff : 0x00, - (size_t) (byte_b - byte_a)); - setrange(self, 8 * byte_b, b, vi); + invert_span(self, a, 8 * ca); + for (i = ca; i < cb; i++) + buff[i] = ~buff[i]; + invert_span(self, 8 * cb, b); + } + else { /* (bit-) range(a, b) */ + for (i = a; i < b; i++) + self->ob_item[i / 8] ^= BITMASK(self, i); + } +} + +/* invert bits self[start:stop:step] in-place */ +static void +invert_range(bitarrayobject *self, + Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step) +{ + assert(step > 0); + + if (step == 1) { + invert_span(self, start, stop); } else { + const char *table = bitmask_table[IS_BE(self)]; + char *buff = self->ob_item; + Py_ssize_t i; + + for (i = start; i < stop; i += step) + buff[i >> 3] ^= table[i & 7]; + } +} + +/* set bits self[a:b] to vi */ +static void +set_span(bitarrayobject *self, Py_ssize_t a, Py_ssize_t b, int vi) +{ + assert(0 <= a && a <= self->nbits); + assert(0 <= b && b <= self->nbits); + assert(self->readonly == 0); + + if (b >= a + 8) { + const Py_ssize_t ca = BYTES(a); /* char-range(ca, cb) */ + const Py_ssize_t cb = b / 8; + + assert(a + 8 > 8 * ca && 8 * cb + 8 > b); + + set_span(self, a, 8 * ca, vi); + memset(self->ob_item + ca, vi ? 0xff : 0x00, (size_t) (cb - ca)); + set_span(self, 8 * cb, b, vi); + } + else { /* (bit-) range(a, b) */ while (a < b) setbit(self, a++, vi); } } +/* set bits self[start:stop:step] to vi */ +static void +set_range(bitarrayobject *self, + Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step, int vi) +{ + assert(step > 0); + + if (step == 1) { + set_span(self, start, stop, vi); + } + else { + const char *table = bitmask_table[IS_BE(self)]; + char *buff = self->ob_item; + Py_ssize_t i; + + if (vi) { + for (i = start; i < stop; i += step) + buff[i >> 3] |= table[i & 7]; + } + else { + for (i = start; i < stop; i += step) + buff[i >> 3] &= ~table[i & 7]; + } + } +} + /* return number of 1 bits in self[a:b] */ static Py_ssize_t -count(bitarrayobject *self, Py_ssize_t a, Py_ssize_t b) +count_span(bitarrayobject *self, Py_ssize_t a, Py_ssize_t b) { const Py_ssize_t n = b - a; Py_ssize_t cnt = 0; assert(0 <= a && a <= self->nbits); assert(0 <= b && b <= self->nbits); - if (n <= 0) - return 0; if (n >= 64) { Py_ssize_t p = BYTES(a), w; /* first full byte */ @@ -497,26 +549,26 @@ count(bitarrayobject *self, Py_ssize_t a assert(8 * p - a < 64 && b - (8 * (p + 8 * w)) < 64 && w >= 0); - cnt += count(self, a, 8 * p); + cnt += count_span(self, a, 8 * p); cnt += popcnt_words((uint64_t *) (self->ob_item + p), w); - cnt += count(self, 8 * (p + 8 * w), b); + cnt += count_span(self, 8 * (p + 8 * w), b); } else if (n >= 8) { - const Py_ssize_t p = BYTES(a); /* first full byte */ - const Py_ssize_t m = b / 8 - p; /* number of full bytes to count */ + const Py_ssize_t ca = BYTES(a); /* char-range(ca, cb) */ + const Py_ssize_t cb = b / 8, m = cb - ca; - assert(8 * p - a < 8 && b - 8 * (p + m) < 8 && 0 <= m && m < 8); + assert(8 * ca - a < 8 && b - 8 * cb < 8 && 0 <= m && m < 8); - cnt += count(self, a, 8 * p); - if (m) { /* starting at p count in m bytes */ + cnt += count_span(self, a, 8 * ca); + if (m) { /* starting at ca count in m bytes */ uint64_t tmp = 0; /* copy bytes we want to count into tmp word */ - memcpy((char *) &tmp, self->ob_item + p, (size_t) m); + memcpy((char *) &tmp, self->ob_item + ca, (size_t) m); cnt += popcnt_64(tmp); } - cnt += count(self, 8 * (p + m), b); + cnt += count_span(self, 8 * cb, b); } - else { + else { /* (bit-) range(a, b) */ while (a < b) cnt += getbit(self, a++); } @@ -525,22 +577,25 @@ count(bitarrayobject *self, Py_ssize_t a /* return number of 1 bits in self[start:stop:step] */ static Py_ssize_t -count_slice(bitarrayobject *self, +count_range(bitarrayobject *self, Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step) { + assert(step > 0); + if (step == 1) { - return count(self, start, stop); + return count_span(self, start, stop); } else { Py_ssize_t cnt = 0, i; - assert(step > 0); + for (i = start; i < stop; i += step) cnt += getbit(self, i); return cnt; } } -/* return first/rightmost occurrence of vi in self[a:b], -1 when not found */ +/* return first (or rightmost in case right=1) occurrence + of vi in self[a:b], -1 when not found */ static Py_ssize_t find_bit(bitarrayobject *self, int vi, Py_ssize_t a, Py_ssize_t b, int right) { @@ -585,32 +640,32 @@ find_bit(bitarrayobject *self, int vi, P } /* For the same reason as above, we cannot check for n >= 8 here. */ if (n > 8) { - const Py_ssize_t byte_a = BYTES(a); /* byte-range(byte_a, byte_b) */ - const Py_ssize_t byte_b = b / 8; + const Py_ssize_t ca = BYTES(a); /* char-range(ca, cb) */ + const Py_ssize_t cb = b / 8; const char *buff = self->ob_item; const char c = vi ? 0 : ~0; if (right) { - if ((res = find_bit(self, vi, 8 * byte_b, b, 1)) >= 0) + if ((res = find_bit(self, vi, 8 * cb, b, 1)) >= 0) return res; - for (i = byte_b - 1; i >= byte_a; i--) { /* skip bytes */ + for (i = cb - 1; i >= ca; i--) { /* skip bytes */ assert_byte_in_range(self, i); if (c ^ buff[i]) return find_bit(self, vi, 8 * i, 8 * i + 8, 1); } - return find_bit(self, vi, a, 8 * byte_a, 1); + return find_bit(self, vi, a, 8 * ca, 1); } else { - if ((res = find_bit(self, vi, a, 8 * byte_a, 0)) >= 0) + if ((res = find_bit(self, vi, a, 8 * ca, 0)) >= 0) return res; - for (i = byte_a; i < byte_b; i++) { /* skip bytes */ + for (i = ca; i < cb; i++) { /* skip bytes */ assert_byte_in_range(self, i); if (c ^ buff[i]) return find_bit(self, vi, 8 * i, 8 * i + 8, 0); } - return find_bit(self, vi, 8 * byte_b, b, 0); + return find_bit(self, vi, 8 * cb, b, 0); } } /* finally, search for the desired bit by stepping one-by-one */ @@ -621,9 +676,9 @@ find_bit(bitarrayobject *self, int vi, P return -1; } -/* Given sub_bitarray, return: +/* Given sub-bitarray, return: -1: on error (after setting exception) - 0, 1: value of integer sub or single item of sub if bitarray of length 1 + 0, 1: value of integer sub or sub[0] if sub-bitarray has length 1 2: when sub is bitarray of length 0, 2, 3, ... */ static int @@ -635,12 +690,11 @@ value_sub(PyObject *sub) } if (bitarray_Check(sub)) { -#define ss ((bitarrayobject *) sub) - return (ss->nbits == 1) ? getbit(ss, 0) : 2; -#undef ss + bitarrayobject *s = (bitarrayobject *) sub; + return (s->nbits == 1) ? getbit(s, 0) : 2; } - PyErr_Format(PyExc_TypeError, "sub_bitarray must the bitarray or int, " + PyErr_Format(PyExc_TypeError, "sub_bitarray must be bitarray or int, " "not '%s'", Py_TYPE(sub)->tp_name); return -1; } @@ -714,16 +768,6 @@ count_sub(bitarrayobject *self, bitarray return cnt; } -/* place self->nbits characters ('0', '1' corresponding to self) into str */ -static void -setstr01(bitarrayobject *self, char *str) -{ - Py_ssize_t i; - - for (i = 0; i < self->nbits; i++) - str[i] = getbit(self, i) + '0'; -} - /* set item i in self to given value */ static int set_item(bitarrayobject *self, Py_ssize_t i, PyObject *value) @@ -755,47 +799,44 @@ extend_bitarray(bitarrayobject *self, bi static int extend_iter(bitarrayobject *self, PyObject *iter) { - const Py_ssize_t original_nbits = self->nbits; + const Py_ssize_t nbits = self->nbits; PyObject *item; assert(PyIter_Check(iter)); while ((item = PyIter_Next(iter))) { - if (resize(self, self->nbits + 1) < 0) - goto error; - if (set_item(self, self->nbits - 1, item) < 0) - goto error; + if (resize(self, self->nbits + 1) < 0 || + set_item(self, self->nbits - 1, item) < 0) + { + Py_DECREF(item); + /* ignore resize() return value as we fail anyhow */ + resize(self, nbits); + return -1; + } Py_DECREF(item); } if (PyErr_Occurred()) return -1; return 0; - error: - Py_DECREF(item); - /* ignore resize() return value as we fail anyhow */ - resize(self, original_nbits); - return -1; } static int extend_sequence(bitarrayobject *self, PyObject *sequence) { - const Py_ssize_t original_nbits = self->nbits; - PyObject *item; + const Py_ssize_t nbits = self->nbits; Py_ssize_t n, i; - n = PySequence_Size(sequence); - if (n < 0) + if ((n = PySequence_Size(sequence)) < 0) return -1; - if (resize(self, self->nbits + n) < 0) + if (resize(self, nbits + n) < 0) return -1; for (i = 0; i < n; i++) { - item = PySequence_GetItem(sequence, i); - if (item == NULL || set_item(self, self->nbits - n + i, item) < 0) { + PyObject *item = PySequence_GetItem(sequence, i); + if (item == NULL || set_item(self, nbits + i, item) < 0) { Py_XDECREF(item); - resize(self, original_nbits); + resize(self, nbits); return -1; } Py_DECREF(item); @@ -804,56 +845,35 @@ extend_sequence(bitarrayobject *self, Py } static int -extend_bytes01(bitarrayobject *self, PyObject *bytes) +extend_unicode01(bitarrayobject *self, PyObject *unicode) { const Py_ssize_t nbits = self->nbits; - Py_ssize_t i = nbits; /* current index */ - unsigned char c; - char *str; - int vi = 0; /* silence uninitialized warning on some compilers */ + const Py_ssize_t length = PyUnicode_GET_LENGTH(unicode); + Py_ssize_t i = nbits, j; /* i is the current index in self */ - assert(PyBytes_Check(bytes)); - str = PyBytes_AS_STRING(bytes); - if (resize(self, nbits + PyBytes_GET_SIZE(bytes)) < 0) + if (resize(self, nbits + length) < 0) return -1; - while ((c = *str++)) { - switch (c) { - case '0': vi = 0; break; - case '1': vi = 1; break; + for (j = 0; j < length; j++) { + Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, j); + + switch (ch) { + case '0': + case '1': + setbit(self, i++, ch - '0'); + continue; case '_': - case ' ': - case '\n': - case '\r': - case '\t': - case '\v': continue; - default: - PyErr_Format(PyExc_ValueError, "expected '0' or '1' " - "(or whitespace, or underscore), got '%c' (0x%02x)", - c, c); - resize(self, nbits); /* no bits added on error */ - return -1; } - setbit(self, i++, vi); - } - /* if we have ignored characters we over-sized earlier */ - return resize(self, i); -} - -static int -extend_unicode01(bitarrayobject *self, PyObject *unicode) -{ - PyObject *bytes; - int res; + if (Py_UNICODE_ISSPACE(ch)) + continue; - assert(PyUnicode_Check(unicode)); - if ((bytes = PyUnicode_AsASCIIString(unicode)) == NULL) + PyErr_Format(PyExc_ValueError, "expected '0' or '1' (or whitespace " + "or underscore), got '%c' (0x%02x)", ch, ch); + resize(self, nbits); /* no bits added on error */ return -1; - - res = extend_bytes01(self, bytes); - Py_DECREF(bytes); /* drop bytes */ - return res; + } + return resize(self, i); /* in case we ignored characters */ } static int @@ -865,17 +885,7 @@ extend_dispatch(bitarrayobject *self, Py if (bitarray_Check(obj)) /* bitarray */ return extend_bitarray(self, (bitarrayobject *) obj); - if (PyBytes_Check(obj)) { /* bytes 01 */ -#if IS_PY3K - PyErr_SetString(PyExc_TypeError, "cannot extend bitarray with " - "'bytes', use .pack() or .frombytes() instead"); - return -1; -#else - return extend_bytes01(self, obj); -#endif - } - - if (PyUnicode_Check(obj)) /* unicode 01 */ + if (PyUnicode_Check(obj)) /* Unicode string */ return extend_unicode01(self, obj); if (PySequence_Check(obj)) /* sequence */ @@ -991,7 +1001,7 @@ bitarray_bytereverse(bitarrayobject *sel return NULL; } if (stop > start) - bytereverse(self -> ob_item + start, stop - start); + bytereverse(self->ob_item + start, stop - start); Py_RETURN_NONE; } @@ -1001,8 +1011,8 @@ PyDoc_STRVAR(bytereverse_doc, For each byte in byte-range(start, stop) reverse bits in-place.\n\ The start and stop indices are given in terms of bytes (not bits).\n\ Also note that this method only changes the buffer; it does not change the\n\ -endianness of the bitarray object. Padbits are left unchanged such that\n\ -two consecutive calls will always leave the bitarray unchanged."); +bit-endianness of the bitarray object. Pad bits are left unchanged such\n\ +that two consecutive calls will always leave the bitarray unchanged."); static PyObject * @@ -1034,7 +1044,7 @@ Return a tuple containing:\n\ \n\ 0. memory address of buffer\n\ 1. buffer size (in bytes)\n\ -2. bit-endianness as a string\n\ +2. bit-endianness as a Unicode string\n\ 3. number of pad bits\n\ 4. allocated memory for the buffer (in bytes)\n\ 5. memory is read-only\n\ @@ -1126,11 +1136,11 @@ bitarray_count(bitarrayobject *self, PyO if (step > 0 && start > self->nbits) return PyLong_FromSsize_t(0); - slicelength = adjust_indices(self->nbits, &start, &stop, step); + slicelength = PySlice_AdjustIndices(self->nbits, &start, &stop, step); if (vi < 2) { /* value count */ adjust_step_positive(slicelength, &start, &stop, &step); - cnt = count_slice(self, start, stop, step); + cnt = count_range(self, start, stop, step); return PyLong_FromSsize_t(vi ? cnt : slicelength - cnt); } @@ -1156,18 +1166,6 @@ occurrences are counted within `[start:s static PyObject * -bitarray_endian(bitarrayobject *self) -{ - return Py_BuildValue("s", ENDIAN_STR(self->endian)); -} - -PyDoc_STRVAR(endian_doc, -"endian() -> str\n\ -\n\ -Return the bit-endianness of the bitarray as a string (`little` or `big`)."); - - -static PyObject * bitarray_extend(bitarrayobject *self, PyObject *obj) { RAISE_IF_READONLY(self, NULL); @@ -1179,8 +1177,8 @@ bitarray_extend(bitarrayobject *self, Py PyDoc_STRVAR(extend_doc, "extend(iterable, /)\n\ \n\ -Append all items from `iterable` to the end of the bitarray.\n\ -If the iterable is a string, each `0` and `1` are appended as\n\ +Append items from to the end of the bitarray.\n\ +If `iterable` is a Unicode string, each `0` and `1` are appended as\n\ bits (ignoring whitespace and underscore)."); @@ -1221,7 +1219,7 @@ bitarray_find(bitarrayobject *self, PyOb /* cannot find anything (including empty sub-bitarray) */ return PyLong_FromSsize_t(-1); - adjust_indices(self->nbits, &start, &stop, 1); + PySlice_AdjustIndices(self->nbits, &start, &stop, 1); pos = find_obj(self, sub, start, stop, right); if (pos == -2) @@ -1250,13 +1248,8 @@ bitarray_index(bitarrayobject *self, PyO assert(PyLong_Check(result)); if (PyLong_AsSsize_t(result) < 0) { Py_DECREF(result); -#if IS_PY3K - PyErr_Format(PyExc_ValueError, "%A not in bitarray", - PyTuple_GET_ITEM(args, 0)); -#else - PyErr_SetString(PyExc_ValueError, "item not in bitarray"); -#endif - return NULL; + return PyErr_Format(PyExc_ValueError, "%A not in bitarray", + PyTuple_GET_ITEM(args, 0)); } return result; } @@ -1272,14 +1265,20 @@ Raises `ValueError` when the sub_bitarra static PyObject * bitarray_insert(bitarrayobject *self, PyObject *args) { - Py_ssize_t i; + Py_ssize_t n = self->nbits, i; int vi; RAISE_IF_READONLY(self, NULL); if (!PyArg_ParseTuple(args, "nO&:insert", &i, conv_pybit, &vi)) return NULL; - adjust_index(self->nbits, &i, 1); + if (i < 0) { + i += n; + if (i < 0) + i = 0; + } + if (i > n) + i = n; if (insert_n(self, i, 1) < 0) return NULL; @@ -1296,26 +1295,45 @@ Insert `value` into bitarray before `ind static PyObject * bitarray_invert(bitarrayobject *self, PyObject *args) { - Py_ssize_t i = PY_SSIZE_T_MAX; + PyObject *arg = Py_None; RAISE_IF_READONLY(self, NULL); - if (!PyArg_ParseTuple(args, "|n:invert", &i)) + if (!PyArg_ParseTuple(args, "|O:invert", &arg)) return NULL; - if (i == PY_SSIZE_T_MAX) { /* default - invert all bits */ - invert(self); + if (PyIndex_Check(arg)) { + Py_ssize_t i; + + i = PyNumber_AsSsize_t(arg, NULL); + if (i == -1 && PyErr_Occurred()) + return NULL; + + if (i < 0) + i += self->nbits; + if (i < 0 || i >= self->nbits) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + self->ob_item[i / 8] ^= BITMASK(self, i); Py_RETURN_NONE; } + if (PySlice_Check(arg)) { + Py_ssize_t start, stop, step, slicelength; - if (i < 0) - i += self->nbits; - - if (i < 0 || i >= self->nbits) { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; + if (PySlice_GetIndicesEx(arg, self->nbits, + &start, &stop, &step, &slicelength) < 0) + return NULL; + adjust_step_positive(slicelength, &start, &stop, &step); + invert_range(self, start, stop, step); + Py_RETURN_NONE; } - self->ob_item[i / 8] ^= BITMASK(self, i); - Py_RETURN_NONE; + if (arg == Py_None) { + invert_span(self, 0, self->nbits); + Py_RETURN_NONE; + } + + return PyErr_Format(PyExc_TypeError, "index expect, not '%s' object", + Py_TYPE(arg)->tp_name); } PyDoc_STRVAR(invert_doc, @@ -1372,29 +1390,24 @@ static PyObject * bitarray_repr(bitarrayobject *self) { PyObject *result; - size_t strsize; + size_t nbits = self->nbits, strsize, i; char *str; - if (self->nbits == 0) - return Py_BuildValue("s", "bitarray()"); - - strsize = self->nbits + 12; /* 12 is the length of "bitarray('')" */ - if (strsize > PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "bitarray too large to represent"); - return NULL; - } + if (nbits == 0) + return PyUnicode_FromString("bitarray()"); - str = (char *) PyMem_Malloc(strsize); + strsize = nbits + 12; /* 12 is length of "bitarray('')" */ + str = PyMem_New(char, strsize); if (str == NULL) return PyErr_NoMemory(); strcpy(str, "bitarray('"); /* has length 10 */ - setstr01(self, str + 10); + for (i = 0; i < nbits; i++) + str[i + 10] = getbit(self, i) + '0'; str[strsize - 2] = '\''; - str[strsize - 1] = ')'; /* no terminating '\0' */ - - result = Py_BuildValue("s#", str, (Py_ssize_t) strsize); + str[strsize - 1] = ')'; + /* we know the string length beforehand - not null-terminated */ + result = PyUnicode_FromStringAndSize(str, strsize); PyMem_Free((void *) str); return result; } @@ -1403,10 +1416,8 @@ bitarray_repr(bitarrayobject *self) static PyObject * bitarray_reverse(bitarrayobject *self) { - const Py_ssize_t nbytes = Py_SIZE(self); const Py_ssize_t p = PADBITS(self); /* number of pad bits */ char *buff = self->ob_item; - Py_ssize_t i, j; RAISE_IF_READONLY(self, NULL); @@ -1416,17 +1427,18 @@ bitarray_reverse(bitarrayobject *self) self->nbits += p; /* reverse order of bytes */ - for (i = 0, j = nbytes - 1; i < j; i++, j--) { - char t = buff[i]; - buff[i] = buff[j]; - buff[j] = t; - } + swap_bytes(buff, Py_SIZE(self)); + /* reverse order of bits within each byte */ - bytereverse(self->ob_item, nbytes); + bytereverse(self->ob_item, Py_SIZE(self)); - /* remove the p pad bits at the end of the original bitarray that - are now the leading p bits */ - delete_n(self, 0, p); + /* Remove the p pad bits at the end of the original bitarray that + are now the leading p bits. + The reason why we don't just call delete_n(self, 0, p) here is that + it calls resize(), and we want to allow reversing an imported + writable buffer. */ + copy_n(self, 0, self, p, self->nbits - p); + self->nbits -= p; Py_RETURN_NONE; } @@ -1438,50 +1450,6 @@ Reverse all bits in bitarray (in-place). static PyObject * -bitarray_search(bitarrayobject *self, PyObject *args) -{ - PyObject *list = NULL, *item = NULL, *x; - Py_ssize_t limit = PY_SSIZE_T_MAX, p = 0; - - if (!PyArg_ParseTuple(args, "O|n:search", &x, &limit)) - return NULL; - - if (value_sub(x) < 0) - return NULL; - if (bitarray_Check(x) && ((bitarrayobject *) x)->nbits == 0) { - PyErr_SetString(PyExc_ValueError, "cannot search for empty bitarray"); - return NULL; - } - - if ((list = PyList_New(0)) == NULL) - goto error; - - while ((p = find_obj(self, x, p, self->nbits, 0)) >= 0) { - if (PyList_Size(list) >= limit) - break; - item = PyLong_FromSsize_t(p++); - if (item == NULL || PyList_Append(list, item) < 0) - goto error; - Py_DECREF(item); - } - return list; - - error: - Py_XDECREF(item); - Py_XDECREF(list); - return NULL; -} - -PyDoc_STRVAR(search_doc, -"search(sub_bitarray, limit=, /) -> list\n\ -\n\ -Searches for given sub_bitarray in self, and return list of start\n\ -positions.\n\ -The optional argument limits the number of search results to the integer\n\ -specified. By default, all search results are returned."); - - -static PyObject * bitarray_setall(bitarrayobject *self, PyObject *value) { int vi; @@ -1490,7 +1458,9 @@ bitarray_setall(bitarrayobject *self, Py if (!conv_pybit(value, &vi)) return NULL; - memset(self->ob_item, vi ? 0xff : 0x00, (size_t) Py_SIZE(self)); + if (self->ob_item) + memset(self->ob_item, vi ? 0xff : 0x00, (size_t) Py_SIZE(self)); + Py_RETURN_NONE; } @@ -1512,15 +1482,15 @@ bitarray_sort(bitarrayobject *self, PyOb if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i:sort", kwlist, &reverse)) return NULL; - cnt1 = count(self, 0, nbits); + cnt1 = count_span(self, 0, nbits); if (reverse) { - setrange(self, 0, cnt1, 1); - setrange(self, cnt1, nbits, 0); + set_span(self, 0, cnt1, 1); + set_span(self, cnt1, nbits, 0); } else { Py_ssize_t cnt0 = nbits - cnt1; - setrange(self, 0, cnt0, 0); - setrange(self, cnt0, nbits, 1); + set_span(self, 0, cnt0, 0); + set_span(self, cnt0, nbits, 1); } Py_RETURN_NONE; } @@ -1555,8 +1525,8 @@ bitarray_tolist(bitarrayobject *self) PyDoc_STRVAR(tolist_doc, "tolist() -> list\n\ \n\ -Return bitarray as list of integer items.\n\ -`a.tolist()` is equal to `list(a)`."); +Return bitarray as list of integers.\n\ +`a.tolist()` equals `list(a)`."); static PyObject * @@ -1574,6 +1544,7 @@ bitarray_frombytes(bitarrayobject *self, if (resize(self, 8 * (n + view.len)) < 0) goto error; + assert(Py_SIZE(self) == n + view.len); memcpy(self->ob_item + n, (char *) view.buf, (size_t) view.len); /* remove pad bits staring at previous bit length (8 * n - p) */ @@ -1582,7 +1553,6 @@ bitarray_frombytes(bitarrayobject *self, PyBuffer_Release(&view); Py_RETURN_NONE; - error: PyBuffer_Release(&view); return NULL; @@ -1608,44 +1578,59 @@ PyDoc_STRVAR(tobytes_doc, Return the bitarray buffer in bytes (pad bits are set to zero)."); +/* Extend self with bytes from f.read(n). Return number of bytes actually + read and extended, or -1 on failure (after setting exception). */ +static Py_ssize_t +extend_fread(bitarrayobject *self, PyObject *f, Py_ssize_t n) +{ + PyObject *bytes, *ret; + Py_ssize_t res; /* result (size or -1) */ + + bytes = PyObject_CallMethod(f, "read", "n", n); + if (bytes == NULL) + return -1; + if (!PyBytes_Check(bytes)) { + Py_DECREF(bytes); + PyErr_Format(PyExc_TypeError, ".read() did not return 'bytes', " + "got '%s'", Py_TYPE(bytes)->tp_name); + return -1; + } + res = PyBytes_GET_SIZE(bytes); + assert(0 <= res && res <= n); + + ret = bitarray_frombytes(self, bytes); + Py_DECREF(bytes); + if (ret == NULL) + res = -1; + Py_DECREF(ret); + return res; +} + static PyObject * bitarray_fromfile(bitarrayobject *self, PyObject *args) { - PyObject *bytes, *f; - Py_ssize_t nread = 0, nbytes = -1; + PyObject *f; + Py_ssize_t nread = 0, n = -1; RAISE_IF_READONLY(self, NULL); - if (!PyArg_ParseTuple(args, "O|n:fromfile", &f, &nbytes)) + if (!PyArg_ParseTuple(args, "O|n:fromfile", &f, &n)) return NULL; - if (nbytes < 0) /* read till EOF */ - nbytes = PY_SSIZE_T_MAX; + if (n < 0) /* read till EOF */ + n = PY_SSIZE_T_MAX; - while (nread < nbytes) { - PyObject *ret; /* return object from bitarray_frombytes() */ - Py_ssize_t nblock = Py_MIN(nbytes - nread, BLOCKSIZE); - int not_enough_bytes; + while (nread < n) { + Py_ssize_t nblock = Py_MIN(n - nread, BLOCKSIZE), size; - bytes = PyObject_CallMethod(f, "read", "n", nblock); - if (bytes == NULL) + size = extend_fread(self, f, nblock); + if (size < 0) return NULL; - if (!PyBytes_Check(bytes)) { - Py_DECREF(bytes); - PyErr_SetString(PyExc_TypeError, "read() didn't return bytes"); - return NULL; - } - not_enough_bytes = PyBytes_GET_SIZE(bytes) < nblock; - nread += PyBytes_GET_SIZE(bytes); - assert(nread >= 0 && nread <= nbytes); - ret = bitarray_frombytes(self, bytes); - Py_DECREF(bytes); - if (ret == NULL) - return NULL; - Py_DECREF(ret); /* drop bitarray_frombytes() result (None) */ + nread += size; + assert(size <= nblock && nread <= n); - if (not_enough_bytes) { - if (nbytes == PY_SSIZE_T_MAX) /* read till EOF */ + if (size < nblock) { + if (n == PY_SSIZE_T_MAX) /* read till EOF */ break; PyErr_SetString(PyExc_EOFError, "not enough bytes to read"); return NULL; @@ -1659,10 +1644,10 @@ PyDoc_STRVAR(fromfile_doc, \n\ Extend bitarray with up to `n` bytes read from file object `f` (or any\n\ other binary stream what supports a `.read()` method, e.g. `io.BytesIO`).\n\ -Each read byte will add eight bits to the bitarray. When `n` is omitted or\n\ -negative, all bytes until EOF are read. When `n` is non-negative but\n\ -exceeds the data available, `EOFError` is raised (but the available data\n\ -is still read and appended)."); +Each read byte will add eight bits to the bitarray. When `n` is omitted\n\ +or negative, reads and extends all data until EOF.\n\ +When `n` is non-negative but exceeds the available data, `EOFError` is\n\ +raised. However, the available data is still read and extended."); static PyObject * @@ -1678,7 +1663,7 @@ bitarray_tofile(bitarrayobject *self, Py assert(size >= 0 && offset + size <= nbytes); /* basically: f.write(memoryview(self)[offset:offset + size] */ - ret = PyObject_CallMethod(f, "write", BYTES_SIZE_FMT, + ret = PyObject_CallMethod(f, "write", "y#", self->ob_item + offset, size); if (ret == NULL) return NULL; @@ -1694,26 +1679,51 @@ Write byte representation of bitarray to static PyObject * -bitarray_to01(bitarrayobject *self) +bitarray_to01(bitarrayobject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"group", "sep", NULL}; + size_t strsize = self->nbits, j, nsep; + Py_ssize_t group = 0, i; PyObject *result; - char *str; + char *sep = " ", *str; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ns:to01", kwlist, + &group, &sep)) + return NULL; + + if (group < 0) + return PyErr_Format(PyExc_ValueError, "non-negative integer " + "expected, got: %zd", group); - str = (char *) PyMem_Malloc((size_t) self->nbits); + nsep = (group && strsize) ? strlen(sep) : 0; /* 0 means no grouping */ + if (nsep) + strsize += nsep * ((strsize - 1) / group); + + str = PyMem_New(char, strsize); if (str == NULL) return PyErr_NoMemory(); - setstr01(self, str); - result = Py_BuildValue("s#", str, self->nbits); + for (i = j = 0; i < self->nbits; i++) { + if (nsep && i && i % group == 0) { + memcpy(str + j, sep, nsep); + j += nsep; + } + str[j++] = getbit(self, i) + '0'; + } + assert(j == strsize); + + result = PyUnicode_FromStringAndSize(str, strsize); PyMem_Free((void *) str); return result; } PyDoc_STRVAR(to01_doc, -"to01() -> str\n\ +"to01(group=0, sep=' ') -> str\n\ \n\ -Return a string containing '0's and '1's, representing the bits in the\n\ -bitarray."); +Return bitarray as Unicode string of '0's and '1's.\n\ +The bits are grouped into `group` bits (default is no grouping).\n\ +When grouped, the string `sep` is inserted between groups\n\ +of `group` characters, default is a space."); static PyObject * @@ -1741,7 +1751,7 @@ bitarray_unpack(bitarrayobject *self, Py PyDoc_STRVAR(unpack_doc, "unpack(zero=b'\\x00', one=b'\\x01') -> bytes\n\ \n\ -Return bytes containing one character for each bit in the bitarray,\n\ +Return bytes that contain one byte for each bit in the bitarray,\n\ using specified mapping."); @@ -1778,22 +1788,22 @@ map to bit 1."); static PyObject * bitarray_pop(bitarrayobject *self, PyObject *args) { - Py_ssize_t i = -1; + Py_ssize_t n = self->nbits, i = -1; long vi; RAISE_IF_READONLY(self, NULL); if (!PyArg_ParseTuple(args, "|n:pop", &i)) return NULL; - if (self->nbits == 0) { + if (n == 0) { /* special case -- most common failure cause */ PyErr_SetString(PyExc_IndexError, "pop from empty bitarray"); return NULL; } if (i < 0) - i += self->nbits; + i += n; - if (i < 0 || i >= self->nbits) { + if (i < 0 || i >= n) { PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } @@ -1870,7 +1880,7 @@ bitarray_freeze(bitarrayobject *self) Py_RETURN_NONE; } -/* ---------- functionality exposed in debug mode for testing ---------- */ +/* -------- bitarray methods exposed in debug mode for testing ---------- */ #ifndef NDEBUG @@ -1903,10 +1913,7 @@ bitarray_copy_n(bitarrayobject *self, Py static PyObject * bitarray_overlap(bitarrayobject *self, PyObject *other) { - if (!bitarray_Check(other)) { - PyErr_SetString(PyExc_TypeError, "bitarray expected"); - return NULL; - } + assert(bitarray_Check(other)); return PyBool_FromLong(buffers_overlap(self, (bitarrayobject *) other)); } @@ -1915,6 +1922,12 @@ bitarray_overlap(bitarrayobject *self, P /* ---------------------- bitarray getset members ---------------------- */ static PyObject * +bitarray_get_endian(bitarrayobject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString(ENDIAN_STR(self->endian)); +} + +static PyObject * bitarray_get_nbytes(bitarrayobject *self, void *Py_UNUSED(ignored)) { return PyLong_FromSsize_t(Py_SIZE(self)); @@ -1932,7 +1945,9 @@ bitarray_get_readonly(bitarrayobject *se return PyBool_FromLong(self->readonly); } -static PyGetSetDef bitarray_getsets [] = { +static PyGetSetDef bitarray_getsets[] = { + {"endian", (getter) bitarray_get_endian, NULL, + PyDoc_STR("bit-endianness as Unicode string")}, {"nbytes", (getter) bitarray_get_nbytes, NULL, PyDoc_STR("buffer size in bytes")}, {"padbits", (getter) bitarray_get_padbits, NULL, @@ -2096,7 +2111,7 @@ ensure_mask_size(bitarrayobject *self, b /* return a new bitarray with items from 'self' masked by bitarray 'mask' */ static PyObject * -getmasked(bitarrayobject *self, bitarrayobject *mask) +getmask(bitarrayobject *self, bitarrayobject *mask) { bitarrayobject *res; Py_ssize_t i, j, n; @@ -2104,7 +2119,7 @@ getmasked(bitarrayobject *self, bitarray if (ensure_mask_size(self, mask) < 0) return NULL; - n = count(mask, 0, mask->nbits); + n = count_span(mask, 0, mask->nbits); res = newbitarrayobject(Py_TYPE(self), n, self->endian); if (res == NULL) return NULL; @@ -2198,7 +2213,7 @@ bitarray_subscr(bitarrayobject *self, Py return getslice(self, item); if (bitarray_Check(item)) - return getmasked(self, (bitarrayobject *) item); + return getmask(self, (bitarrayobject *) item); if (subscr_seq_check(item) < 0) return NULL; @@ -2236,11 +2251,11 @@ setslice_bitarray(bitarrayobject *self, if (step == 1) { if (increase > 0) { /* increase self */ if (insert_n(self, start + slicelength, increase) < 0) - goto error; + goto finish; } if (increase < 0) { /* decrease self */ if (delete_n(self, start + other->nbits, -increase) < 0) - goto error; + goto finish; } /* copy new values into self */ copy_n(self, start, other, 0, other->nbits); @@ -2252,14 +2267,14 @@ setslice_bitarray(bitarrayobject *self, PyErr_Format(PyExc_ValueError, "attempt to assign sequence of " "size %zd to extended slice of size %zd", other->nbits, slicelength); - goto error; + goto finish; } for (i = 0, j = start; i < slicelength; i++, j += step) setbit(self, j, getbit(other, i)); } res = 0; - error: + finish: if (other_copied) Py_DECREF(other); return res; @@ -2281,23 +2296,7 @@ setslice_bool(bitarrayobject *self, PyOb return -1; adjust_step_positive(slicelength, &start, &stop, &step); - if (step == 1) { - setrange(self, start, stop, vi); - } - else { - const char *table = bitmask_table[IS_BE(self)]; - char *buff = self->ob_item; - Py_ssize_t i; - - if (vi) { - for (i = start; i < stop; i += step) - buff[i >> 3] |= table[i & 7]; - } - else { - for (i = start; i < stop; i += step) - buff[i >> 3] &= ~table[i & 7]; - } - } + set_range(self, start, stop, step, vi); return 0; } @@ -2314,13 +2313,23 @@ delslice(bitarrayobject *self, PyObject adjust_step_positive(slicelength, &start, &stop, &step); if (step > 1) { - Py_ssize_t i, j; - /* set items not to be removed (up to stop) */ - for (i = j = start; i < stop; i++) { - if ((i - start) % step != 0) - setbit(self, j++, getbit(self, i)); + Py_ssize_t i = start + 1, j = start; + + if (step >= 4) { + for (; i < stop; i += step) { + Py_ssize_t length = Py_MIN(step - 1, stop - i); + copy_n(self, j, self, i, length); + j += length; + } } + else { + for (; i < stop; i++) { + if ((i - start) % step != 0) + setbit(self, j++, getbit(self, i)); + } + } + assert(slicelength == 0 || j == stop - slicelength); } return delete_n(self, stop - slicelength, slicelength); } @@ -2343,6 +2352,48 @@ assign_slice(bitarrayobject *self, PyObj return -1; } +/* The following functions are called from assign_mask(). */ + +/* assign mask of bitarray self to bitarray other */ +static int +setmask_bitarray(bitarrayobject *self, bitarrayobject *mask, + bitarrayobject *other) +{ + Py_ssize_t n, i, j; + + assert(self->nbits == mask->nbits); + n = count_span(mask, 0, mask->nbits); /* mask size */ + if (n != other->nbits) { + PyErr_Format(PyExc_IndexError, "attempt to assign mask of size %zd " + "to bitarray of size %zd", n, other->nbits); + return -1; + } + + for (i = j = 0; i < mask->nbits; i++) { + if (getbit(mask, i)) + setbit(self, i, getbit(other, j++)); + } + assert(j == n); + return 0; +} + +/* assign mask of bitarray self to boolean value */ +static int +setmask_bool(bitarrayobject *self, bitarrayobject *mask, PyObject *value) +{ + static char *expr[] = {"a &= ~mask", /* a[mask] = 0 */ + "a |= mask"}; /* a[mask] = 1 */ + int vi; + + if (!conv_pybit(value, &vi)) + return -1; + + PyErr_Format(PyExc_NotImplementedError, "mask assignment to bool not " + "implemented;\n`a[mask] = %d` equivalent to `%s`", + vi, expr[vi]); + return -1; +} + /* delete items in self, specified by mask */ static int delmask(bitarrayobject *self, bitarrayobject *mask) @@ -2354,7 +2405,8 @@ delmask(bitarrayobject *self, bitarrayob if (getbit(mask, i) == 0) /* set items we want to keep */ setbit(self, n++, getbit(self, i)); } - assert(self == mask || n == mask->nbits - count(mask, 0, mask->nbits)); + assert(self == mask || + n == mask->nbits - count_span(mask, 0, mask->nbits)); return resize(self, n); } @@ -2369,41 +2421,18 @@ assign_mask(bitarrayobject *self, bitarr if (value == NULL) return delmask(self, mask); - if (bitarray_Check(value)) { - PyErr_SetString(PyExc_NotImplementedError, - "mask assignment to bitarrays not implemented"); - return -1; - } + if (bitarray_Check(value)) + return setmask_bitarray(self, mask, (bitarrayobject *) value); - if (PyIndex_Check(value)) { - PyErr_SetString(PyExc_NotImplementedError, "mask assignment to " - "bool not implemented - use bitwise operations"); - return -1; - } + if (PyIndex_Check(value)) + return setmask_bool(self, mask, value); PyErr_Format(PyExc_TypeError, "bitarray or int expected for mask " "assignment, not '%s'", Py_TYPE(value)->tp_name); return -1; } -/* assign sequence (of indices) of bitarray self to Boolean value */ -static int -setseq_bool(bitarrayobject *self, PyObject *seq, PyObject *value) -{ - Py_ssize_t n, i, j; - int vi; - - if (!conv_pybit(value, &vi)) - return -1; - - n = PySequence_Size(seq); - for (j = 0; j < n; j++) { - if ((i = index_from_seq(seq, j, self->nbits)) < 0) - return -1; - setbit(self, i, vi); - } - return 0; -} +/* The following functions are called from assign_sequence(). */ /* assign sequence (of indices) of bitarray self to bitarray */ static int @@ -2427,42 +2456,72 @@ setseq_bitarray(bitarrayobject *self, Py for (j = 0; j < n; j++) { if ((i = index_from_seq(seq, j, self->nbits)) < 0) - goto error; + goto finish; setbit(self, i, getbit(other, j)); } res = 0; - error: + finish: if (other_copied) Py_DECREF(other); return res; } +/* assign sequence (of indices) of bitarray self to Boolean value */ +static int +setseq_bool(bitarrayobject *self, PyObject *seq, PyObject *value) +{ + Py_ssize_t n, i, j; + int vi; + + if (!conv_pybit(value, &vi)) + return -1; + + n = PySequence_Size(seq); + for (j = 0; j < n; j++) { + if ((i = index_from_seq(seq, j, self->nbits)) < 0) + return -1; + setbit(self, i, vi); + } + return 0; +} + /* delete items in self, specified by sequence of indices */ static int delsequence(bitarrayobject *self, PyObject *seq) { + const Py_ssize_t nbits = self->nbits; + const Py_ssize_t nseq = PySequence_Size(seq); bitarrayobject *mask; /* temporary bitarray masking items to remove */ - Py_ssize_t nseq, i, j; + Py_ssize_t i, j; int res = -1; - nseq = PySequence_Size(seq); - if (nseq == 0) /* shortcut - sequence is empty - nothing to delete */ - return 0; + /* shortcuts for removing 0 or 1 items to avoid creating mask */ + if (nseq < 2) { + if (nseq == 0) + /* use resize to check for BufferError */ + return resize(self, nbits); - /* create mask bitarray - note that it's endianness is irrelevant */ - mask = newbitarrayobject(&Bitarray_Type, self->nbits, ENDIAN_LITTLE); + assert(nseq == 1); + if ((i = index_from_seq(seq, 0, nbits)) < 0) + return -1; + return delete_n(self, i, 1); + } + + /* create mask bitarray - note that its bit-endianness is irrelevant */ + mask = newbitarrayobject(&Bitarray_Type, nbits, ENDIAN_LITTLE); if (mask == NULL) return -1; - memset(mask->ob_item, 0x00, (size_t) Py_SIZE(mask)); + if (self->ob_item) + memset(mask->ob_item, 0x00, (size_t) Py_SIZE(mask)); /* set indices from sequence in mask */ for (j = 0; j < nseq; j++) { - if ((i = index_from_seq(seq, j, self->nbits)) < 0) - goto error; + if ((i = index_from_seq(seq, j, nbits)) < 0) + goto finish; setbit(mask, i, 1); } res = delmask(self, mask); /* do actual work here */ - error: + finish: Py_DECREF(mask); return res; } @@ -2531,7 +2590,7 @@ bitarray_cpinvert(bitarrayobject *self) if ((res = bitarray_cp(self)) == NULL) return NULL; - invert(res); + invert_span(res, 0, res->nbits); return freeze_if_frozen(res); } @@ -2578,7 +2637,7 @@ bitwise(bitarrayobject *self, bitarrayob } /* Return 0 if both a and b are bitarray objects with same length and - endianness. Otherwise, set exception and return -1. */ + bit-endianness. Otherwise, set exception and return -1. */ static int bitwise_check(PyObject *a, PyObject *b, const char *ostr) { @@ -2588,11 +2647,7 @@ bitwise_check(PyObject *a, PyObject *b, ostr, Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); return -1; } - - if (ensure_eq_size_endian((bitarrayobject *) a, (bitarrayobject *) b) < 0) - return -1; - - return 0; + return ensure_eq_size_endian((bitarrayobject *) a, (bitarrayobject *) b); } #define BITWISE_FUNC(name, inplace, ostr) \ @@ -2633,20 +2688,18 @@ shift(bitarrayobject *self, Py_ssize_t n { const Py_ssize_t nbits = self->nbits; - assert(self->readonly == 0); - if (n >= nbits) { - memset(self->ob_item, 0x00, (size_t) Py_SIZE(self)); - return; - } + assert(n >= 0 && self->readonly == 0); + if (n > nbits) + n = nbits; - assert(0 <= n && n < nbits); + assert(n <= nbits); if (right) { /* rshift */ copy_n(self, n, self, 0, nbits - n); - setrange(self, 0, n, 0); + set_span(self, 0, n, 0); } else { /* lshift */ copy_n(self, 0, self, n, nbits - n); - setrange(self, nbits - n, nbits, 0); + set_span(self, nbits - n, nbits, 0); } } @@ -2708,9 +2761,6 @@ static PyNumberMethods bitarray_as_numbe 0, /* nb_add */ 0, /* nb_subtract */ 0, /* nb_multiply */ -#if PY_MAJOR_VERSION == 2 - 0, /* nb_divide */ -#endif 0, /* nb_remainder */ 0, /* nb_divmod */ 0, /* nb_power */ @@ -2724,22 +2774,12 @@ static PyNumberMethods bitarray_as_numbe (binaryfunc) bitarray_and, /* nb_and */ (binaryfunc) bitarray_xor, /* nb_xor */ (binaryfunc) bitarray_or, /* nb_or */ -#if PY_MAJOR_VERSION == 2 - 0, /* nb_coerce */ -#endif 0, /* nb_int */ 0, /* nb_reserved (was nb_long) */ 0, /* nb_float */ -#if PY_MAJOR_VERSION == 2 - 0, /* nb_oct */ - 0, /* nb_hex */ -#endif 0, /* nb_inplace_add */ 0, /* nb_inplace_subtract */ 0, /* nb_inplace_multiply */ -#if PY_MAJOR_VERSION == 2 - 0, /* nb_inplace_divide */ -#endif 0, /* nb_inplace_remainder */ 0, /* nb_inplace_power */ (binaryfunc) bitarray_ilshift, /* nb_inplace_lshift */ @@ -2751,9 +2791,7 @@ static PyNumberMethods bitarray_as_numbe 0, /* nb_true_divide */ 0, /* nb_inplace_floor_divide */ 0, /* nb_inplace_true_divide */ -#if PY_MAJOR_VERSION == 3 0, /* nb_index */ -#endif }; /************************************************************************** @@ -2810,13 +2848,8 @@ bitarray_encode(bitarrayobject *self, Py value = PyDict_GetItem(codedict, symbol); Py_DECREF(symbol); if (value == NULL) { -#if IS_PY3K PyErr_Format(PyExc_ValueError, "symbol not defined in prefix code: %A", symbol); -#else - PyErr_SetString(PyExc_ValueError, - "symbol not defined in prefix code"); -#endif goto error; } if (check_value(value) < 0 || @@ -2855,7 +2888,7 @@ binode_new(void) { binode *nd; - nd = (binode *) PyMem_Malloc(sizeof(binode)); + nd = PyMem_New(binode, 1); if (nd == NULL) { PyErr_NoMemory(); return NULL; @@ -2911,11 +2944,7 @@ binode_insert_symbol(binode *tree, bitar return 0; ambiguity: -#if IS_PY3K PyErr_Format(PyExc_ValueError, "prefix code ambiguous: %A", symbol); -#else - PyErr_SetString(PyExc_ValueError, "prefix code ambiguous"); -#endif return -1; } @@ -2986,9 +3015,7 @@ binode_to_dict(binode *nd, PyObject *dic if (nd->symbol) { assert(nd->child[0] == NULL && nd->child[1] == NULL); - if (PyDict_SetItem(dict, nd->symbol, (PyObject *) prefix) < 0) - return -1; - return 0; + return PyDict_SetItem(dict, nd->symbol, (PyObject *) prefix); } for (k = 0; k < 2; k++) { @@ -3164,22 +3191,17 @@ static PyMethodDef decodetree_methods[] {"todict", (PyCFunction) decodetree_todict, METH_NOARGS, todict_doc}, {"__sizeof__", (PyCFunction) decodetree_sizeof, METH_NOARGS, 0}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(decodetree_doc, "decodetree(code, /) -> decodetree\n\ \n\ Given a prefix code (a dict mapping symbols to bitarrays),\n\ -create a binary tree object to be passed to `.decode()` or `.iterdecode()`."); +create a binary tree object to be passed to `.decode()`."); static PyTypeObject DecodeTree_Type = { -#if IS_PY3K PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ -#endif "bitarray.decodetree", /* tp_name */ sizeof(decodetreeobject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -3238,43 +3260,6 @@ get_tree(PyObject *obj) return binode_make_tree(obj); } -static PyObject * -bitarray_decode(bitarrayobject *self, PyObject *obj) -{ - binode *tree; - PyObject *list, *symbol; - Py_ssize_t index = 0; - - if ((tree = get_tree(obj)) == NULL) - return NULL; - - if ((list = PyList_New(0)) == NULL) - goto error; - - while ((symbol = binode_traverse(tree, self, &index))) { - if (PyList_Append(list, symbol) < 0) - goto error; - } - if (PyErr_Occurred()) - goto error; - - if (!DecodeTree_Check(obj)) - binode_delete(tree); - return list; - - error: - if (!DecodeTree_Check(obj)) - binode_delete(tree); - Py_XDECREF(list); - return NULL; -} - -PyDoc_STRVAR(decode_doc, -"decode(code, /) -> list\n\ -\n\ -Given a prefix code (a dict mapping symbols to bitarrays, or `decodetree`\n\ -object), decode content of bitarray and return it as a list of symbols."); - /*********************** (bitarray) Decode Iterator ***********************/ typedef struct { @@ -3289,7 +3274,7 @@ static PyTypeObject DecodeIter_Type; /* create a new initialized bitarray decode iterator object */ static PyObject * -bitarray_iterdecode(bitarrayobject *self, PyObject *obj) +bitarray_decode(bitarrayobject *self, PyObject *obj) { decodeiterobject *it; /* iterator to be returned */ binode *tree; @@ -3314,12 +3299,12 @@ bitarray_iterdecode(bitarrayobject *self return (PyObject *) it; } -PyDoc_STRVAR(iterdecode_doc, -"iterdecode(code, /) -> iterator\n\ +PyDoc_STRVAR(decode_doc, +"decode(code, /) -> iterator\n\ \n\ Given a prefix code (a dict mapping symbols to bitarrays, or `decodetree`\n\ object), decode content of bitarray and return an iterator over\n\ -the symbols."); +corresponding symbols."); static PyObject * @@ -3356,12 +3341,7 @@ decodeiter_traverse(decodeiterobject *it } static PyTypeObject DecodeIter_Type = { -#if IS_PY3K PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ -#endif "bitarray.decodeiterator", /* tp_name */ sizeof(decodeiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -3407,7 +3387,7 @@ static PyTypeObject SearchIter_Type; /* create a new initialized bitarray search iterator object */ static PyObject * -bitarray_itersearch(bitarrayobject *self, PyObject *args, PyObject *kwds) +bitarray_search(bitarrayobject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"", "", "", "right", NULL}; Py_ssize_t start = 0, stop = PY_SSIZE_T_MAX; @@ -3422,7 +3402,7 @@ bitarray_itersearch(bitarrayobject *self if (value_sub(sub) < 0) return NULL; - adjust_indices(self->nbits, &start, &stop, 1); + PySlice_AdjustIndices(self->nbits, &start, &stop, 1); it = PyObject_GC_New(searchiterobject, &SearchIter_Type); if (it == NULL) @@ -3439,15 +3419,16 @@ bitarray_itersearch(bitarrayobject *self return (PyObject *) it; } -PyDoc_STRVAR(itersearch_doc, -"itersearch(sub_bitarray, start=0, stop=, /, right=False) -> iterator\n\ +PyDoc_STRVAR(search_doc, +"search(sub_bitarray, start=0, stop=, /, right=False) -> iterator\n\ \n\ Return iterator over indices where sub_bitarray is found, such that\n\ sub_bitarray is contained within `[start:stop]`.\n\ The indices are iterated in ascending order (from lowest to highest),\n\ -unless `right=True`, which will iterate in descending oder (starting with\n\ +unless `right=True`, which will iterate in descending order (starting with\n\ rightmost match)."); + static PyObject * searchiter_next(searchiterobject *it) { @@ -3455,12 +3436,11 @@ searchiter_next(searchiterobject *it) /* range checks necessary in case self changed during iteration */ assert(it->start >= 0); - if (it->start > nbits || it->stop < 0 || it->stop > nbits) { + if (it->start > nbits || it->stop < 0 || it->stop > nbits) return NULL; /* stop iteration */ - } pos = find_obj(it->self, it->sub, it->start, it->stop, it->right); - assert(pos > -2); /* cannot happen - we called value_sub() before */ + assert(pos > -2); /* pos cannot be -2 as we called value_sub() before */ if (pos < 0) /* no more positions -- stop iteration */ return NULL; @@ -3491,12 +3471,7 @@ searchiter_traverse(searchiterobject *it } static PyTypeObject SearchIter_Type = { -#if IS_PY3K PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ -#endif "bitarray.searchiterator", /* tp_name */ sizeof(searchiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -3548,12 +3523,8 @@ static PyMethodDef bitarray_methods[] = count_doc}, {"decode", (PyCFunction) bitarray_decode, METH_O, decode_doc}, - {"iterdecode", (PyCFunction) bitarray_iterdecode, METH_O, - iterdecode_doc}, {"encode", (PyCFunction) bitarray_encode, METH_VARARGS, encode_doc}, - {"endian", (PyCFunction) bitarray_endian, METH_NOARGS, - endian_doc}, {"extend", (PyCFunction) bitarray_extend, METH_O, extend_doc}, {"fill", (PyCFunction) bitarray_fill, METH_NOARGS, @@ -3580,17 +3551,16 @@ static PyMethodDef bitarray_methods[] = remove_doc}, {"reverse", (PyCFunction) bitarray_reverse, METH_NOARGS, reverse_doc}, - {"search", (PyCFunction) bitarray_search, METH_VARARGS, - search_doc}, - {"itersearch", (PyCFunction) bitarray_itersearch, METH_VARARGS | + {"search", (PyCFunction) bitarray_search, METH_VARARGS | METH_KEYWORDS, - itersearch_doc}, + search_doc}, {"setall", (PyCFunction) bitarray_setall, METH_O, setall_doc}, {"sort", (PyCFunction) bitarray_sort, METH_VARARGS | METH_KEYWORDS, sort_doc}, - {"to01", (PyCFunction) bitarray_to01, METH_NOARGS, + {"to01", (PyCFunction) bitarray_to01, METH_VARARGS | + METH_KEYWORDS, to01_doc}, {"tobytes", (PyCFunction) bitarray_tobytes, METH_NOARGS, tobytes_doc}, @@ -3624,7 +3594,7 @@ static PyMethodDef bitarray_methods[] = /* ------------------------ bitarray initialization -------------------- */ -/* Given string 'str', return an integer representing the endianness. +/* Given string 'str', return an integer representing the bit-endianness. If the string is invalid, set exception and return -1. */ static int endian_from_string(const char *str) @@ -3671,7 +3641,7 @@ newbitarray_from_buffer(PyTypeObject *ty obj->weakreflist = NULL; obj->readonly = view.readonly; - obj->buffer = (Py_buffer *) PyMem_Malloc(sizeof(Py_buffer)); + obj->buffer = PyMem_New(Py_buffer, 1); if (obj->buffer == NULL) { PyObject_Del(obj); PyBuffer_Release(&view); @@ -3704,54 +3674,31 @@ newbitarray_from_index(PyTypeObject *typ if ((res = newbitarrayobject(type, nbits, endian)) == NULL) return NULL; - if (init_zero) + if (init_zero && nbits) memset(res->ob_item, 0x00, (size_t) Py_SIZE(res)); return (PyObject *) res; } -/* Return a new bitarray from pickle bytes (created by .__reduce__()). - The head byte specifies the number of pad bits, the remaining bytes - consist of the buffer itself. As the bit-endianness must be known, - we pass this function the actual argument endian_str (and not just - endian, which would default to the default bit-endianness). This way, - we can raise an exception when the endian argument was not provided to - bitarray(). Also, we only call this function with a non-empty PyBytes - object. - */ +/* return new bitarray from bytes-like object */ static PyObject * -newbitarray_from_pickle(PyTypeObject *type, PyObject *bytes, char *endian_str) +newbitarray_from_bytes(PyTypeObject *type, PyObject *buffer, int endian) { bitarrayobject *res; - Py_ssize_t nbytes; - char *str; - unsigned char head; - int endian; + Py_buffer view; - if (endian_str == NULL) { - PyErr_SetString(PyExc_ValueError, "endianness missing for pickle"); + if (PyObject_GetBuffer(buffer, &view, PyBUF_SIMPLE) < 0) return NULL; - } - endian = endian_from_string(endian_str); - assert(endian >= 0); /* endian_str was checked before */ - - assert(PyBytes_Check(bytes)); - nbytes = PyBytes_GET_SIZE(bytes); - assert(nbytes > 0); /* verified in bitarray_new() */ - str = PyBytes_AS_STRING(bytes); - head = *str; - assert((head & 0xf8) == 0); /* verified in bitarray_new() */ - - if (nbytes == 1 && head) - return PyErr_Format(PyExc_ValueError, - "invalid pickle header byte: 0x%02x", head); - res = newbitarrayobject(type, - 8 * (nbytes - 1) - ((Py_ssize_t) head), - endian); - if (res == NULL) + res = newbitarrayobject(type, 8 * view.len, endian); + if (res == NULL) { + PyBuffer_Release(&view); return NULL; - memcpy(res->ob_item, str + 1, (size_t) nbytes - 1); + } + assert(Py_SIZE(res) == view.len); + memcpy(res->ob_item, (char *) view.buf, (size_t) view.len); + + PyBuffer_Release(&view); return (PyObject *) res; } @@ -3764,14 +3711,14 @@ newbitarray_from_pickle(PyTypeObject *ty static PyObject * bitarray_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *initial = Py_None, *buffer = Py_None; + static char *kwlist[] = {"", "endian", "buffer", NULL}; + PyObject *initializer = Py_None, *buffer = Py_None; bitarrayobject *res; char *endian_str = NULL; int endian; - static char *kwlist[] = {"", "endian", "buffer", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OzO:bitarray", kwlist, - &initial, &endian_str, &buffer)) + &initializer, &endian_str, &buffer)) return NULL; if ((endian = endian_from_string(endian_str)) < 0) @@ -3779,45 +3726,42 @@ bitarray_new(PyTypeObject *type, PyObjec /* import buffer */ if (buffer != Py_None && buffer != Py_Ellipsis) { - if (initial != Py_None) { + if (initializer != Py_None) { PyErr_SetString(PyExc_TypeError, - "buffer requires no initial argument"); + "buffer requires no initializer argument"); return NULL; } return newbitarray_from_buffer(type, buffer, endian); } /* no arg / None */ - if (initial == Py_None) + if (initializer == Py_None) return (PyObject *) newbitarrayobject(type, 0, endian); /* bool */ - if (PyBool_Check(initial)) { - PyErr_SetString(PyExc_TypeError, "cannot create bitarray from bool"); + if (PyBool_Check(initializer)) { + PyErr_SetString(PyExc_TypeError, + "cannot create bitarray from 'bool' object"); return NULL; } /* index (a number) */ - if (PyIndex_Check(initial)) - return newbitarray_from_index(type, initial, endian, + if (PyIndex_Check(initializer)) + return newbitarray_from_index(type, initializer, endian, buffer == Py_None); - /* bytes (for pickling) - to be removed, see #206 */ - if (PyBytes_Check(initial) && PyBytes_GET_SIZE(initial) > 0) { - char head = *PyBytes_AS_STRING(initial); - if ((head & 0xf8) == 0) - return newbitarray_from_pickle(type, initial, endian_str); - } - - /* bitarray: use its endianness (when endian argument missing) */ - if (bitarray_Check(initial) && endian_str == NULL) - endian = ((bitarrayobject *) initial)->endian; + /* bytes or bytearray */ + if (PyBytes_Check(initializer) || PyByteArray_Check(initializer)) + return newbitarray_from_bytes(type, initializer, endian); + + /* bitarray: use its bit-endianness when endian argument is None */ + if (bitarray_Check(initializer) && endian_str == NULL) + endian = ((bitarrayobject *) initializer)->endian; /* leave remaining type dispatch to extend method */ - res = newbitarrayobject(type, 0, endian); - if (res == NULL) + if ((res = newbitarrayobject(type, 0, endian)) == NULL) return NULL; - if (extend_dispatch(res, initial) < 0) { + if (extend_dispatch(res, initializer) < 0) { Py_DECREF(res); return NULL; } @@ -3841,7 +3785,7 @@ ssize_richcompare(Py_ssize_t v, Py_ssize static PyObject * richcompare(PyObject *v, PyObject *w, int op) { - Py_ssize_t i = 0, vs, ws, c; + Py_ssize_t i, vs, ws, c; bitarrayobject *va, *wa; char *vb, *wb; @@ -3874,6 +3818,7 @@ richcompare(PyObject *v, PyObject *w, in /* search for the first index where items are different */ c = Py_MIN(vs, ws) / 8; /* common buffer size */ + i = 0; /* byte index */ if (va->endian == wa->endian) { /* equal endianness - skip ahead by comparing bytes directly */ while (i < c && vb[i] == wb[i]) @@ -3930,13 +3875,9 @@ bitarray_iter(bitarrayobject *self) static PyObject * bitarrayiter_next(bitarrayiterobject *it) { - long vi; + if (it->index < it->self->nbits) + return PyLong_FromLong(getbit(it->self, it->index++)); - if (it->index < it->self->nbits) { - vi = getbit(it->self, it->index); - it->index++; - return PyLong_FromLong(vi); - } return NULL; /* stop iteration */ } @@ -3956,12 +3897,7 @@ bitarrayiter_traverse(bitarrayiterobject } static PyTypeObject BitarrayIter_Type = { -#if IS_PY3K PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ -#endif "bitarray.bitarrayiterator", /* tp_name */ sizeof(bitarrayiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -3995,7 +3931,7 @@ static PyTypeObject BitarrayIter_Type = /******************** bitarray buffer export interface ********************/ /* Here we create bitarray_as_buffer for exporting bitarray buffers. - Buffer imports, are NOT handled here, but in newbitarray_from_buffer(). + Buffer imports are handled in newbitarray_from_buffer(). */ static int @@ -4025,60 +3961,7 @@ bitarray_releasebuffer(bitarrayobject *s self->ob_exports--; } -#if PY_MAJOR_VERSION == 2 /* old buffer protocol */ -static Py_ssize_t -bitarray_buffer_getreadbuf(bitarrayobject *self, - Py_ssize_t index, const void **ptr) -{ - if (index != 0) { - PyErr_SetString(PyExc_SystemError, "accessing non-existent segment"); - return -1; - } - *ptr = (void *) self->ob_item; - return Py_SIZE(self); -} - -static Py_ssize_t -bitarray_buffer_getwritebuf(bitarrayobject *self, - Py_ssize_t index, const void **ptr) -{ - if (index != 0) { - PyErr_SetString(PyExc_SystemError, "accessing non-existent segment"); - return -1; - } - *ptr = (void *) self->ob_item; - return Py_SIZE(self); -} - -static Py_ssize_t -bitarray_buffer_getsegcount(bitarrayobject *self, Py_ssize_t *lenp) -{ - if (lenp) - *lenp = Py_SIZE(self); - return 1; -} - -static Py_ssize_t -bitarray_buffer_getcharbuf(bitarrayobject *self, - Py_ssize_t index, const char **ptr) -{ - if (index != 0) { - PyErr_SetString(PyExc_SystemError, "accessing non-existent segment"); - return -1; - } - *ptr = self->ob_item; - return Py_SIZE(self); -} -#endif /* old buffer protocol */ - - static PyBufferProcs bitarray_as_buffer = { -#if PY_MAJOR_VERSION == 2 /* old buffer protocol */ - (readbufferproc) bitarray_buffer_getreadbuf, - (writebufferproc) bitarray_buffer_getwritebuf, - (segcountproc) bitarray_buffer_getsegcount, - (charbufferproc) bitarray_buffer_getcharbuf, -#endif (getbufferproc) bitarray_getbuffer, (releasebufferproc) bitarray_releasebuffer, }; @@ -4089,15 +3972,12 @@ PyDoc_STRVAR(bitarraytype_doc, "bitarray(initializer=0, /, endian='big', buffer=None) -> bitarray\n\ \n\ Return a new bitarray object whose items are bits initialized from\n\ -the optional initial object, and endianness.\n\ -The initializer may be of the following types:\n\ -\n\ -`int`: Create a bitarray of given integer length. The initial values are\n\ -all `0`.\n\ -\n\ -`str`: Create bitarray from a string of `0` and `1`.\n\ -\n\ -`iterable`: Create bitarray from iterable or sequence of integers 0 or 1.\n\ +the optional initializer, and bit-endianness.\n\ +The initializer may be one of the following types:\n\ +a.) `int` bitarray, initialized to zeros, of given length\n\ +b.) `bytes` or `bytearray` to initialize buffer directly\n\ +c.) `str` of 0s and 1s, ignoring whitespace and \"_\"\n\ +d.) iterable of integers 0 or 1.\n\ \n\ Optional keyword arguments:\n\ \n\ @@ -4111,12 +3991,7 @@ read-only or writable, depending on the static PyTypeObject Bitarray_Type = { -#if IS_PY3K PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ -#endif "bitarray.bitarray", /* tp_name */ sizeof(bitarrayobject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -4136,12 +4011,7 @@ static PyTypeObject Bitarray_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &bitarray_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE -#if PY_MAJOR_VERSION == 2 - | Py_TPFLAGS_HAVE_WEAKREFS - | Py_TPFLAGS_HAVE_NEWBUFFER | Py_TPFLAGS_CHECKTYPES -#endif - , /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ bitarraytype_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -4166,6 +4036,46 @@ static PyTypeObject Bitarray_Type = { /***************************** Module functions ***************************/ static PyObject * +bits2bytes(PyObject *module, PyObject *n) +{ + PyObject *zero, *seven, *eight, *a, *b; + int cmp_res; + + if (!PyLong_Check(n)) + return PyErr_Format(PyExc_TypeError, "'int' object expected, " + "got '%s'", Py_TYPE(n)->tp_name); + + zero = PyLong_FromLong(0); + cmp_res = PyObject_RichCompareBool(n, zero, Py_LT); + Py_DECREF(zero); + if (cmp_res < 0) + return NULL; + if (cmp_res) { + PyErr_SetString(PyExc_ValueError, "non-negative int expected"); + return NULL; + } + + seven = PyLong_FromLong(7); + a = PyNumber_Add(n, seven); /* a = n + 7 */ + Py_DECREF(seven); + if (a == NULL) + return NULL; + + eight = PyLong_FromLong(8); + b = PyNumber_FloorDivide(a, eight); /* b = a // 8 */ + Py_DECREF(eight); + Py_DECREF(a); + + return b; +} + +PyDoc_STRVAR(bits2bytes_doc, +"bits2bytes(n, /) -> int\n\ +\n\ +Return the number of bytes necessary to store n bits."); + + +static PyObject * reconstructor(PyObject *module, PyObject *args) { PyTypeObject *type; @@ -4195,13 +4105,14 @@ reconstructor(PyObject *module, PyObject return NULL; nbytes = PyBytes_GET_SIZE(bytes); - if (padbits < 0 || padbits >= 8 || (nbytes == 0 && padbits != 0)) + if (padbits >> 3 || (nbytes == 0 && padbits)) return PyErr_Format(PyExc_ValueError, - "invalid number of padbits: %d", padbits); + "invalid number of pad bits: %d", padbits); res = newbitarrayobject(type, 8 * nbytes - padbits, endian); if (res == NULL) return NULL; + assert(Py_SIZE(res) == nbytes); memcpy(res->ob_item, PyBytes_AS_STRING(bytes), (size_t) nbytes); if (readonly) { set_padbits(res); @@ -4214,15 +4125,15 @@ reconstructor(PyObject *module, PyObject static PyObject * get_default_endian(PyObject *module) { - return Py_BuildValue("s", ENDIAN_STR(default_endian)); + return PyUnicode_FromString(ENDIAN_STR(default_endian)); } PyDoc_STRVAR(get_default_endian_doc, "get_default_endian() -> str\n\ \n\ -Return the default endianness for new bitarray objects being created.\n\ -Unless `_set_default_endian('little')` was called, the default endianness\n\ -is `big`."); +Return the default bit-endianness for new bitarray objects being created.\n\ +Unless `_set_default_endian('little')` was called, the default\n\ +bit-endianness is `big`."); static PyObject * @@ -4235,7 +4146,7 @@ set_default_endian(PyObject *module, PyO return NULL; /* As endian_from_string() might return -1, we have to store its value - in a temporary variable BEFORE setting default_endian. */ + in a temporary variable before setting default_endian. */ if ((t = endian_from_string(endian_str)) < 0) return NULL; default_endian = t; @@ -4286,6 +4197,8 @@ Return tuple containing:\n\ static PyMethodDef module_functions[] = { + {"bits2bytes", (PyCFunction) bits2bytes, METH_O, + bits2bytes_doc}, {"_bitarray_reconstructor", (PyCFunction) reconstructor, METH_VARARGS, reduce_doc}, @@ -4295,12 +4208,11 @@ static PyMethodDef module_functions[] = set_default_endian_doc}, {"_sysinfo", (PyCFunction) sysinfo, METH_NOARGS, sysinfo_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; /******************************* Install Module ***************************/ -#if IS_PY3K /* register bitarray as collections.abc.MutableSequence */ static int register_abc(void) @@ -4328,64 +4240,48 @@ register_abc(void) static PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_bitarray", 0, -1, module_functions, }; -#endif PyMODINIT_FUNC -#if IS_PY3K PyInit__bitarray(void) -#else -init_bitarray(void) -#endif { PyObject *m; - setup_reverse_trans(); + /* setup translation table, which maps each byte to its reversed: + reverse_trans = {0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, ..., 0xff} */ + setup_table(reverse_trans, 'r'); -#if IS_PY3K - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_bitarray", module_functions, 0); -#endif - if (m == NULL) - goto error; + if ((m = PyModule_Create(&moduledef)) == NULL) + return NULL; if (PyType_Ready(&Bitarray_Type) < 0) - goto error; + return NULL; Py_SET_TYPE(&Bitarray_Type, &PyType_Type); Py_INCREF((PyObject *) &Bitarray_Type); PyModule_AddObject(m, "bitarray", (PyObject *) &Bitarray_Type); -#if IS_PY3K if (register_abc() < 0) - goto error; -#endif + return NULL; if (PyType_Ready(&DecodeTree_Type) < 0) - goto error; + return NULL; Py_SET_TYPE(&DecodeTree_Type, &PyType_Type); Py_INCREF((PyObject *) &DecodeTree_Type); PyModule_AddObject(m, "decodetree", (PyObject *) &DecodeTree_Type); if (PyType_Ready(&DecodeIter_Type) < 0) - goto error; + return NULL; Py_SET_TYPE(&DecodeIter_Type, &PyType_Type); if (PyType_Ready(&BitarrayIter_Type) < 0) - goto error; + return NULL; Py_SET_TYPE(&BitarrayIter_Type, &PyType_Type); if (PyType_Ready(&SearchIter_Type) < 0) - goto error; + return NULL; Py_SET_TYPE(&SearchIter_Type, &PyType_Type); PyModule_AddObject(m, "__version__", - Py_BuildValue("s", BITARRAY_VERSION)); -#if IS_PY3K + PyUnicode_FromString(BITARRAY_VERSION)); + return m; - error: - return NULL; -#else - error: - return; -#endif } diff -pruN 2.9.2-1/bitarray/_util.c 3.6.1-1/bitarray/_util.c --- 2.9.2-1/bitarray/_util.c 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/_util.c 2025-08-12 08:35:42.000000000 +0000 @@ -1,5 +1,5 @@ /* - Copyright (c) 2019 - 2024, Ilan Schnell; All Rights Reserved + Copyright (c) 2019 - 2025, Ilan Schnell; All Rights Reserved bitarray is published under the PSF license. This file contains the C implementation of some useful utility functions. @@ -13,24 +13,20 @@ #include "bitarray.h" /* set during module initialization */ -static PyObject *bitarray_type_obj; +static PyTypeObject *bitarray_type; + +#define bitarray_Check(obj) PyObject_TypeCheck((obj), bitarray_type) /* Return 0 if obj is bitarray. If not, set exception and return -1. */ static int ensure_bitarray(PyObject *obj) { - int t; - - t = PyObject_IsInstance(obj, bitarray_type_obj); - if (t < 0) - return -1; + if (bitarray_Check(obj)) + return 0; - if (t == 0) { - PyErr_Format(PyExc_TypeError, "bitarray expected, not '%s'", - Py_TYPE(obj)->tp_name); - return -1; - } - return 0; + PyErr_Format(PyExc_TypeError, "bitarray expected, not '%s'", + Py_TYPE(obj)->tp_name); + return -1; } /* Return new bitarray of length 'nbits', endianness given by the PyObject @@ -47,29 +43,72 @@ new_bitarray(Py_ssize_t nbits, PyObject return NULL; /* equivalent to: res = bitarray(nbits, endian, Ellipsis) */ - res = (bitarrayobject *) PyObject_CallObject(bitarray_type_obj, args); + res = (bitarrayobject *) PyObject_CallObject((PyObject *) bitarray_type, + args); Py_DECREF(args); if (res == NULL) return NULL; assert(res->nbits == nbits && res->readonly == 0 && res->buffer == NULL); assert(-1 <= c && c < 256); - if (c >= 0) + if (c >= 0 && nbits) memset(res->ob_item, c, (size_t) Py_SIZE(res)); return res; } -/* Starting from word index 'i', count remaining population in bitarray - buffer. Equivalent to: a[64 * i:].count() */ +/* Starting from 64-bit word index i, count remaining population + in bitarray a. Basically equivalent to: a[64 * i:].count() */ static Py_ssize_t count_from_word(bitarrayobject *a, Py_ssize_t i) { + const Py_ssize_t nbits = a->nbits; + Py_ssize_t cnt; + assert(i >= 0); - if (64 * i >= a->nbits) + if (64 * i >= nbits) return 0; + cnt = popcnt_words(WBUFF(a) + i, nbits / 64 - i); /* complete words */ + if (nbits % 64) + cnt += popcnt_64(zlw(a)); /* remaining bits */ + return cnt; +} + +/* like resize() but without over-allocation or buffer import/export checks */ +static int +resize_lite(bitarrayobject *self, Py_ssize_t nbits) +{ + const Py_ssize_t newsize = BYTES(nbits); - return popcnt_words(WBUFF(a) + i, a->nbits / 64 - i) + popcnt_64(zlw(a)); + assert(self->allocated >= Py_SIZE(self)); + assert(self->readonly == 0); + assert(self->ob_exports == 0); + assert(self->buffer == NULL); + + /* bypass everything when buffer size hasn't changed */ + if (newsize == Py_SIZE(self)) { + self->nbits = nbits; + return 0; + } + + if (newsize == 0) { + PyMem_Free(self->ob_item); + self->ob_item = NULL; + Py_SET_SIZE(self, 0); + self->allocated = 0; + self->nbits = 0; + return 0; + } + + self->ob_item = PyMem_Realloc(self->ob_item, newsize); + if (self->ob_item == NULL) { + PyErr_NoMemory(); + return -1; + } + Py_SET_SIZE(self, newsize); + self->allocated = newsize; + self->nbits = nbits; + return 0; } /* ---------------------------- zeros / ones --------------------------- */ @@ -79,20 +118,20 @@ zeros(PyObject *module, PyObject *args, { static char *kwlist[] = {"", "endian", NULL}; PyObject *endian = Py_None; - Py_ssize_t nbits; + Py_ssize_t n; if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|O:zeros", kwlist, - &nbits, &endian)) + &n, &endian)) return NULL; - return (PyObject *) new_bitarray(nbits, endian, 0); + return (PyObject *) new_bitarray(n, endian, 0); } PyDoc_STRVAR(zeros_doc, -"zeros(length, /, endian=None) -> bitarray\n\ +"zeros(n, /, endian=None) -> bitarray\n\ \n\ -Create a bitarray of length, with all values 0, and optional\n\ -endianness, which may be 'big', 'little'."); +Create a bitarray of length `n`, with all values `0`, and optional\n\ +bit-endianness (`little` or `big`)."); static PyObject * @@ -100,27 +139,26 @@ ones(PyObject *module, PyObject *args, P { static char *kwlist[] = {"", "endian", NULL}; PyObject *endian = Py_None; - Py_ssize_t nbits; + Py_ssize_t n; if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|O:ones", kwlist, - &nbits, &endian)) + &n, &endian)) return NULL; - return (PyObject *) new_bitarray(nbits, endian, 0xff); + return (PyObject *) new_bitarray(n, endian, 0xff); } PyDoc_STRVAR(ones_doc, -"ones(length, /, endian=None) -> bitarray\n\ +"ones(n, /, endian=None) -> bitarray\n\ \n\ -Create a bitarray of length, with all values 1, and optional\n\ -endianness, which may be 'big', 'little'."); +Create a bitarray of length `n`, with all values `1`, and optional\n\ +bit-endianness (`little` or `big`)."); /* ------------------------------- count_n ----------------------------- */ -/* Return smallest index i for which a.count(vi, 0, i) == n. - When n exceeds the total count, the result is a negative - number; the negative of the total count + 1, which is useful - for displaying error messages. */ +/* Return smallest index i for which a.count(vi, 0, i) == n. When n exceeds + the total count, the result is a negative number; the negative of the + total count + 1, which is useful for displaying error messages. */ static Py_ssize_t count_n_core(bitarrayobject *a, Py_ssize_t n, int vi) { @@ -174,7 +212,7 @@ count_n(PyObject *module, PyObject *args Py_ssize_t n, i; int vi = 1; - if (!PyArg_ParseTuple(args, "O!n|O&:count_n", bitarray_type_obj, + if (!PyArg_ParseTuple(args, "O!n|O&:count_n", bitarray_type, (PyObject *) &a, &n, conv_pybit, &vi)) return NULL; if (n < 0) { @@ -183,13 +221,12 @@ count_n(PyObject *module, PyObject *args } if (n > a->nbits) return PyErr_Format(PyExc_ValueError, "n = %zd larger than bitarray " - "size (len(a) = %zd)", n, a->nbits); + "length %zd", n, a->nbits); i = count_n_core(a, n, vi); /* do actual work here */ - if (i < 0) return PyErr_Format(PyExc_ValueError, "n = %zd exceeds total count " - "(a.count(%d) = %zd)", n, vi, -i - 1); + "(a.count(%d) = %zd)", n, vi, -(i + 1)); return PyLong_FromSsize_t(i); } @@ -215,12 +252,10 @@ parity(PyObject *module, PyObject *obj) a = (bitarrayobject *) obj; wbuff = WBUFF(a); x = zlw(a); - for (i = 0; i < a->nbits / 64; i++) + i = a->nbits / 64; + while (i--) x ^= *wbuff++; - for (i = 32; i > 0; i /= 2) - x ^= x >> i; - x &= 1; - return PyLong_FromLong((long) x); + return PyLong_FromLong(parity_64(x)); } PyDoc_STRVAR(parity_doc, @@ -229,6 +264,91 @@ PyDoc_STRVAR(parity_doc, Return parity of bitarray `a`.\n\ `parity(a)` is equivalent to `a.count() % 2` but more efficient."); + +/* Internal functions, like sum_indices(), but bitarrays are limited in + size. For details see: devel/test_sum_indices.py +*/ +static PyObject * +ssqi(PyObject *module, PyObject *args) +{ + static char count_table[256], sum_table[256], sum_sqr_table[256]; + static int setup = -1; /* endianness of tables */ + bitarrayobject *a; + uint64_t nbytes, i; + uint64_t sm = 0; /* accumulated sum */ + int mode = 1; + + if (!PyArg_ParseTuple(args, "O!|i", bitarray_type, + (PyObject *) &a, &mode)) + return NULL; + if (mode < 1 || mode > 2) + return PyErr_Format(PyExc_ValueError, "unexpected mode %d", mode); + if (((uint64_t) a->nbits) > (mode == 1 ? 6074001000LLU : 3810778LLU)) + return PyErr_Format(PyExc_OverflowError, "ssqi %zd", a->nbits); + + if (setup != a->endian) { + setup_table(count_table, 'c'); + setup_table(sum_table, IS_LE(a) ? 'a' : 'A'); + setup_table(sum_sqr_table, IS_LE(a) ? 's' : 'S'); + setup = a->endian; + } + + nbytes = Py_SIZE(a); + set_padbits(a); + for (i = 0; i < nbytes; i++) { + unsigned char c = a->ob_item[i]; + if (c) { + uint64_t k = count_table[c], z1 = sum_table[c]; + if (mode == 1) { + sm += k * 8LLU * i + z1; + } + else { + uint64_t z2 = (unsigned char) sum_sqr_table[c]; + sm += (k * 64LLU * i + 16LLU * z1) * i + z2; + } + } + } + return PyLong_FromUnsignedLongLong(sm); +} + + +static PyObject * +xor_indices(PyObject *module, PyObject *obj) +{ + static char parity_table[256], xor_table[256]; + static int setup = -1; /* endianness of xor_table */ + bitarrayobject *a; + Py_ssize_t res = 0, nbytes, i; + + if (ensure_bitarray(obj) < 0) + return NULL; + + a = (bitarrayobject *) obj; + nbytes = Py_SIZE(a); + set_padbits(a); + + if (setup != a->endian) { + setup_table(xor_table, IS_LE(a) ? 'x' : 'X'); + setup_table(parity_table, 'p'); + setup = a->endian; + } + + for (i = 0; i < nbytes; i++) { + unsigned char c = a->ob_item[i]; + if (parity_table[c]) + res ^= i << 3; + res ^= xor_table[c]; + } + return PyLong_FromSsize_t(res); +} + +PyDoc_STRVAR(xor_indices_doc, +"xor_indices(a, /) -> int\n\ +\n\ +Return xor reduced indices of all active bits in bitarray `a`.\n\ +This is essentially equivalent to\n\ +`reduce(operator.xor, (i for i, v in enumerate(a) if v))`."); + /* --------------------------- binary functions ------------------------ */ static PyObject * @@ -240,8 +360,8 @@ binary_function(PyObject *args, const ch int rbits; if (!PyArg_ParseTuple(args, format, - bitarray_type_obj, (PyObject *) &a, - bitarray_type_obj, (PyObject *) &b)) + bitarray_type, (PyObject *) &a, + bitarray_type, (PyObject *) &b)) return NULL; if (ensure_eq_size_endian(a, b) < 0) return NULL; @@ -345,9 +465,9 @@ correspond_all(PyObject *module, PyObjec uint64_t u, v, not_u, not_v; int rbits; - if (!PyArg_ParseTuple(args, "O!O!:_correspond_all", - bitarray_type_obj, (PyObject *) &a, - bitarray_type_obj, (PyObject *) &b)) + if (!PyArg_ParseTuple(args, "O!O!:correspond_all", + bitarray_type, (PyObject *) &a, + bitarray_type, (PyObject *) &b)) return NULL; if (ensure_eq_size_endian(a, b) < 0) return NULL; @@ -381,39 +501,115 @@ correspond_all(PyObject *module, PyObjec } PyDoc_STRVAR(correspond_all_doc, -"_correspond_all(a, b, /) -> tuple\n\ +"correspond_all(a, b, /) -> tuple\n\ \n\ Return tuple with counts of: ~a & ~b, ~a & b, a & ~b, a & b"); + +static void +byteswap_core(Py_buffer view, Py_ssize_t n) +{ + char *buff = view.buf; + Py_ssize_t m = view.len / n, k; + + assert(n >= 1 && n * m == view.len); + + if (n == 8 && HAVE_BUILTIN_BSWAP64) { + uint64_t *w = (uint64_t *) buff; + for (k = 0; k < m; k++) + w[k] = builtin_bswap64(w[k]); + } +#if (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5))) + else if (n == 4) { + uint32_t *w = (uint32_t *) buff; + for (k = 0; k < m; k++) + w[k] = __builtin_bswap32(w[k]); + } + else if (n == 2) { + uint16_t *w = (uint16_t *) buff; + for (k = 0; k < m; k++) + w[k] = __builtin_bswap16(w[k]); + } +#endif + else if (n >= 2) { + for (k = 0; k < view.len; k += n) + swap_bytes(buff + k, n); + } +} + +static PyObject * +byteswap(PyObject *module, PyObject *args) +{ + PyObject *buffer; + Py_buffer view; + Py_ssize_t n = 0; + + if (!PyArg_ParseTuple(args, "O|n:byteswap", &buffer, &n)) + return NULL; + + if (n < 0) + return PyErr_Format(PyExc_ValueError, + "positive int expect, got %zd", n); + + if (PyObject_GetBuffer(buffer, &view, PyBUF_SIMPLE | PyBUF_WRITABLE) < 0) + return NULL; + + if (n == 0) + /* avoid division by zero below when view.len = 0 */ + n = Py_MAX(1, view.len); + + if (view.len % n) { + PyErr_Format(PyExc_ValueError, "buffer size %zd not multiple of %zd", + view.len, n); + PyBuffer_Release(&view); + return NULL; + } + + byteswap_core(view, n); + + PyBuffer_Release(&view); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(byteswap_doc, +"byteswap(a, /, n=)\n\ +\n\ +Reverse every `n` consecutive bytes of `a` in-place.\n\ +By default, all bytes are reversed. Note that `n` is not limited to 2, 4\n\ +or 8, but can be any positive integer.\n\ +Also, `a` may be any object that exposes a writable buffer.\n\ +Nothing about this function is specific to bitarray objects."); + /* ---------------------------- serialization -------------------------- */ /* The binary format used here is similar to the one used for pickling - bitarray objects. This format encodes the bit-endianness in the head byte, - whereas the binary pickle blob does not. + bitarray objects. However, this format has a head byte which encodes both + the bit-endianness and the number of pad bits, whereas the binary pickle + blob does not. */ static PyObject * -serialize(PyObject *module, PyObject *a) +serialize(PyObject *module, PyObject *obj) { + bitarrayobject *a; PyObject *result; Py_ssize_t nbytes; char *str; - if (ensure_bitarray(a) < 0) + if (ensure_bitarray(obj) < 0) return NULL; + a = (bitarrayobject *) obj; nbytes = Py_SIZE(a); result = PyBytes_FromStringAndSize(NULL, nbytes + 1); if (result == NULL) return NULL; str = PyBytes_AsString(result); -#define aa ((bitarrayobject *) a) - set_padbits(aa); - *str = (IS_BE(aa) ? 0x10 : 0x00) | ((char) PADBITS(aa)); - memcpy(str + 1, aa->ob_item, (size_t) nbytes); -#undef aa + set_padbits(a); + *str = (IS_BE(a) ? 0x10 : 0x00) | ((char) PADBITS(a)); + memcpy(str + 1, a->ob_item, (size_t) nbytes); return result; } @@ -431,6 +627,7 @@ deserialize(PyObject *module, PyObject * Py_buffer view; bitarrayobject *a; unsigned char head; + Py_ssize_t nbits; if (PyObject_GetBuffer(buffer, &view, PyBUF_SIMPLE) < 0) return NULL; @@ -448,9 +645,8 @@ deserialize(PyObject *module, PyObject * goto error; } /* create bitarray of desired length */ - a = new_bitarray(8 * (view.len - 1) - ((Py_ssize_t) (head & 0x07)), - Py_None, -1); - if (a == NULL) + nbits = 8 * (view.len - 1) - ((Py_ssize_t) (head & 0x07)); + if ((a = new_bitarray(nbits, Py_None, -1)) == NULL) goto error; /* set bit-endianness and buffer */ a->endian = head & 0x10 ? ENDIAN_BIG : ENDIAN_LITTLE; @@ -459,6 +655,7 @@ deserialize(PyObject *module, PyObject * PyBuffer_Release(&view); return (PyObject *) a; + error: PyBuffer_Release(&view); return NULL; @@ -486,101 +683,112 @@ hex_to_int(char c) return -1; } -/* create hexadecimal string from bitarray */ +/* return hexadecimal string from bitarray, + on failure set exception and return NULL */ static char * -ba2hex_core(bitarrayobject *a) +ba2hex_core(bitarrayobject *a, Py_ssize_t group, char *sep) { - const int le = IS_LE(a), be = IS_BE(a); - const size_t strsize = a->nbits / 4; + const int be = IS_BE(a); + size_t strsize = a->nbits / 4, j, nsep; Py_ssize_t i; - char *str; + char *buff = a->ob_item, *str; - assert(a->nbits % 4 == 0 && 2 * Py_SIZE(a) - 1 <= (Py_ssize_t) strsize); + nsep = (group && strsize) ? strlen(sep) : 0; /* 0 means no grouping */ + if (nsep) + strsize += nsep * ((strsize - 1) / group); - str = (char *) PyMem_Malloc(strsize + 1); - if (str == NULL) + str = PyMem_New(char, strsize + 1); + if (str == NULL) { + PyErr_NoMemory(); return NULL; + } + for (i = j = 0; i < a->nbits / 4; i++) { + unsigned char c = buff[i / 2]; - /* translate entire bitarray buffer, even when we have 4 padbits */ - for (i = 0; i < 2 * Py_SIZE(a); i += 2) { - unsigned char c = a->ob_item[i / 2]; - str[i + le] = hexdigits[c >> 4]; - str[i + be] = hexdigits[0x0f & c]; + if (nsep && i && i % group == 0) { + memcpy(str + j, sep, nsep); + j += nsep; + } + str[j++] = hexdigits[(i + be) % 2 ? c >> 4 : 0x0f & c]; } + assert(j == strsize); str[strsize] = 0; /* terminate string */ return str; } static PyObject * -ba2hex(PyObject *module, PyObject *obj) +ba2hex(PyObject *module, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"", "group", "sep", NULL}; PyObject *result; bitarrayobject *a; - char *str; + Py_ssize_t group = 0; + char *sep = " ", *str; - if (ensure_bitarray(obj) < 0) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!|ns:ba2hex", kwlist, + bitarray_type, (PyObject *) &a, + &group, &sep)) return NULL; - a = (bitarrayobject *) obj; - if (a->nbits % 4) - return PyErr_Format(PyExc_ValueError, "bitarray length %zd not " - "multiple of 4", a->nbits); + if (a->nbits % 4) { + PyErr_Format(PyExc_ValueError, "bitarray length %zd not " + "multiple of 4", a->nbits); + return NULL; + } + if (group < 0) { + PyErr_Format(PyExc_ValueError, "non-negative integer " + "expected for group, got: %zd", group); + return NULL; + } - if ((str = ba2hex_core(a)) == NULL) - return PyErr_NoMemory(); + str = ba2hex_core(a, group, sep); + if (str == NULL) + return NULL; -#if IS_PY3K result = PyUnicode_FromString(str); -#else - result = Py_BuildValue("s", str); -#endif PyMem_Free((void *) str); return result; } PyDoc_STRVAR(ba2hex_doc, -"ba2hex(bitarray, /) -> hexstr\n\ +"ba2hex(bitarray, /, group=0, sep=' ') -> hexstr\n\ \n\ Return a string containing the hexadecimal representation of\n\ -the bitarray (which has to be multiple of 4 in length)."); +the bitarray (which has to be multiple of 4 in length).\n\ +When grouped, the string `sep` is inserted between groups\n\ +of `group` characters, default is a space."); -/* Translate hexadecimal digits from 'hexstr' into the bitarray 'a' buffer. +/* Translate hexadecimal digits from 'hexstr' into the bitarray 'a' buffer, + which must be initialized to zeros. Each digit corresponds to 4 bits in the bitarray. Note that the number of hexadecimal digits may be odd. */ static int hex2ba_core(bitarrayobject *a, Py_buffer hexstr) { + const int be = IS_BE(a); const char *str = hexstr.buf; - const Py_ssize_t strsize = hexstr.len; - const int le = IS_LE(a), be = IS_BE(a); - Py_ssize_t i; + Py_ssize_t i = 0, j; - assert(a->nbits == 4 * strsize && str[strsize] == 0); + assert(a->nbits == 4 * hexstr.len); - for (i = 0; i < strsize; i += 2) { - int x, y; + for (j = 0; j < hexstr.len; j++) { + unsigned char c = str[j]; + int x = hex_to_int(c); -#define GETHEX(z, j) \ - z = hex_to_int(str[j]); \ - if (z < 0) { \ - if (j == strsize) { /* ignore terminating NUL */ \ - z = 0; \ - } else { /* invalid character */ \ - unsigned char c = str[j]; \ - PyErr_Format(PyExc_ValueError, \ - "non-hexadecimal digit found, " \ - "got '%c' (0x%02x)", c, c); \ - return -1; \ - } \ + if (x < 0) { + if (Py_UNICODE_ISSPACE(c)) + continue; + PyErr_Format(PyExc_ValueError, "invalid digit found for " + "base16, got '%c' (0x%02x)", c, c); + return -1; } - GETHEX(x, i + le); - GETHEX(y, i + be); -#undef GETHEX - assert(0 <= x && x < 16 && 0 <= y && y < 16); - a->ob_item[i / 2] = x << 4 | y; + assert(x >> 4 == 0); + a->ob_item[i / 2] |= x << 4 * ((i + be) % 2); + i++; } - return 0; + assert(i <= a->nbits); + return resize_lite(a, 4 * i); /* in case we ignored whitespace */ } static PyObject * @@ -595,7 +803,7 @@ hex2ba(PyObject *module, PyObject *args, &hexstr, &endian)) return NULL; - a = new_bitarray(4 * hexstr.len, endian, -1); + a = new_bitarray(4 * hexstr.len, endian, 0); if (a == NULL) goto error; @@ -604,6 +812,7 @@ hex2ba(PyObject *module, PyObject *args, PyBuffer_Release(&hexstr); return (PyObject *) a; + error: PyBuffer_Release(&hexstr); Py_XDECREF((PyObject *) a); @@ -614,49 +823,46 @@ PyDoc_STRVAR(hex2ba_doc, "hex2ba(hexstr, /, endian=None) -> bitarray\n\ \n\ Bitarray of hexadecimal representation. hexstr may contain any number\n\ -(including odd numbers) of hex digits (upper or lower case)."); +(including odd numbers) of hex digits (upper or lower case).\n\ +Whitespace is ignored."); /* ----------------------- base 2, 4, 8, 16, 32, 64 -------------------- */ /* RFC 4648 Base32 alphabet */ static const char base32_alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; -/* standard base 64 alphabet - also described on RFC 4648 */ +/* standard base 64 alphabet - also described in RFC 4648 */ static const char base64_alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +/* Given the length of the base m in [1..6] and a character c, return + its index in the base 2**m alphabet , or -1 if when c is not included. + Note: i >> m checks if i is in range(0, 2**m) */ static int -digit_to_int(int n, char c) +digit_to_int(int m, char c) { + static signed char table[2][128]; + static int setup = 0; int i; - switch (n) { - case 32: /* base 32 */ - if ('A' <= c && c <= 'Z') - return c - 'A'; - if ('2' <= c && c <= '7') - return c - '2' + 26; - break; + assert(1 <= m && m <= 6); + if (m < 5) { /* base 2, 4, 8, 16 */ + i = hex_to_int(c); + return i >> m ? -1 : i; + } - case 64: /* base 64 */ - if ('A' <= c && c <= 'Z') - return c - 'A'; - if ('a' <= c && c <= 'z') - return c - 'a' + 26; - if ('0' <= c && c <= '9') - return c - '0' + 52; - if (c == '+') - return 62; - if (c == '/') - return 63; - break; + if (0x80 & c) /* non-ASCII */ + return -1; - default: /* base 2, 4, 8, 16 */ - i = hex_to_int(c); - if (i < n) - return i; + if (!setup) { + memset(table, 0xff, sizeof table); /* (signed char) 0xff -> -1 */ + for (i = 0; i < 32; i++) + table[0][(unsigned char) base32_alphabet[i]] = i; + for (i = 0; i < 64; i++) + table[1][(unsigned char) base64_alphabet[i]] = i; + setup = 1; } - return -1; + return table[m - 5][(unsigned char) c]; /* base 32, 64 */ } /* return m = log2(n) for m in [1..6] */ @@ -674,14 +880,15 @@ base_to_length(int n) return -1; } -/* create ASCII string from bitarray and base length m */ +/* return ASCII string from bitarray and base length m, + on failure set exception and return NULL */ static char * -ba2base_core(bitarrayobject *a, int m) +ba2base_core(bitarrayobject *a, int m, Py_ssize_t group, char *sep) { const int le = IS_LE(a); - const size_t strsize = a->nbits / m; const char *alphabet; - size_t i; + size_t strsize = a->nbits / m, j, nsep; + Py_ssize_t i; char *str; assert(1 <= m && m <= 6 && a->nbits % m == 0); @@ -692,63 +899,86 @@ ba2base_core(bitarrayobject *a, int m) default: alphabet = hexdigits; } - str = (char *) PyMem_Malloc(strsize + 1); - if (str == NULL) - return NULL; + nsep = (group && strsize) ? strlen(sep) : 0; /* 0 means no grouping */ + if (nsep) + strsize += nsep * ((strsize - 1) / group); - for (i = 0; i < strsize; i++) { - int j, x = 0; + str = PyMem_New(char, strsize + 1); + if (str == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (i = j = 0; i < a->nbits / m; i++) { + int k, x = 0; - for (j = 0; j < m; j++) { - int k = le ? j : (m - j - 1); - x |= getbit(a, i * m + k) << j; + if (nsep && i && i % group == 0) { + memcpy(str + j, sep, nsep); + j += nsep; + } + for (k = 0; k < m; k++) { + int q = le ? k : (m - k - 1); + x |= getbit(a, i * m + k) << q; } - str[i] = alphabet[x]; + assert(x >> m == 0); + str[j++] = alphabet[x]; } + assert(j == strsize); str[strsize] = 0; /* terminate string */ return str; } static PyObject * -ba2base(PyObject *module, PyObject *args) +ba2base(PyObject *module, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"", "", "group", "sep", NULL}; bitarrayobject *a; PyObject *result; - char *str; + Py_ssize_t group = 0; + char *sep = " ", *str; int n, m; - if (!PyArg_ParseTuple(args, "iO!:ba2base", &n, - bitarray_type_obj, (PyObject *) &a)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iO!|ns:ba2base", kwlist, + &n, bitarray_type, (PyObject *) &a, + &group, &sep)) return NULL; if ((m = base_to_length(n)) < 0) return NULL; - if (a->nbits % m) - return PyErr_Format(PyExc_ValueError, - "bitarray length must be multiple of %d", m); + if (a->nbits % m) { + PyErr_Format(PyExc_ValueError, "bitarray length %zd not " + "multiple of %d", a->nbits, m); + return NULL; + } + if (group < 0) { + PyErr_Format(PyExc_ValueError, "non-negative integer " + "expected for group, got: %zd", group); + return NULL; + } + + if (m == 4) + str = ba2hex_core(a, group, sep); + else + str = ba2base_core(a, m, group, sep); - str = m == 4 ? ba2hex_core(a) : ba2base_core(a, m); if (str == NULL) - return PyErr_NoMemory(); + return NULL; -#if IS_PY3K result = PyUnicode_FromString(str); -#else - result = Py_BuildValue("s", str); -#endif PyMem_Free((void *) str); return result; } PyDoc_STRVAR(ba2base_doc, -"ba2base(n, bitarray, /) -> str\n\ +"ba2base(n, bitarray, /, group=0, sep=' ') -> str\n\ \n\ Return a string containing the base `n` ASCII representation of\n\ the bitarray. Allowed values for `n` are 2, 4, 8, 16, 32 and 64.\n\ The bitarray has to be multiple of length 1, 2, 3, 4, 5 or 6 respectively.\n\ For `n=32` the RFC 4648 Base32 alphabet is used, and for `n=64` the\n\ -standard base 64 alphabet is used."); +standard base 64 alphabet is used.\n\ +When grouped, the string `sep` is inserted between groups\n\ +of `group` characters, default is a space."); /* translate ASCII digits (with base length m) into bitarray buffer */ @@ -756,26 +986,30 @@ static int base2ba_core(bitarrayobject *a, Py_buffer asciistr, int m) { const char *str = asciistr.buf; - const int le = IS_LE(a), n = 1 << m; + const int le = IS_LE(a); Py_ssize_t i = 0, j; assert(a->nbits == asciistr.len * m && 1 <= m && m <= 6); for (j = 0; j < asciistr.len; j++) { unsigned char c = str[j]; - int j, d = digit_to_int(n, c); + int k, x = digit_to_int(m, c); - if (d < 0) { + if (x < 0) { + if (Py_UNICODE_ISSPACE(c)) + continue; PyErr_Format(PyExc_ValueError, "invalid digit found for " - "base %d, got '%c' (0x%02x)", n, c, c); + "base%d, got '%c' (0x%02x)", 1 << m, c, c); return -1; } - for (j = 0; j < m; j++) { - int k = le ? j : (m - j - 1); - setbit(a, i++, d & (1 << k)); + assert(x >> m == 0); + for (k = 0; k < m; k++) { + int q = le ? k : (m - k - 1); + setbit(a, i++, x & (1 << q)); } } - return 0; + assert(i <= a->nbits); + return resize_lite(a, i); /* in case we ignored whitespace */ } static PyObject * @@ -785,7 +1019,7 @@ base2ba(PyObject *module, PyObject *args PyObject *endian = Py_None; Py_buffer asciistr; bitarrayobject *a = NULL; - int m, n, t; /* n = 2^m */ + int m, n, t; /* n = 2**m */ if (!PyArg_ParseTupleAndKeywords(args, kwds, "is*|O:base2ba", kwlist, &n, &asciistr, &endian)) @@ -794,16 +1028,17 @@ base2ba(PyObject *module, PyObject *args if ((m = base_to_length(n)) < 0) goto error; - a = new_bitarray(m * asciistr.len, endian, -1); + a = new_bitarray(m * asciistr.len, endian, m == 4 ? 0 : -1); if (a == NULL) goto error; - t = m == 4 ? hex2ba_core(a, asciistr) : base2ba_core(a, asciistr, m); + t = (m == 4) ? hex2ba_core(a, asciistr) : base2ba_core(a, asciistr, m); if (t < 0) goto error; PyBuffer_Release(&asciistr); return (PyObject *) a; + error: PyBuffer_Release(&asciistr); Py_XDECREF((PyObject *) a); @@ -816,103 +1051,46 @@ PyDoc_STRVAR(base2ba_doc, Bitarray of base `n` ASCII representation.\n\ Allowed values for `n` are 2, 4, 8, 16, 32 and 64.\n\ For `n=32` the RFC 4648 Base32 alphabet is used, and for `n=64` the\n\ -standard base 64 alphabet is used."); +standard base 64 alphabet is used. Whitespace is ignored."); /* ------------------------ utility C functions ------------------------ */ -/* like resize() */ -static int -resize_lite(bitarrayobject *self, Py_ssize_t nbits) -{ - const Py_ssize_t allocated = self->allocated, size = Py_SIZE(self); - const Py_ssize_t newsize = BYTES(nbits); - size_t new_allocated; - - assert(allocated >= size && size == BYTES(self->nbits)); - assert(self->ob_exports == 0); - assert(self->buffer == NULL); - assert(self->readonly == 0); - - if (newsize == size) { - self->nbits = nbits; - return 0; - } - - if (allocated >= newsize && newsize >= (allocated >> 1)) { - assert(self->ob_item != NULL || newsize == 0); - Py_SET_SIZE(self, newsize); - self->nbits = nbits; - return 0; - } - - if (newsize == 0) { - PyMem_Free(self->ob_item); - self->ob_item = NULL; - Py_SET_SIZE(self, 0); - self->allocated = 0; - self->nbits = 0; - return 0; - } - - new_allocated = ((size_t) newsize + (newsize >> 4) + - (newsize < 8 ? 3 : 7)) & ~(size_t) 3; - - if (newsize - size > (Py_ssize_t) new_allocated - newsize) - new_allocated = ((size_t) newsize + 3) & ~(size_t) 3; - - assert(new_allocated >= (size_t) newsize); - self->ob_item = PyMem_Realloc(self->ob_item, new_allocated); - if (self->ob_item == NULL) { - PyErr_NoMemory(); - return -1; - } - Py_SET_SIZE(self, newsize); - self->allocated = new_allocated; - self->nbits = nbits; - return 0; -} - -/* Consume one byte from iteratior and return it's value as an integer - in range(256). On failure, set an exception and return -1. */ +/* Consume one item from iterator and return its value as an integer + in range(256). On failure, set an exception and return -1. */ static int next_char(PyObject *iter) { PyObject *item; - unsigned char c; + Py_ssize_t v; - item = PyIter_Next(iter); - if (item == NULL) { + if ((item = PyIter_Next(iter)) == NULL) { if (PyErr_Occurred()) /* from PyIter_Next() */ return -1; - PyErr_SetString(PyExc_ValueError, "unexpected end of stream"); + PyErr_SetString(PyExc_StopIteration, "unexpected end of stream"); return -1; } -#if IS_PY3K - if (PyLong_Check(item)) - c = (unsigned char) PyLong_AsLong(item); -#else - if (PyBytes_Check(item)) - c = (unsigned char) *PyBytes_AS_STRING(item); -#endif - else { - PyErr_Format(PyExc_TypeError, "int iterator expected, " - "got '%s' element", Py_TYPE(item)->tp_name); - Py_DECREF(item); + v = PyNumber_AsSsize_t(item, NULL); + Py_DECREF(item); + if (v == -1 && PyErr_Occurred()) + return -1; + + if (v >> 8) { + PyErr_Format(PyExc_ValueError, + "byte must be in range(0, 256), got: %zd", v); return -1; } - Py_DECREF(item); - return (int) c; + return (int) v; } -/* write n bytes (into buffer str) representing integer i (using - little endian byte-order) */ +/* write n bytes (into buffer str) representing non-negative integer i, + using little endian byte-order */ static void write_n(char *str, int n, Py_ssize_t i) { int len = 0; - assert(n <= 8); + assert(n <= 8 && i >= 0); while (len < n) { str[len++] = (char) i & 0xff; i >>= 8; @@ -920,14 +1098,15 @@ write_n(char *str, int n, Py_ssize_t i) assert(i == 0); } -/* read n bytes from iter and return corresponding positive integer, +/* read n bytes from iter and return corresponding non-negative integer, using little endian byte-order */ static Py_ssize_t -read_n(int n, PyObject *iter) +read_n(PyObject *iter, int n) { Py_ssize_t i = 0; int j, c; + assert(PyIter_Check(iter)); assert(n <= 8); for (j = 0; j < n; j++) { if ((c = next_char(iter)) < 0) @@ -942,7 +1121,7 @@ read_n(int n, PyObject *iter) return i; } -/* return number of bytes necessary to represent i */ +/* return number of bytes necessary to represent non-negative integer i */ static int byte_length(Py_ssize_t i) { @@ -956,26 +1135,37 @@ byte_length(Py_ssize_t i) return n; } -/* ---------------------- sparse compressed bitarray ------------------- +/*********************** sparse bitarray compression ***************** * * see also: doc/sparse_compression.rst */ -/* Bitarray buffer size (in bytes) that can be indexed by n bytes. E.g.: - with 1 byte you can index 256 bits which have a buffer size of 32 bytes. - BSI(1) = 32, BSI(2) = 8_192, BSI(3) = 2_097_152, BSI(4) = 536_870_912 */ +/* Bitarray buffer size (in bytes) that can be indexed by n bytes. + + A sparse block of type n uses n bytes to index each bit. + The decoded block size, that is the bitarray buffer size corresponding + to a sparse block of type n, is given by BSI(n). Using 1 byte we can + index 256 bits which have a decoded block size of 32 bytes: + + BSI(1) = 32 (BSI = Buffer Size Indexable) + + Moving from block type n to n + 1 multiplies the decoded block size + by a factor of 256 (as the extra byte can index 256 times as much): + + BSI(n + 1) = 256 * BSI(n) +*/ #define BSI(n) (((Py_ssize_t) 1) << (8 * (n) - 3)) -/* segment size in bytes - Although of little practical value, the code - below will also work for SEGSIZE values of: 8, 16 and 32 - BSI(1) = 32 must be divisible by SEGSIZE. - SEGSIZE must also be a multiple of the word size (sizeof(uint64_t) = 8). - The size 32 is rooted in the fact that a bitarray of 32 bytes (256 bits) - can be indexed with one index byte (BSI(1) = 32). */ +/* segment size in bytes (not to be confused with block size, see below) + + Although of little practical value, the code will work for + values of SEGSIZE of: 8, 16, 32 + BSI(n) must be divisible by SEGSIZE. So, 32 must be divisible by SEGSIZE. + SEGSIZE must also be divisible by the word size sizeof(uint64_t) = 8. */ #define SEGSIZE 32 -/* number of segments for given number of bits */ -#define NSEG(nbits) (((nbits) + 8 * SEGSIZE - 1) / (8 * SEGSIZE)) +/* number of segments for given bitarray */ +#define NSEG(self) ((Py_SIZE(self) + SEGSIZE - 1) / SEGSIZE) /* Calculate an array with the running totals (rts) for 256 bit segments. Note that we call these "segments", as opposed to "blocks", in order to @@ -991,18 +1181,18 @@ byte_length(Py_ssize_t i) In this example we have a bitarray of length nbits = 987. Note that: - * The number of segments is given by NSEG(nbits). - Here we have 4 segments: NSEG(nbits) = NSEG(987) = 4 + * The number of segments is given by NSEG(self). + Here we have 4 segments: NSEG(self) = 4 - * The rts array has always NSEG(nbits) + 1 elements, such that - last element is always indexed by NSEG(nbits). + * The rts array has always NSEG(self) + 1 elements, such that + last element is always indexed by NSEG(self). * The element rts[0] is always zero. - * The last element rts[NSEG(nbits)] is always the total count. - Here: rts[NSEG(nbits)] = rts[NSEG(987)] = rts[4] = 12 + * The last element rts[NSEG(self)] is always the total count. + Here: rts[NSEG(self)] = rts[4] = 12 - * The last segment may be partial. In that case, it's size it given + * The last segment may be partial. In that case, its size it given by nbits % 256. Here: nbits % 256 = 987 % 256 = 219 As each segment (at large) covers 256 bits (32 bytes), and each element @@ -1015,29 +1205,27 @@ byte_length(Py_ssize_t i) The function sc_write_indices() also takes advantage of the running totals array. It loops over segments and skips to the next segment as - soon as the count the current segment is reached. + soon as the index count (population) of the current segment is reached. */ static Py_ssize_t * -sc_calc_rts(bitarrayobject *a) +sc_rts(bitarrayobject *a) { - const Py_ssize_t n_seg = NSEG(a->nbits); /* total number of segments */ + const Py_ssize_t n_seg = NSEG(a); /* total number of segments */ const Py_ssize_t c_seg = a->nbits / (8 * SEGSIZE); /* complete segments */ char zeros[SEGSIZE]; /* segment with only zeros */ + char *buff = a->ob_item; /* buffer in current segment */ Py_ssize_t cnt = 0; /* current count */ - char *buff; /* buffer in current segment */ Py_ssize_t *res, m; memset(zeros, 0x00, SEGSIZE); - res = (Py_ssize_t *) PyMem_Malloc((size_t) - sizeof(Py_ssize_t) * (n_seg + 1)); - if (res == NULL) - return (Py_ssize_t *) PyErr_NoMemory(); - - for (m = 0; m < c_seg; m++) { /* loop all complete segments */ + res = PyMem_New(Py_ssize_t, n_seg + 1); + if (res == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (m = 0; m < c_seg; m++, buff += SEGSIZE) { /* complete segments */ res[m] = cnt; - buff = a->ob_item + m * SEGSIZE; - assert((m + 1) * SEGSIZE <= Py_SIZE(a)); - + assert(buff + SEGSIZE <= a->ob_item + Py_SIZE(a)); if (memcmp(buff, zeros, SEGSIZE)) /* segment has not only zeros */ cnt += popcnt_words((uint64_t *) buff, SEGSIZE / 8); } @@ -1050,79 +1238,63 @@ sc_calc_rts(bitarrayobject *a) return res; } -/* expose sc_calc_rts() to Python during debug mode for testing */ +/* expose sc_rts() to Python during debug mode for testing */ #ifndef NDEBUG static PyObject * -sc_rts(PyObject *module, PyObject *obj) +module_sc_rts(PyObject *module, PyObject *obj) { PyObject *list; bitarrayobject *a; Py_ssize_t *rts, i; - if (ensure_bitarray(obj) < 0) - return NULL; - + assert(bitarray_Check(obj)); a = (bitarrayobject *) obj; - if ((rts = sc_calc_rts(a)) == NULL) + if ((rts = sc_rts(a)) == NULL) return NULL; - if ((list = PyList_New(NSEG(a->nbits) + 1)) == NULL) - return NULL; + if ((list = PyList_New(NSEG(a) + 1)) == NULL) + goto error; - for (i = 0; i <= NSEG(a->nbits); i++) { + for (i = 0; i <= NSEG(a); i++) { PyObject *item = PyLong_FromSsize_t(rts[i]); - if (item == NULL) { - Py_DECREF(list); - return NULL; - } + if (item == NULL) + goto error; PyList_SET_ITEM(list, i, item); } PyMem_Free(rts); return list; + error: + Py_XDECREF(list); + PyMem_Free(rts); + return NULL; } #endif /* NDEBUG */ -/* Equivalent to the Python expression: +/* Return population count for the decoded block size of type n. + Roughly equivalent to the Python expression: a.count(1, 8 * offset, 8 * offset + (1 << (8 * n))) - The offset is required to be multiple of 32 (the segment size), as this - functions makes use of running totals (stored in Py_ssize_t array rts). */ + The offset must be divisible by SEGSIZE, as this functions makes use of + running totals, stored in rts[]. */ static Py_ssize_t sc_count(bitarrayobject *a, Py_ssize_t *rts, Py_ssize_t offset, int n) { - Py_ssize_t nbits; - - assert(offset % SEGSIZE == 0 && n > 0); - if (offset >= Py_SIZE(a)) - return 0; - - /* The desired number of bits to count up to (limited by remaining - bitarray size) is given by: + const Py_ssize_t i = offset / SEGSIZE; /* indices into rts[] */ + const Py_ssize_t j = Py_MIN(i + BSI(n) / SEGSIZE, NSEG(a)); - nbits = Py_MIN(8 * BSI(n), a->nbits - 8 * offset); - - However, on 32-bit machines this will fail for n=4 because 8 * BSI(4) - is 1 << 32. This is problematic, as 32-bit machines can address - at least partially filled type 4 blocks). Therefore, we first - limit BSI(n) by the buffer size before multiplying 8. */ - nbits = Py_MIN(8 * Py_MIN(BSI(n), Py_SIZE(a)), a->nbits - 8 * offset); - assert(nbits >= 0); - - offset /= SEGSIZE; /* offset in terms of segments now */ - assert(NSEG(nbits) + offset <= NSEG(a->nbits)); - - return rts[NSEG(nbits) + offset] - rts[offset]; + assert(offset % SEGSIZE == 0 && 1 <= n && n <= 4); + assert(0 <= i && i <= j && j <= NSEG(a)); + return rts[j] - rts[i]; } -/* Calculate number of bytes [1..4096] of the raw block starting at offset, - encode the block (write the header and copy the bytes into the encode - buffer str), and return the number of raw bytes. +/* Write a raw block, and return number of bytes copied. + Note that the encoded block size is the return value + 1 (the head byte). + The header byte is in range(0x01, 0xa0). - range(0x01, 0x20) refers to number of raw bytes directly. - range(0x20, 0xa0) refers to number of (32 byte) segments. - Note that the encoded block size is the return value + 1. */ + * range(0x01, 0x20) number of raw bytes + * range(0x20, 0xa0) number of 32-byte segments */ static int sc_write_raw(char *str, bitarrayobject *a, Py_ssize_t *rts, Py_ssize_t offset) { @@ -1131,18 +1303,18 @@ sc_write_raw(char *str, bitarrayobject * assert(nbytes > 0); if (k == 32) { - /* We already know the first 32 bytes are better represented using - raw bytes (otherwise this function wouldn't have been called). - Now also check the next 127 segments. */ - while (k < 32 * 128 && k + 32 <= nbytes && - 32 <= sc_count(a, rts, offset + k, 1)) + /* The first 32 bytes are better represented using raw bytes. + Now check up to the next 127 (32-byte) segments. */ + const Py_ssize_t kmax = Py_MIN(32 * 128, nbytes); + while (k + 32 <= kmax && sc_count(a, rts, offset + k, 1) >= 32) k += 32; } assert(0 < k && k <= 32 * 128 && k <= nbytes); - assert((k >= 32 || k == nbytes) && (k <= 32 || k % 32 == 0)); + assert(k >= 32 || k == nbytes); + assert(k <= 32 || k % 32 == 0); /* block header */ - *str = (char) (k <= 32 ? k : (k / 32) + 31); + *str = (char) (k <= 32 ? k : k / 32 + 31); /* block data */ assert(offset + k <= Py_SIZE(a)); @@ -1162,29 +1334,30 @@ sc_write_indices(char *str, bitarrayobje { const char *str_stop = str + n * k; /* stop position in buffer 'str' */ const char *buff = a->ob_item + offset; - Py_ssize_t m, i; + Py_ssize_t m; - assert(1 <= n && n <= 4); assert(0 < k && k < 256); /* note that k cannot be 0 in this function */ assert(k == sc_count(a, rts, offset, n)); /* see above */ - assert(offset % SEGSIZE == 0); rts += offset / SEGSIZE; /* rts index relative to offset now */ for (m = 0;;) { /* loop segments */ - int j, ni; + Py_ssize_t i, ni; - assert(m + offset / SEGSIZE < NSEG(a->nbits)); - ni = (int) (rts[m + 1] - rts[m]); /* indices in this segment */ - if (ni == 0) + assert(m + offset / SEGSIZE < NSEG(a)); + /* number of indices in this segment, i.e. the segment population */ + if ((ni = rts[m + 1] - rts[m]) == 0) goto next_segment; for (i = m * SEGSIZE;; i++) { /* loop bytes in segment */ + int j; + assert(i < (m + 1) * SEGSIZE && i + offset < Py_SIZE(a)); - if (buff[i] == 0) + if (buff[i] == 0x00) continue; - for (j = 0; j < 8; j++) /* loop bits */ + for (j = 0; j < 8; j++) { /* loop bits */ + assert(8 * (offset + i) + j < a->nbits); if (buff[i] & BITMASK(a, j)) { write_n(str, n, 8 * i + j); str += n; @@ -1195,15 +1368,18 @@ sc_write_indices(char *str, bitarrayobje goto next_segment; } } + } } + Py_UNREACHABLE(); next_segment: m++; } Py_UNREACHABLE(); } -/* Write one sparse block (from 'offset', and up to 'k' one bits) of type 'n'. - Return number of bytes written to buffer 'str' (encoded block size). */ +/* Write a sparse block of type 'n' with 'k' indices. + Return number of bytes written to buffer 'str' (encoded block size). + Note that the decoded block size is always BSI(n). */ static Py_ssize_t sc_write_sparse(char *str, bitarrayobject *a, Py_ssize_t *rts, Py_ssize_t offset, int n, int k) @@ -1214,15 +1390,15 @@ sc_write_sparse(char *str, bitarrayobjec assert(0 <= k && k < 256); /* write block header */ - if (n == 1) { /* type 1 - single byte for each position */ + if (n == 1) { /* type 1 - one header byte */ assert(k < 32); - str[len++] = (char) (0xa0 + k); + str[len++] = (char) (0xa0 + k); /* index count in 0xa0 .. 0xbf */ } - else { /* type 2, 3, 4 - n bytes for each positions */ - str[len++] = (char) (0xc0 + n); - str[len++] = (char) k; + else { /* type 2, 3, 4 - two header bytes */ + str[len++] = (char) (0xc0 + n); /* block type */ + str[len++] = (char) k; /* index count */ } - if (k == 0) /* no index bytes */ + if (k == 0) /* no index bytes - sc_write_sparse() does not allow k = 0 */ return len; /* write block data - k indices, n bytes per index */ @@ -1230,8 +1406,9 @@ sc_write_sparse(char *str, bitarrayobjec return len + n * k; } -/* Encode one block (starting at offset) and return offset increment. - The output is written into str buffer and len is increased. +/* Encode one block (starting at offset) and return offset increment, + i.e. the decoded block size. + The output is written into buffer 'str' and 'len' is increased. Notes: @@ -1239,33 +1416,50 @@ sc_write_sparse(char *str, bitarrayobjec Hence, if the bit count of the first 32 bytes of the bitarray buffer is greater or equal to 32, we choose a raw block (type 0). - - If a raw block is used, we check if the next 127 segments + - Arguably, n index bytes always take up as much space as n raw bytes. + So what makes 32 special here? A bitarray with a 32 byte buffer has + 256 items (bits), and these 256 bits can be addressed using one + index byte. That is, BSI(1) = 32, see above. + This is also the reason, why the index count of type 1 blocks is limited + to below 32. + + - If a raw block is used, we check if up to the next 127 32-byte segments are also suitable for raw encoding, see sc_write_raw(). Therefore, we have type 0 blocks with up to 128 * 32 = 4096 raw bytes. - - Now we decide which sparse block type to use. We do this by - first calculating the population count for the bitarray buffer size of - the NEXT block type. If the this count is larger than 255 (too large - for the count byte) we have to stick with the current type. - Otherwise we compare the encoded sizes of (a) sticking with the current - type n, and (b) moving to the next type n+1. These sizes are calculated - as follows: - - (a) Although we consider sticking with the current type n, we are - looking at the population for the next type block size. We - have to calculate the encoded size of ALL the type n blocks - which would otherwise just be a single type n+1 block. This - size is: - - header size * number of blocks + n * population - - (b) Calculating the encoded size of a single block of type n+1 is - much easier: - - header size + (n + 1) * population - - As we only need to know which of these sizes is bigger, we can - substract n * population from both sizes. + - If no raw block was used, we move on to deciding which type of sparse + representation to use. Starting at type n = 1, we do this by first + calculating the population count for the decoded block size of + the *next* block type n+1. + If this population is larger than 255 (too large for the count byte) we + have to stick with type n. + Otherwise we compare the encoded sizes of (a) sticking with + many (up to 256) blocks of type n, and (b) moving to a single block of + type n+1. These sizes are calculated as follows: + + (a) The encoded size of many blocks of type n is given by: + + header_size * number_of_blocks + n * population + + Regardless of the exact index count for each block, the total size + of the index bytes is (n * population), as all blocks are of type n. + The number_of_blocks is 256 (unless limited by the bitarray size). + The header_size is only 1 byte for type 1 and 2 bytes otherwise. + + (b) The encoded size of a single block of type n+1 is: + + header_size + (n + 1) * population + + As n >= 1, the header_size will is always 2 bytes here. + + - As we only need to know which of these sizes is bigger, we can + subtract (n * population) from both sizes. Hence, the costs are: + (a) header_size * number_of_blocks + (b) header_size + population + + The question of whether to choose type n or type n+1 ultimately comes + down to whether the additional byte for each index is less expensive than + having additional headers. */ static Py_ssize_t sc_encode_block(char *str, Py_ssize_t *len, @@ -1277,31 +1471,34 @@ sc_encode_block(char *str, Py_ssize_t *l assert(nbytes > 0); count = (int) sc_count(a, rts, offset, 1); - /* are there fewer or equal raw bytes than index bytes */ - if (Py_MIN(32, nbytes) <= count) { /* type 0 - raw bytes */ + /* the number of index bytes exceeds the number of raw bytes */ + if (count >= Py_MIN(32, nbytes)) { /* type 0 - raw bytes */ int k = sc_write_raw(str + *len, a, rts, offset); *len += 1 + k; return k; } for (n = 1; n < 4; n++) { - Py_ssize_t next_count, size_a, size_b; + Py_ssize_t next_count, nblocks, cost_a, cost_b; /* population for next block type n+1 */ next_count = sc_count(a, rts, offset, n + 1); - if (next_count >= 256) - /* too many index bytes for next block type n+1 */ + if (next_count > 255) + /* too many index bytes for next block type n+1 - use type n */ break; - /* encoded size of (up to 256) blocks of type n */ - size_a = (n == 1 ? 1 : 2) * Py_MIN(256, (nbytes - 1) / BSI(n) + 1); - /* encoded size of (a single) block of type n+1 */ - size_b = 2 + next_count; + /* number of blocks of type n */ + nblocks = Py_MIN(256, (nbytes - 1) / BSI(n) + 1); + /* cost of nblocks blocks of type n */ + cost_a = (n == 1 ? 1 : 2) * nblocks; + /* cost of a single block of type n+1 */ + cost_b = 2 + next_count; - if (size_a <= size_b) - /* next block type n+1 is not smaller - use block type n */ + if (cost_b >= cost_a) + /* block type n+1 is equally or more expensive - use type n */ break; + /* we proceed with type n+1 - we already calculated its population */ count = (int) next_count; } @@ -1309,6 +1506,7 @@ sc_encode_block(char *str, Py_ssize_t *l return BSI(n); } +/* write header and return number or bytes written to buffer 'str' */ static int sc_encode_header(char *str, bitarrayobject *a) { @@ -1321,6 +1519,10 @@ sc_encode_header(char *str, bitarrayobje return 1 + len; } +/* initial size of output buffer, and amount by which we increase our + allocation if we run out */ +#define ALLOC_SIZE 32768 + static PyObject * sc_encode(PyObject *module, PyObject *obj) { @@ -1329,38 +1531,36 @@ sc_encode(PyObject *module, PyObject *ob Py_ssize_t len = 0; /* bytes written into output buffer */ bitarrayobject *a; Py_ssize_t offset = 0; /* block offset into bitarray a in bytes */ - Py_ssize_t *rts; /* running totals for 256 bit segments */ - Py_ssize_t t; /* total population count of bitarray a */ + Py_ssize_t *rts; /* running totals of segments */ + Py_ssize_t total; /* total population count of bitarray */ if (ensure_bitarray(obj) < 0) return NULL; a = (bitarrayobject *) obj; set_padbits(a); - if ((rts = sc_calc_rts(a)) == NULL) + if ((rts = sc_rts(a)) == NULL) return NULL; - out = PyBytes_FromStringAndSize(NULL, 32768); - if (out == NULL) - return NULL; + if ((out = PyBytes_FromStringAndSize(NULL, ALLOC_SIZE)) == NULL) + goto error; str = PyBytes_AS_STRING(out); len += sc_encode_header(str, a); - t = rts[NSEG(a->nbits)]; + total = rts[NSEG(a)]; /* encode blocks as long as we haven't reached the end of the bitarray and haven't reached the total population count yet */ - while (offset < Py_SIZE(a) && rts[offset / SEGSIZE] != t) { - Py_ssize_t allocated; /* size (in bytes) of output buffer */ + while (offset < Py_SIZE(a) && rts[offset / SEGSIZE] != total) { + Py_ssize_t allocated = PyBytes_GET_SIZE(out); - /* Make sure we have enough space in output buffer for next block. + /* Make sure we have enough memory in output buffer for next block. The largest block possible is a type 0 block with 128 segments. - It's size is: 1 head bytes + 128 * 32 raw bytes. + Its size is: 1 head bytes + 128 * 32 raw bytes. Plus, we also may have the stop byte. */ - allocated = PyBytes_GET_SIZE(out); - if (allocated < len + 1 + 128 * 32 + 1) { /* increase allocation */ - if (_PyBytes_Resize(&out, allocated + 32768) < 0) - return NULL; + if (allocated < len + 1 + 128 * 32 + 1) { + if (_PyBytes_Resize(&out, allocated + ALLOC_SIZE) < 0) + goto error; str = PyBytes_AS_STRING(out); } offset += sc_encode_block(str, &len, a, rts, offset); @@ -1372,7 +1572,12 @@ sc_encode(PyObject *module, PyObject *ob return NULL; return out; + + error: + PyMem_Free(rts); + return NULL; } +#undef ALLOC_SIZE PyDoc_STRVAR(sc_encode_doc, "sc_encode(bitarray, /) -> bytes\n\ @@ -1382,6 +1587,8 @@ This representation is useful for effici Use `sc_decode()` for decompressing (decoding)."); +/* read header from 'iter' and set 'endian' and 'nbits', return 0 on success + and -1 of failure (after setting exception) */ static int sc_decode_header(PyObject *iter, int *endian, Py_ssize_t *nbits) { @@ -1399,12 +1606,11 @@ sc_decode_header(PyObject *iter, int *en len = head & 0x0f; if (len > (int) sizeof(Py_ssize_t)) { - PyErr_Format(PyExc_OverflowError, - "sizeof(Py_ssize_t) = %d: cannot read %d bytes", - (int) sizeof(Py_ssize_t), len); + PyErr_Format(PyExc_OverflowError, "sizeof(Py_ssize_t) = %d: cannot " + "read %d bytes", (int) sizeof(Py_ssize_t), len); return -1; } - if ((*nbits = read_n(len, iter)) < 0) + if ((*nbits = read_n(iter, len)) < 0) return -1; return 0; @@ -1442,7 +1648,7 @@ sc_read_sparse(bitarrayobject *a, Py_ssi while (k--) { Py_ssize_t i; - if ((i = read_n(n, iter)) < 0) + if ((i = read_n(iter, n)) < 0) return -1; i += 8 * offset; @@ -1457,8 +1663,8 @@ sc_read_sparse(bitarrayobject *a, Py_ssi return BSI(n); } -/* Decode one block: consume iter and set bitarray buffer at offset. - Return size of offset increment in bytes, or -1 on failure. */ +/* Decode one block: consume iter and set bitarray buffer starting at + offset. Return decoded block size, or -1 on failure. */ static Py_ssize_t sc_decode_block(bitarrayobject *a, Py_ssize_t offset, PyObject *iter) { @@ -1497,8 +1703,7 @@ sc_decode(PyObject *module, PyObject *ob Py_ssize_t offset = 0, increase, nbits; int endian; - iter = PyObject_GetIter(obj); - if (iter == NULL) + if ((iter = PyObject_GetIter(obj)) == NULL) return PyErr_Format(PyExc_TypeError, "'%s' object is not iterable", Py_TYPE(obj)->tp_name); @@ -1506,8 +1711,7 @@ sc_decode(PyObject *module, PyObject *ob goto error; /* create bitarray of length nbits */ - a = new_bitarray(nbits, Py_None, 0); - if (a == NULL) + if ((a = new_bitarray(nbits, Py_None, 0)) == NULL) goto error; a->endian = endian; @@ -1535,11 +1739,12 @@ This function consumes only one bitarray untouched. Use `sc_encode()` for compressing (encoding)."); #undef BSI +#undef NSEG /* ------------------- variable length bitarray format ----------------- */ -/* LEN_PAD_BITS is always 3 - the number of bits (length) that is necessary to - represent the number of pad bits. The number of padding bits itself is +/* LEN_PAD_BITS is always 3 - the number of bits (length) that is necessary + to represent the number of pad bits. The number of padding bits itself is called 'padding' below. 'padding' refers to the pad bits within the variable length format. @@ -1549,21 +1754,24 @@ untouched. Use `sc_encode()` for compre */ #define LEN_PAD_BITS 3 +/* initial number of bits we allocate in vl_decode(), and amount by which + we increase our allocation by in vl_decode_core() if we run out */ +#define ALLOC_BITS 1024 + /* Consume 'iter' while extending bitarray 'a'. Return 0 on success. On failure, set exception and return -1. */ static int vl_decode_core(bitarrayobject *a, PyObject *iter) { - Py_ssize_t padding; /* number of pad bits read from header byte */ Py_ssize_t i = 0; /* bit counter */ - int k, c; + int padding, k, c; - if ((c = next_char(iter)) < 0) /* header byte */ + if ((c = next_char(iter)) < 0) /* head byte */ return -1; padding = (c & 0x70) >> 4; - if (padding >= 7 || ((c & 0x80) == 0 && padding > 4)) { - PyErr_Format(PyExc_ValueError, "invalid header byte: 0x%02x", c); + if (padding == 7 || ((c & 0x80) == 0 && padding > 4)) { + PyErr_Format(PyExc_ValueError, "invalid head byte: 0x%02x", c); return -1; } for (k = 0; k < 4; k++) @@ -1573,7 +1781,8 @@ vl_decode_core(bitarrayobject *a, PyObje if ((c = next_char(iter)) < 0) return -1; - if (resize_lite(a, i + 7) < 0) + /* ensure bitarray is large enough to accommodate seven more bits */ + if (a->nbits < i + 7 && resize_lite(a, a->nbits + ALLOC_BITS) < 0) return -1; assert(i + 6 < a->nbits); @@ -1600,7 +1809,7 @@ vl_decode(PyObject *module, PyObject *ar return PyErr_Format(PyExc_TypeError, "'%s' object is not iterable", Py_TYPE(obj)->tp_name); - a = new_bitarray(32, endian, -1); + a = new_bitarray(ALLOC_BITS, endian, -1); if (a == NULL) goto error; @@ -1609,11 +1818,13 @@ vl_decode(PyObject *module, PyObject *ar Py_DECREF(iter); return (PyObject *) a; + error: Py_DECREF(iter); Py_XDECREF((PyObject *) a); return NULL; } +#undef ALLOC_BITS PyDoc_STRVAR(vl_decode_doc, "vl_decode(stream, /, endian=None) -> bitarray\n\ @@ -1628,35 +1839,34 @@ vl_encode(PyObject *module, PyObject *ob { PyObject *result; bitarrayobject *a; - Py_ssize_t padding, n, m, i; - Py_ssize_t j = 0; /* byte conter */ + Py_ssize_t nbits, n, i, j = 0; /* j: byte counter */ + int padding; char *str; if (ensure_bitarray(obj) < 0) return NULL; a = (bitarrayobject *) obj; - n = (a->nbits + LEN_PAD_BITS + 6) / 7; /* number of resulting bytes */ - m = 7 * n - LEN_PAD_BITS; /* number of bits resulting bytes can hold */ - padding = m - a->nbits; /* number of pad bits */ - assert(0 <= padding && padding < 7); + nbits = a->nbits; + n = (nbits + LEN_PAD_BITS + 6) / 7; /* number of resulting bytes */ + padding = (int) (7 * n - LEN_PAD_BITS - nbits); result = PyBytes_FromStringAndSize(NULL, n); if (result == NULL) return NULL; str = PyBytes_AsString(result); - str[0] = a->nbits > 4 ? 0x80 : 0x00; /* leading bit */ - str[0] |= padding << 4; /* encode padding */ - for (i = 0; i < 4 && i < a->nbits; i++) + str[0] = nbits > 4 ? 0x80 : 0x00; /* lead bit */ + str[0] |= padding << 4; /* encode padding */ + for (i = 0; i < 4 && i < nbits; i++) str[0] |= (0x08 >> i) * getbit(a, i); - for (i = 4; i < a->nbits; i++) { + for (i = 4; i < nbits; i++) { int k = (i - 4) % 7; if (k == 0) { j++; - str[j] = j < n - 1 ? 0x80 : 0x00; /* leading bit */ + str[j] = j < n - 1 ? 0x80 : 0x00; /* lead bit */ } str[j] |= (0x40 >> k) * getbit(a, i); } @@ -1672,6 +1882,8 @@ Return variable length binary representa This representation is useful for efficiently storing small bitarray\n\ in a binary stream. Use `vl_decode()` for decoding."); +#undef LEN_PAD_BITS + /* ----------------------- canonical Huffman decoder ------------------- */ /* @@ -1681,7 +1893,7 @@ in a binary stream. Use `vl_decode()` f - symbol is a Python sequence of the symbols in canonical order where the number of entries is the sum of the counts in count[]. */ -#define MAXBITS 31 /* maximum bits in a code */ +#define MAXBITS 31 /* maximum bit length in a code */ typedef struct { PyObject_HEAD @@ -1693,40 +1905,38 @@ typedef struct { static PyTypeObject CHDI_Type; -/* set elements in count (from seq) and return their sum, or -1 on error */ +/* set elements in count (from sequence) and return their sum, + or -1 on error after setting exception */ static Py_ssize_t set_count(int *count, PyObject *sequence) { - Py_ssize_t n, c, res = 0; + Py_ssize_t n, res = 0; int i; - n = PySequence_Size(sequence); - if (n < 0) + if ((n = PySequence_Size(sequence)) < 0) return -1; - if (n > MAXBITS) { + if (n > MAXBITS + 1) { PyErr_Format(PyExc_ValueError, "len(count) cannot be larger than %d", - MAXBITS); + MAXBITS + 1); return -1; } - for (i = 1; i <= MAXBITS; i++) { - c = 0; - if (i < n) { - PyObject *item = PySequence_GetItem(sequence, i); - Py_ssize_t maxcount = ((Py_ssize_t) 1) << i; - - if (item == NULL) - return -1; - c = PyNumber_AsSsize_t(item, PyExc_OverflowError); - Py_DECREF(item); - if (c == -1 && PyErr_Occurred()) - return -1; - if (c < 0 || c > maxcount) { - PyErr_Format(PyExc_ValueError, "count[%d] cannot be negative" - " or larger than %zd, got %zd", i, maxcount, c); - return -1; - } + memset(count, 0, sizeof(int) * (MAXBITS + 1)); + for (i = 1; i < n; i++) { + PyObject *item; + Py_ssize_t c; + + if ((item = PySequence_GetItem(sequence, i)) == NULL) + return -1; + c = PyNumber_AsSsize_t(item, PyExc_OverflowError); + Py_DECREF(item); + if (c == -1 && PyErr_Occurred()) + return -1; + if (c >> i && (c - 1) >> i) { + PyErr_Format(PyExc_ValueError, "count[%d] not in [0..%zu], " + "got %zd", i, ((size_t) 1) << i, c); + return -1; } count[i] = (int) c; res += c; @@ -1743,7 +1953,7 @@ chdi_new(PyObject *module, PyObject *arg chdi_obj *it; /* iterator object to be returned */ if (!PyArg_ParseTuple(args, "O!OO:canonical_decode", - bitarray_type_obj, &a, &count, &symbol)) + bitarray_type, &a, &count, &symbol)) return NULL; if (!PySequence_Check(count)) return PyErr_Format(PyExc_TypeError, "count expected to be sequence, " @@ -1843,13 +2053,10 @@ chdi_traverse(chdi_obj *it, visitproc vi return 0; } +#undef MAXBITS + static PyTypeObject CHDI_Type = { -#if IS_PY3K PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ -#endif "bitarray.util.canonical_decodeiter", /* tp_name */ sizeof(chdi_obj), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -1880,7 +2087,84 @@ static PyTypeObject CHDI_Type = { 0, /* tp_methods */ }; -/* --------------------------------------------------------------------- */ +/* ---------- module functions exposed in debug mode for testing ------- */ + +#ifndef NDEBUG + +static PyObject * +module_setup_table(PyObject *module, PyObject *obj) +{ + char table[256]; + + assert(PyUnicode_Check(obj)); + assert(PyUnicode_GET_LENGTH(obj) == 1); + setup_table(table, PyUnicode_READ_CHAR(obj, 0)); + return PyBytes_FromStringAndSize(table, 256); +} + +/* Return zlw(a) as a new bitarray, rather than an int object. + This makes testing easier, because the int result would depend + on the machine byteorder. */ +static PyObject * +module_zlw(PyObject *module, PyObject *obj) +{ + bitarrayobject *a, *res; + uint64_t w; + + assert(bitarray_Check(obj)); + a = (bitarrayobject *) obj; + w = zlw(a); + if ((res = new_bitarray(64, Py_None, -1)) == NULL) + return NULL; + res->endian = a->endian; + memcpy(res->ob_item, &w, 8); + return (PyObject *) res; +} + +static PyObject * +module_cfw(PyObject *module, PyObject *args) /* count_from_word() */ +{ + bitarrayobject *a; + Py_ssize_t i; + + if (!PyArg_ParseTuple(args, "O!n", bitarray_type, (PyObject *) &a, &i)) + return NULL; + return PyLong_FromSsize_t(count_from_word(a, i)); +} + +static PyObject * +module_read_n(PyObject *module, PyObject *args) +{ + PyObject *iter; + Py_ssize_t i; + int n; + + if (!PyArg_ParseTuple(args, "Oi", &iter, &n)) + return NULL; + if ((i = read_n(iter, n)) < 0) + return NULL; + return PyLong_FromSsize_t(i); +} + +static PyObject * +module_write_n(PyObject *module, PyObject *args) +{ + PyObject *result; + char *str; + Py_ssize_t i; + int n; + + if (!PyArg_ParseTuple(args, "in", &n, &i)) + return NULL; + if ((result = PyBytes_FromStringAndSize(NULL, n)) == NULL) + return NULL; + str = PyBytes_AsString(result); + write_n(str, n, i); + return result; +} + +#endif /* NDEBUG */ + static PyMethodDef module_functions[] = { {"zeros", (PyCFunction) zeros, METH_KEYWORDS | @@ -1889,22 +2173,25 @@ static PyMethodDef module_functions[] = METH_VARARGS, ones_doc}, {"count_n", (PyCFunction) count_n, METH_VARARGS, count_n_doc}, {"parity", (PyCFunction) parity, METH_O, parity_doc}, + {"_ssqi", (PyCFunction) ssqi, METH_VARARGS, 0}, + {"xor_indices", (PyCFunction) xor_indices, METH_O, xor_indices_doc}, {"count_and", (PyCFunction) count_and, METH_VARARGS, count_and_doc}, {"count_or", (PyCFunction) count_or, METH_VARARGS, count_or_doc}, {"count_xor", (PyCFunction) count_xor, METH_VARARGS, count_xor_doc}, {"any_and", (PyCFunction) any_and, METH_VARARGS, any_and_doc}, {"subset", (PyCFunction) subset, METH_VARARGS, subset_doc}, - {"_correspond_all", + {"correspond_all", (PyCFunction) correspond_all, METH_VARARGS, correspond_all_doc}, - {"serialize", (PyCFunction) serialize, METH_O, serialize_doc}, - {"deserialize", - (PyCFunction) deserialize, - METH_O, deserialize_doc}, - {"ba2hex", (PyCFunction) ba2hex, METH_O, ba2hex_doc}, + {"byteswap", (PyCFunction) byteswap, METH_VARARGS, byteswap_doc}, + {"serialize", (PyCFunction) serialize, METH_O, serialize_doc}, + {"deserialize", (PyCFunction) deserialize, METH_O, deserialize_doc}, + {"ba2hex", (PyCFunction) ba2hex, METH_KEYWORDS | + METH_VARARGS, ba2hex_doc}, {"hex2ba", (PyCFunction) hex2ba, METH_KEYWORDS | METH_VARARGS, hex2ba_doc}, - {"ba2base", (PyCFunction) ba2base, METH_VARARGS, ba2base_doc}, + {"ba2base", (PyCFunction) ba2base, METH_KEYWORDS | + METH_VARARGS, ba2base_doc}, {"base2ba", (PyCFunction) base2ba, METH_KEYWORDS | METH_VARARGS, base2ba_doc}, {"sc_encode", (PyCFunction) sc_encode, METH_O, sc_encode_doc}, @@ -1914,59 +2201,50 @@ static PyMethodDef module_functions[] = METH_VARARGS, vl_decode_doc}, {"canonical_decode", (PyCFunction) chdi_new, METH_VARARGS, chdi_doc}, + #ifndef NDEBUG - /* functionality exposed in debug mode for testing */ - {"_sc_rts", (PyCFunction) sc_rts, METH_O, 0}, + /* functions exposed in debug mode for testing */ + {"_setup_table", (PyCFunction) module_setup_table, METH_O, 0}, + {"_zlw", (PyCFunction) module_zlw, METH_O, 0}, + {"_cfw", (PyCFunction) module_cfw, METH_VARARGS, 0}, + {"_read_n", (PyCFunction) module_read_n, METH_VARARGS, 0}, + {"_write_n", (PyCFunction) module_write_n, METH_VARARGS, 0}, + {"_sc_rts", (PyCFunction) module_sc_rts, METH_O, 0}, #endif + {NULL, NULL} /* sentinel */ }; /******************************* Install Module ***************************/ -#if IS_PY3K static PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_util", 0, -1, module_functions, }; -#endif PyMODINIT_FUNC -#if IS_PY3K PyInit__util(void) -#else -init_util(void) -#endif { PyObject *m, *bitarray_module; - if ((bitarray_module = PyImport_ImportModule("bitarray")) == NULL) - goto error; - bitarray_type_obj = PyObject_GetAttrString(bitarray_module, "bitarray"); + bitarray_module = PyImport_ImportModule("bitarray"); + if (bitarray_module == NULL) + return NULL; + bitarray_type = (PyTypeObject *) PyObject_GetAttrString(bitarray_module, + "bitarray"); Py_DECREF(bitarray_module); - if (bitarray_type_obj == NULL) - goto error; + if (bitarray_type == NULL) + return NULL; -#if IS_PY3K - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule3("_util", module_functions, 0); -#endif - if (m == NULL) - goto error; + if ((m = PyModule_Create(&moduledef)) == NULL) + return NULL; if (PyType_Ready(&CHDI_Type) < 0) - goto error; + return NULL; Py_SET_TYPE(&CHDI_Type, &PyType_Type); #ifndef NDEBUG /* expose segment size in debug mode for testing */ PyModule_AddObject(m, "_SEGSIZE", PyLong_FromSsize_t(SEGSIZE)); #endif -#if IS_PY3K return m; - error: - return NULL; -#else - error: - return; -#endif } diff -pruN 2.9.2-1/bitarray/architecture.txt 3.6.1-1/bitarray/architecture.txt --- 2.9.2-1/bitarray/architecture.txt 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/architecture.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,26 +0,0 @@ -Dependency of files -------------------- - -A depends on B A --------> B - - -A imports B in a function A - - - - > B - - - - +------------+ +------------+ - | util.py |------->| _util.c | - +------------+ +------------+ - | ^ - | | - | +--------------------------------------------------+ - | | | - V v | - +-------------+ - - - > +------------------+ - - - > +--------------+ - | __init__.py | | test_bitarray.py | | test_util.py | - +-------------+ <------ +------------------+ <------ +--------------+ - | - V - +-------------+ - | _bitarray.c | - +-------------+ diff -pruN 2.9.2-1/bitarray/bitarray.h 3.6.1-1/bitarray/bitarray.h --- 2.9.2-1/bitarray/bitarray.h 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/bitarray.h 2025-08-12 08:35:42.000000000 +0000 @@ -1,10 +1,10 @@ /* - Copyright (c) 2008 - 2024, Ilan Schnell; All Rights Reserved + Copyright (c) 2008 - 2025, Ilan Schnell; All Rights Reserved bitarray is published under the PSF license. Author: Ilan Schnell */ -#define BITARRAY_VERSION "2.9.2" +#define BITARRAY_VERSION "3.6.1" #ifdef STDC_HEADERS # include @@ -30,27 +30,12 @@ /* Py_UNREACHABLE was introduced in Python 3.7 */ #ifndef Py_UNREACHABLE -#define Py_UNREACHABLE() abort() -#endif - -#if PY_MAJOR_VERSION >= 3 -#define IS_PY3K 1 -#define BYTES_SIZE_FMT "y#" -#else -#define IS_PY3K 0 -/* the Py_MIN and Py_MAX macros were introduced in Python 3.3 */ -#define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) -#define Py_MAX(x, y) (((x) > (y)) ? (x) : (y)) -#define PySlice_GetIndicesEx(slice, len, start, stop, step, slicelength) \ - PySlice_GetIndicesEx(((PySliceObject *) slice), \ - (len), (start), (stop), (step), (slicelength)) -#define PyLong_FromLong PyInt_FromLong -#define BYTES_SIZE_FMT "s#" +#define Py_UNREACHABLE() assert(0) #endif /* --- bitarrayobject --- */ -/* .ob_size is buffer size (in bytes), not the number of elements. +/* .ob_size is the buffer size (in bytes), not the number of elements. The number of elements (bits) is .nbits. */ typedef struct { PyObject_VAR_HEAD @@ -75,9 +60,9 @@ typedef struct { #define ENDIAN_STR(endian) ((endian) == ENDIAN_LITTLE ? "little" : "big") /* number of pad bits */ -#define PADBITS(self) (8 * Py_SIZE(self) - (self)->nbits) +#define PADBITS(self) ((8 - (self)->nbits % 8) % 8) -/* number of bytes necessary to store given bits */ +/* number of bytes necessary to store given nunmber of bits */ #define BYTES(bits) (((bits) + 7) >> 3) /* we're not using bitmask_table here, as it is actually slower */ @@ -148,22 +133,21 @@ zlc(bitarrayobject *self) /* zlc = /* Return a uint64_t word representing the last (up to 63) remaining bits of the buffer. All missing bytes (to complete the word) and padbits are treated as zeros. - If the length of the bitarray is a multiple of 64 (which includes an empty - bitarray), 0 is returned. */ + If the length of the bitarray is a multiple of 64 (which also includes + an empty bitarray), 0 is returned. */ static inline uint64_t zlw(bitarrayobject *self) /* zlw = zeroed last word */ { - const Py_ssize_t nbits = self->nbits; - const Py_ssize_t nw = 8 * (nbits / 64); /* bytes in complete words */ - const int nr = (nbits % 64) / 8; /* complete remaining bytes */ + const size_t nbits = self->nbits; + const size_t nw = (nbits / 64) * 8; /* bytes in complete words */ + const size_t nr = (nbits % 64) / 8; /* complete remaining bytes */ uint64_t res = 0; - assert(nw + nr == nbits / 8 && nw + nr <= Py_SIZE(self)); - memcpy((char *) &res, self->ob_item + nw, (size_t) nr); + assert(nw + nr == nbits / 8 && 8 * (nw + nr) + nbits % 8 == nbits); + memcpy((char *) &res, self->ob_item + nw, nr); if (nbits % 8) *(((char *) &res) + nr) = zlc(self); - assert(nbits % 64 || res == 0); return res; } @@ -171,10 +155,11 @@ zlw(bitarrayobject *self) /* zlw = static inline void set_padbits(bitarrayobject *self) { - const int r = self->nbits % 8; /* index into mask table */ - - if (self->readonly == 0 && r) - self->ob_item[Py_SIZE(self) - 1] &= ones_table[IS_BE(self)][r]; + if (self->readonly == 0) { + int r = self->nbits % 8; /* index into mask table */ + if (r) + self->ob_item[Py_SIZE(self) - 1] &= ones_table[IS_BE(self)][r]; + } } /* population count - number of 1's in uint64 */ @@ -197,13 +182,26 @@ popcnt_64(uint64_t x) #endif } +static inline int +parity_64(uint64_t x) +{ +#if (defined(__clang__) || defined(__GNUC__)) + return __builtin_parityll(x); +#else + int i; + for (i = 32; i > 0; i /= 2) + x ^= x >> i; + return x & 1; +#endif +} + static inline uint64_t builtin_bswap64(uint64_t word) { #if (defined(__clang__) || \ (defined(__GNUC__) \ && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 3)))) - /* __builtin_bswap64() is available since GCC 4.3. */ + /* __builtin_bswap64() is available since GCC 4.3 */ # define HAVE_BUILTIN_BSWAP64 1 return __builtin_bswap64(word); #elif defined(_MSC_VER) @@ -215,6 +213,47 @@ builtin_bswap64(uint64_t word) #endif } +/* reverse order of first n bytes of p */ +static inline void +swap_bytes(char *p, Py_ssize_t n) +{ + Py_ssize_t i, j; + for (i = 0, j = n - 1; i < j; i++, j--) { + char t = p[i]; + p[i] = p[j]; + p[j] = t; + } +} + +/* write 256 characters into table for given kernel operation */ +static inline void +setup_table(char *table, char kop) +{ + int k; + for (k = 0; k < 256; k++) { + char t = 0, j; + for (j = 0; j < 8; j++) { + if (k & 1 << j) + /* j are the indices of active bits in k (little endian) */ + switch (kop) { + case 'a': t += j; break; /* add active indices */ + case 'A': t += 7 - j; break; /* 'a' for big endian */ + case 's': t += j * j; /* add squares of active indices */ + break; + case 'S': t += (7-j) * (7-j); /* 's' for big endian */ + break; + case 'x': t ^= j; break; /* xor active indices */ + case 'X': t ^= 7 - j; break; /* 'x' for big endian */ + case 'c': t++; break; /* bit count */ + case 'p': t ^= 1; break; /* parity */ + case 'r': t |= 128 >> j; break; /* reverse bits */ + default: Py_UNREACHABLE(); + } + } + table[k] = t; + } +} + /* Return distance [0..3] to next aligned pointer. While on modern compilers uint64_t pointers may be misaligned, it may cause problems on older ones. Moreover, it may lead to slowdown (even @@ -223,7 +262,7 @@ static inline int to_aligned(void *p) { int r = ((uintptr_t) p) % 4; - return r ? 4 - r : 0; + return (4 - r) % 4; } /* population count of n words starting from at uint64_t pointer w */ @@ -232,56 +271,17 @@ popcnt_words(uint64_t *w, Py_ssize_t n) { Py_ssize_t cnt = 0; - assert(n >= 0 && to_aligned((void *) w) == 0); + assert(n >= 0 && ((uintptr_t) w) % 4 == 0); while (n--) cnt += popcnt_64(*w++); return cnt; } -/* adjust index a manner consistent with the handling of normal slices */ -static inline void -adjust_index(Py_ssize_t length, Py_ssize_t *i, Py_ssize_t step) -{ - if (*i < 0) { - *i += length; - if (*i < 0) - *i = (step < 0) ? -1 : 0; - } - else if (*i >= length) { - *i = (step < 0) ? length - 1 : length; - } -} - -/* same as PySlice_AdjustIndices() which was introduced in Python 3.6.1 */ -static inline Py_ssize_t -adjust_indices(Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, - Py_ssize_t step) -{ -#if PY_VERSION_HEX > 0x03060100 - return PySlice_AdjustIndices(length, start, stop, step); -#else - assert(step != 0); - adjust_index(length, start, step); - adjust_index(length, stop, step); - /* - a / b does integer division. If either a or b is negative, the result - depends on the compiler (rounding can go toward 0 or negative infinity). - Therefore, we are careful that both a and b are always positive. - */ - if (step < 0) { - if (*stop < *start) - return (*start - *stop - 1) / (-step) + 1; - } - else { - if (*start < *stop) - return (*stop - *start - 1) / step + 1; - } - return 0; -#endif -} - -/* adjust slice parameters such that step is always positive; produces - simpler loops over elements when their order is irrelevant */ +/* Adjust slice parameters such that step is always positive. + This produces simpler loops over elements when their order is irrelevant. + Moreover, for step = -1, we can now use set_span() in set_range() and + count_span() in count_range(). +*/ static inline void adjust_step_positive(Py_ssize_t slicelength, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step) @@ -309,7 +309,7 @@ conv_pybit(PyObject *value, int *vi) if (n == -1 && PyErr_Occurred()) return 0; - if (n < 0 || n > 1) { + if (n >> 1) { PyErr_Format(PyExc_ValueError, "bit must be 0 or 1, got %zd", n); return 0; } diff -pruN 2.9.2-1/bitarray/pythoncapi_compat.h 3.6.1-1/bitarray/pythoncapi_compat.h --- 2.9.2-1/bitarray/pythoncapi_compat.h 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/pythoncapi_compat.h 2025-08-12 08:35:42.000000000 +0000 @@ -7,7 +7,7 @@ // https://github.com/python/pythoncapi_compat // // Latest version: -// https://raw.githubusercontent.com/python/pythoncapi_compat/master/pythoncapi_compat.h +// https://raw.githubusercontent.com/python/pythoncapi-compat/main/pythoncapi_compat.h // // SPDX-License-Identifier: 0BSD @@ -19,16 +19,14 @@ extern "C" { #endif #include -#include "frameobject.h" // PyFrameObject, PyFrame_GetBack() +#include // offsetof() - -// Compatibility with Visual Studio 2013 and older which don't support -// the inline keyword in C (only in C++): use __inline instead. -#if (defined(_MSC_VER) && _MSC_VER < 1900 \ - && !defined(__cplusplus) && !defined(inline)) -# define PYCAPI_COMPAT_STATIC_INLINE(TYPE) static __inline TYPE -#else -# define PYCAPI_COMPAT_STATIC_INLINE(TYPE) static inline TYPE +// Python 3.11.0b4 added PyFrame_Back() to Python.h +#if PY_VERSION_HEX < 0x030b00B4 && !defined(PYPY_VERSION) +# include "frameobject.h" // PyFrameObject, PyFrame_GetBack() +#endif +#if PY_VERSION_HEX < 0x030C00A3 +# include // T_SHORT, READONLY #endif @@ -36,10 +34,12 @@ extern "C" { # define _Py_CAST(type, expr) ((type)(expr)) #endif -// On C++11 and newer, _Py_NULL is defined as nullptr on C++11, -// otherwise it is defined as NULL. +// Static inline functions should use _Py_NULL rather than using directly NULL +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. #ifndef _Py_NULL -# if defined(__cplusplus) && __cplusplus >= 201103 +# if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) # define _Py_NULL nullptr # else # define _Py_NULL NULL @@ -51,11 +51,17 @@ extern "C" { # define _PyObject_CAST(op) _Py_CAST(PyObject*, op) #endif +#ifndef Py_BUILD_ASSERT +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)sizeof(char [1 - 2 * !(cond)]); \ + } while(0) +#endif + // bpo-42262 added Py_NewRef() to Python 3.10.0a3 #if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_NewRef) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -_Py_NewRef(PyObject *obj) +static inline PyObject* _Py_NewRef(PyObject *obj) { Py_INCREF(obj); return obj; @@ -66,8 +72,7 @@ _Py_NewRef(PyObject *obj) // bpo-42262 added Py_XNewRef() to Python 3.10.0a3 #if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_XNewRef) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -_Py_XNewRef(PyObject *obj) +static inline PyObject* _Py_XNewRef(PyObject *obj) { Py_XINCREF(obj); return obj; @@ -78,8 +83,7 @@ _Py_XNewRef(PyObject *obj) // bpo-39573 added Py_SET_REFCNT() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_REFCNT) -PYCAPI_COMPAT_STATIC_INLINE(void) -_Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) +static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { ob->ob_refcnt = refcnt; } @@ -116,18 +120,17 @@ _Py_SET_REFCNT(PyObject *ob, Py_ssize_t #if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsNone) # define Py_IsNone(x) Py_Is(x, Py_None) #endif -#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsTrue) +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsTrue) # define Py_IsTrue(x) Py_Is(x, Py_True) #endif -#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsFalse) +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsFalse) # define Py_IsFalse(x) Py_Is(x, Py_False) #endif // bpo-39573 added Py_SET_TYPE() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE) -PYCAPI_COMPAT_STATIC_INLINE(void) -_Py_SET_TYPE(PyObject *ob, PyTypeObject *type) +static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -137,8 +140,7 @@ _Py_SET_TYPE(PyObject *ob, PyTypeObject // bpo-39573 added Py_SET_SIZE() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE) -PYCAPI_COMPAT_STATIC_INLINE(void) -_Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) +static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { ob->ob_size = size; } @@ -148,8 +150,7 @@ _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t // bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1 #if PY_VERSION_HEX < 0x030900B1 || defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyCodeObject*) -PyFrame_GetCode(PyFrameObject *frame) +static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame) { assert(frame != _Py_NULL); assert(frame->f_code != _Py_NULL); @@ -157,8 +158,7 @@ PyFrame_GetCode(PyFrameObject *frame) } #endif -PYCAPI_COMPAT_STATIC_INLINE(PyCodeObject*) -_PyFrame_GetCodeBorrow(PyFrameObject *frame) +static inline PyCodeObject* _PyFrame_GetCodeBorrow(PyFrameObject *frame) { PyCodeObject *code = PyFrame_GetCode(frame); Py_DECREF(code); @@ -168,8 +168,7 @@ _PyFrame_GetCodeBorrow(PyFrameObject *fr // bpo-40421 added PyFrame_GetBack() to Python 3.9.0b1 #if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) -PyFrame_GetBack(PyFrameObject *frame) +static inline PyFrameObject* PyFrame_GetBack(PyFrameObject *frame) { assert(frame != _Py_NULL); return _Py_CAST(PyFrameObject*, Py_XNewRef(frame->f_back)); @@ -177,8 +176,7 @@ PyFrame_GetBack(PyFrameObject *frame) #endif #if !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) -_PyFrame_GetBackBorrow(PyFrameObject *frame) +static inline PyFrameObject* _PyFrame_GetBackBorrow(PyFrameObject *frame) { PyFrameObject *back = PyFrame_GetBack(frame); Py_XDECREF(back); @@ -189,8 +187,7 @@ _PyFrame_GetBackBorrow(PyFrameObject *fr // bpo-40421 added PyFrame_GetLocals() to Python 3.11.0a7 #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyFrame_GetLocals(PyFrameObject *frame) +static inline PyObject* PyFrame_GetLocals(PyFrameObject *frame) { #if PY_VERSION_HEX >= 0x030400B1 if (PyFrame_FastToLocalsWithError(frame) < 0) { @@ -206,8 +203,7 @@ PyFrame_GetLocals(PyFrameObject *frame) // bpo-40421 added PyFrame_GetGlobals() to Python 3.11.0a7 #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyFrame_GetGlobals(PyFrameObject *frame) +static inline PyObject* PyFrame_GetGlobals(PyFrameObject *frame) { return Py_NewRef(frame->f_globals); } @@ -216,8 +212,7 @@ PyFrame_GetGlobals(PyFrameObject *frame) // bpo-40421 added PyFrame_GetBuiltins() to Python 3.11.0a7 #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyFrame_GetBuiltins(PyFrameObject *frame) +static inline PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) { return Py_NewRef(frame->f_builtins); } @@ -226,8 +221,7 @@ PyFrame_GetBuiltins(PyFrameObject *frame // bpo-40421 added PyFrame_GetLasti() to Python 3.11.0b1 #if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFrame_GetLasti(PyFrameObject *frame) +static inline int PyFrame_GetLasti(PyFrameObject *frame) { #if PY_VERSION_HEX >= 0x030A00A7 // bpo-27129: Since Python 3.10.0a7, f_lasti is an instruction offset, @@ -246,8 +240,7 @@ PyFrame_GetLasti(PyFrameObject *frame) // gh-91248 added PyFrame_GetVar() to Python 3.12.0a2 #if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyFrame_GetVar(PyFrameObject *frame, PyObject *name) +static inline PyObject* PyFrame_GetVar(PyFrameObject *frame, PyObject *name) { PyObject *locals, *value; @@ -258,7 +251,7 @@ PyFrame_GetVar(PyFrameObject *frame, PyO #if PY_VERSION_HEX >= 0x03000000 value = PyDict_GetItemWithError(locals, name); #else - value = PyDict_GetItem(locals, name); + value = _PyDict_GetItemWithError(locals, name); #endif Py_DECREF(locals); @@ -280,11 +273,15 @@ PyFrame_GetVar(PyFrameObject *frame, PyO // gh-91248 added PyFrame_GetVarString() to Python 3.12.0a2 #if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) +static inline PyObject* PyFrame_GetVarString(PyFrameObject *frame, const char *name) { PyObject *name_obj, *value; +#if PY_VERSION_HEX >= 0x03000000 name_obj = PyUnicode_FromString(name); +#else + name_obj = PyString_FromString(name); +#endif if (name_obj == NULL) { return NULL; } @@ -296,8 +293,8 @@ PyFrame_GetVarString(PyFrameObject *fram // bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5 -#if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyInterpreterState *) +#if PY_VERSION_HEX < 0x030900A5 || (defined(PYPY_VERSION) && PY_VERSION_HEX < 0x030B0000) +static inline PyInterpreterState * PyThreadState_GetInterpreter(PyThreadState *tstate) { assert(tstate != _Py_NULL); @@ -308,8 +305,7 @@ PyThreadState_GetInterpreter(PyThreadSta // bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1 #if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) -PyThreadState_GetFrame(PyThreadState *tstate) +static inline PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) { assert(tstate != _Py_NULL); return _Py_CAST(PyFrameObject *, Py_XNewRef(tstate->frame)); @@ -317,7 +313,7 @@ PyThreadState_GetFrame(PyThreadState *ts #endif #if !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) +static inline PyFrameObject* _PyThreadState_GetFrameBorrow(PyThreadState *tstate) { PyFrameObject *frame = PyThreadState_GetFrame(tstate); @@ -329,8 +325,7 @@ _PyThreadState_GetFrameBorrow(PyThreadSt // bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5 #if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyInterpreterState*) -PyInterpreterState_Get(void) +static inline PyInterpreterState* PyInterpreterState_Get(void) { PyThreadState *tstate; PyInterpreterState *interp; @@ -350,8 +345,7 @@ PyInterpreterState_Get(void) // bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a6 #if 0x030700A1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(uint64_t) -PyThreadState_GetID(PyThreadState *tstate) +static inline uint64_t PyThreadState_GetID(PyThreadState *tstate) { assert(tstate != _Py_NULL); return tstate->id; @@ -360,8 +354,7 @@ PyThreadState_GetID(PyThreadState *tstat // bpo-43760 added PyThreadState_EnterTracing() to Python 3.11.0a2 #if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(void) -PyThreadState_EnterTracing(PyThreadState *tstate) +static inline void PyThreadState_EnterTracing(PyThreadState *tstate) { tstate->tracing++; #if PY_VERSION_HEX >= 0x030A00A1 @@ -374,8 +367,7 @@ PyThreadState_EnterTracing(PyThreadState // bpo-43760 added PyThreadState_LeaveTracing() to Python 3.11.0a2 #if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(void) -PyThreadState_LeaveTracing(PyThreadState *tstate) +static inline void PyThreadState_LeaveTracing(PyThreadState *tstate) { int use_tracing = (tstate->c_tracefunc != _Py_NULL || tstate->c_profilefunc != _Py_NULL); @@ -392,8 +384,7 @@ PyThreadState_LeaveTracing(PyThreadState // bpo-37194 added PyObject_CallNoArgs() to Python 3.9.0a1 // PyObject_CallNoArgs() added to PyPy 3.9.16-v7.3.11 #if !defined(PyObject_CallNoArgs) && PY_VERSION_HEX < 0x030900A1 -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyObject_CallNoArgs(PyObject *func) +static inline PyObject* PyObject_CallNoArgs(PyObject *func) { return PyObject_CallFunctionObjArgs(func, NULL); } @@ -404,8 +395,7 @@ PyObject_CallNoArgs(PyObject *func) // _PyObject_CallOneArg) in Python 3.9.0a4 // PyObject_CallOneArg() added to PyPy 3.9.16-v7.3.11 #if !defined(PyObject_CallOneArg) && PY_VERSION_HEX < 0x030900A4 -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyObject_CallOneArg(PyObject *func, PyObject *arg) +static inline PyObject* PyObject_CallOneArg(PyObject *func, PyObject *arg) { return PyObject_CallFunctionObjArgs(func, arg, NULL); } @@ -414,10 +404,19 @@ PyObject_CallOneArg(PyObject *func, PyOb // bpo-1635741 added PyModule_AddObjectRef() to Python 3.10.0a3 #if PY_VERSION_HEX < 0x030A00A3 -PYCAPI_COMPAT_STATIC_INLINE(int) +static inline int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) { int res; + + if (!value && !PyErr_Occurred()) { + // PyModule_AddObject() raises TypeError in this case + PyErr_SetString(PyExc_SystemError, + "PyModule_AddObjectRef() must be called " + "with an exception raised if value is NULL"); + return -1; + } + Py_XINCREF(value); res = PyModule_AddObject(module, name, value); if (res < 0) { @@ -430,8 +429,7 @@ PyModule_AddObjectRef(PyObject *module, // bpo-40024 added PyModule_AddType() to Python 3.9.0a5 #if PY_VERSION_HEX < 0x030900A5 -PYCAPI_COMPAT_STATIC_INLINE(int) -PyModule_AddType(PyObject *module, PyTypeObject *type) +static inline int PyModule_AddType(PyObject *module, PyTypeObject *type) { const char *name, *dot; @@ -455,8 +453,7 @@ PyModule_AddType(PyObject *module, PyTyp // bpo-40241 added PyObject_GC_IsTracked() to Python 3.9.0a6. // bpo-4688 added _PyObject_GC_IS_TRACKED() to Python 2.7.0a2. #if PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyObject_GC_IsTracked(PyObject* obj) +static inline int PyObject_GC_IsTracked(PyObject* obj) { return (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)); } @@ -465,8 +462,7 @@ PyObject_GC_IsTracked(PyObject* obj) // bpo-40241 added PyObject_GC_IsFinalized() to Python 3.9.0a6. // bpo-18112 added _PyGCHead_FINALIZED() to Python 3.4.0 final. #if PY_VERSION_HEX < 0x030900A6 && PY_VERSION_HEX >= 0x030400F0 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyObject_GC_IsFinalized(PyObject *obj) +static inline int PyObject_GC_IsFinalized(PyObject *obj) { PyGC_Head *gc = _Py_CAST(PyGC_Head*, obj) - 1; return (PyObject_IS_GC(obj) && _PyGCHead_FINALIZED(gc)); @@ -476,8 +472,7 @@ PyObject_GC_IsFinalized(PyObject *obj) // bpo-39573 added Py_IS_TYPE() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_IS_TYPE) -PYCAPI_COMPAT_STATIC_INLINE(int) -_Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { +static inline int _Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; } #define Py_IS_TYPE(ob, type) _Py_IS_TYPE(_PyObject_CAST(ob), type) @@ -489,12 +484,10 @@ _Py_IS_TYPE(PyObject *ob, PyTypeObject * // Python 3.11a2 moved _PyFloat_Pack2() and _PyFloat_Unpack2() to the internal // C API: Python 3.11a2-3.11a6 versions are not supported. #if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFloat_Pack2(double x, char *p, int le) +static inline int PyFloat_Pack2(double x, char *p, int le) { return _PyFloat_Pack2(x, (unsigned char*)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(double) -PyFloat_Unpack2(const char *p, int le) +static inline double PyFloat_Unpack2(const char *p, int le) { return _PyFloat_Unpack2((const unsigned char *)p, le); } #endif @@ -505,28 +498,23 @@ PyFloat_Unpack2(const char *p, int le) // and _PyFloat_Unpack8() to the internal C API: Python 3.11a2-3.11a6 versions // are not supported. #if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFloat_Pack4(double x, char *p, int le) +static inline int PyFloat_Pack4(double x, char *p, int le) { return _PyFloat_Pack4(x, (unsigned char*)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFloat_Pack8(double x, char *p, int le) +static inline int PyFloat_Pack8(double x, char *p, int le) { return _PyFloat_Pack8(x, (unsigned char*)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(double) -PyFloat_Unpack4(const char *p, int le) +static inline double PyFloat_Unpack4(const char *p, int le) { return _PyFloat_Unpack4((const unsigned char *)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(double) -PyFloat_Unpack8(const char *p, int le) +static inline double PyFloat_Unpack8(const char *p, int le) { return _PyFloat_Unpack8((const unsigned char *)p, le); } #endif // gh-92154 added PyCode_GetCode() to Python 3.11.0b1 #if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyCode_GetCode(PyCodeObject *code) +static inline PyObject* PyCode_GetCode(PyCodeObject *code) { return Py_NewRef(code->co_code); } @@ -535,8 +523,7 @@ PyCode_GetCode(PyCodeObject *code) // gh-95008 added PyCode_GetVarnames() to Python 3.11.0rc1 #if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyCode_GetVarnames(PyCodeObject *code) +static inline PyObject* PyCode_GetVarnames(PyCodeObject *code) { return Py_NewRef(code->co_varnames); } @@ -544,8 +531,7 @@ PyCode_GetVarnames(PyCodeObject *code) // gh-95008 added PyCode_GetFreevars() to Python 3.11.0rc1 #if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyCode_GetFreevars(PyCodeObject *code) +static inline PyObject* PyCode_GetFreevars(PyCodeObject *code) { return Py_NewRef(code->co_freevars); } @@ -553,8 +539,7 @@ PyCode_GetFreevars(PyCodeObject *code) // gh-95008 added PyCode_GetCellvars() to Python 3.11.0rc1 #if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyCode_GetCellvars(PyCodeObject *code) +static inline PyObject* PyCode_GetCellvars(PyCodeObject *code) { return Py_NewRef(code->co_cellvars); } @@ -571,6 +556,1649 @@ PyCode_GetCellvars(PyCodeObject *code) #endif +// gh-105922 added PyImport_AddModuleRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A0 +static inline PyObject* PyImport_AddModuleRef(const char *name) +{ + return Py_XNewRef(PyImport_AddModule(name)); +} +#endif + + +// gh-105927 added PyWeakref_GetRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D0000 +static inline int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) +{ + PyObject *obj; + if (ref != NULL && !PyWeakref_Check(ref)) { + *pobj = NULL; + PyErr_SetString(PyExc_TypeError, "expected a weakref"); + return -1; + } + obj = PyWeakref_GetObject(ref); + if (obj == NULL) { + // SystemError if ref is NULL + *pobj = NULL; + return -1; + } + if (obj == Py_None) { + *pobj = NULL; + return 0; + } + *pobj = Py_NewRef(obj); + return 1; +} +#endif + + +// bpo-36974 added PY_VECTORCALL_ARGUMENTS_OFFSET to Python 3.8b1 +#ifndef PY_VECTORCALL_ARGUMENTS_OFFSET +# define PY_VECTORCALL_ARGUMENTS_OFFSET (_Py_CAST(size_t, 1) << (8 * sizeof(size_t) - 1)) +#endif + +// bpo-36974 added PyVectorcall_NARGS() to Python 3.8b1 +#if PY_VERSION_HEX < 0x030800B1 +static inline Py_ssize_t PyVectorcall_NARGS(size_t n) +{ + return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET; +} +#endif + + +// gh-105922 added PyObject_Vectorcall() to Python 3.9.0a4 +#if PY_VERSION_HEX < 0x030900A4 +static inline PyObject* +PyObject_Vectorcall(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ +#if PY_VERSION_HEX >= 0x030800B1 && !defined(PYPY_VERSION) + // bpo-36974 added _PyObject_Vectorcall() to Python 3.8.0b1 + return _PyObject_Vectorcall(callable, args, nargsf, kwnames); +#else + PyObject *posargs = NULL, *kwargs = NULL; + PyObject *res; + Py_ssize_t nposargs, nkwargs, i; + + if (nargsf != 0 && args == NULL) { + PyErr_BadInternalCall(); + goto error; + } + if (kwnames != NULL && !PyTuple_Check(kwnames)) { + PyErr_BadInternalCall(); + goto error; + } + + nposargs = (Py_ssize_t)PyVectorcall_NARGS(nargsf); + if (kwnames) { + nkwargs = PyTuple_GET_SIZE(kwnames); + } + else { + nkwargs = 0; + } + + posargs = PyTuple_New(nposargs); + if (posargs == NULL) { + goto error; + } + if (nposargs) { + for (i=0; i < nposargs; i++) { + PyTuple_SET_ITEM(posargs, i, Py_NewRef(*args)); + args++; + } + } + + if (nkwargs) { + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto error; + } + + for (i = 0; i < nkwargs; i++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = *args; + args++; + if (PyDict_SetItem(kwargs, key, value) < 0) { + goto error; + } + } + } + else { + kwargs = NULL; + } + + res = PyObject_Call(callable, posargs, kwargs); + Py_DECREF(posargs); + Py_XDECREF(kwargs); + return res; + +error: + Py_DECREF(posargs); + Py_XDECREF(kwargs); + return NULL; +#endif +} +#endif + + +// gh-106521 added PyObject_GetOptionalAttr() and +// PyObject_GetOptionalAttrString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result) +{ + // bpo-32571 added _PyObject_LookupAttr() to Python 3.7.0b1 +#if PY_VERSION_HEX >= 0x030700B1 && !defined(PYPY_VERSION) + return _PyObject_LookupAttr(obj, attr_name, result); +#else + *result = PyObject_GetAttr(obj, attr_name); + if (*result != NULL) { + return 1; + } + if (!PyErr_Occurred()) { + return 0; + } + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + return 0; + } + return -1; +#endif +} + +static inline int +PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result) +{ + PyObject *name_obj; + int rc; +#if PY_VERSION_HEX >= 0x03000000 + name_obj = PyUnicode_FromString(attr_name); +#else + name_obj = PyString_FromString(attr_name); +#endif + if (name_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyObject_GetOptionalAttr(obj, name_obj, result); + Py_DECREF(name_obj); + return rc; +} +#endif + + +// gh-106307 added PyObject_GetOptionalAttr() and +// PyMapping_GetOptionalItemString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) +{ + *result = PyObject_GetItem(obj, key); + if (*result) { + return 1; + } + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; +} + +static inline int +PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) +{ + PyObject *key_obj; + int rc; +#if PY_VERSION_HEX >= 0x03000000 + key_obj = PyUnicode_FromString(key); +#else + key_obj = PyString_FromString(key); +#endif + if (key_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyMapping_GetOptionalItem(obj, key_obj, result); + Py_DECREF(key_obj); + return rc; +} +#endif + +// gh-108511 added PyMapping_HasKeyWithError() and +// PyMapping_HasKeyStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_HasKeyWithError(PyObject *obj, PyObject *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItem(obj, key, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyMapping_HasKeyStringWithError(PyObject *obj, const char *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItemString(obj, key, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-108511 added PyObject_HasAttrWithError() and +// PyObject_HasAttrStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_HasAttrWithError(PyObject *obj, PyObject *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttr(obj, attr, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyObject_HasAttrStringWithError(PyObject *obj, const char *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttrString(obj, attr, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-106004 added PyDict_GetItemRef() and PyDict_GetItemStringRef() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) +{ +#if PY_VERSION_HEX >= 0x03000000 + PyObject *item = PyDict_GetItemWithError(mp, key); +#else + PyObject *item = _PyDict_GetItemWithError(mp, key); +#endif + if (item != NULL) { + *result = Py_NewRef(item); + return 1; // found + } + if (!PyErr_Occurred()) { + *result = NULL; + return 0; // not found + } + *result = NULL; + return -1; +} + +static inline int +PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result) +{ + int res; +#if PY_VERSION_HEX >= 0x03000000 + PyObject *key_obj = PyUnicode_FromString(key); +#else + PyObject *key_obj = PyString_FromString(key); +#endif + if (key_obj == NULL) { + *result = NULL; + return -1; + } + res = PyDict_GetItemRef(mp, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-106307 added PyModule_Add() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyModule_Add(PyObject *mod, const char *name, PyObject *value) +{ + int res = PyModule_AddObjectRef(mod, name, value); + Py_XDECREF(value); + return res; +} +#endif + + +// gh-108014 added Py_IsFinalizing() to Python 3.13.0a1 +// bpo-1856 added _Py_Finalizing to Python 3.2.1b1. +// _Py_IsFinalizing() was added to PyPy 7.3.0. +#if (0x030201B1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030D00A1) \ + && (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x7030000) +static inline int Py_IsFinalizing(void) +{ +#if PY_VERSION_HEX >= 0x030700A1 + // _Py_IsFinalizing() was added to Python 3.7.0a1. + return _Py_IsFinalizing(); +#else + return (_Py_Finalizing != NULL); +#endif +} +#endif + + +// gh-108323 added PyDict_ContainsString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyDict_ContainsString(PyObject *op, const char *key) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + return -1; + } + int res = PyDict_Contains(op, key_obj); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-108445 added PyLong_AsInt() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyLong_AsInt(PyObject *obj) +{ +#ifdef PYPY_VERSION + long value = PyLong_AsLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + if (value < (long)INT_MIN || (long)INT_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)value; +#else + return _PyLong_AsInt(obj); +#endif +} +#endif + + +// gh-107073 added PyObject_VisitManagedDict() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (dict == NULL || *dict == NULL) { + return -1; + } + Py_VISIT(*dict); + return 0; +} + +static inline void +PyObject_ClearManagedDict(PyObject *obj) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (dict == NULL || *dict == NULL) { + return; + } + Py_CLEAR(*dict); +} +#endif + +// gh-108867 added PyThreadState_GetUnchecked() to Python 3.13.0a1 +// Python 3.5.2 added _PyThreadState_UncheckedGet(). +#if PY_VERSION_HEX >= 0x03050200 && PY_VERSION_HEX < 0x030D00A1 +static inline PyThreadState* +PyThreadState_GetUnchecked(void) +{ + return _PyThreadState_UncheckedGet(); +} +#endif + +// gh-110289 added PyUnicode_EqualToUTF8() and PyUnicode_EqualToUTF8AndSize() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *str, Py_ssize_t str_len) +{ + Py_ssize_t len; + const void *utf8; + PyObject *exc_type, *exc_value, *exc_tb; + int res; + + // API cannot report errors so save/restore the exception + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + // Python 3.3.0a1 added PyUnicode_AsUTF8AndSize() +#if PY_VERSION_HEX >= 0x030300A1 + if (PyUnicode_IS_ASCII(unicode)) { + utf8 = PyUnicode_DATA(unicode); + len = PyUnicode_GET_LENGTH(unicode); + } + else { + utf8 = PyUnicode_AsUTF8AndSize(unicode, &len); + if (utf8 == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + } + + if (len != str_len) { + res = 0; + goto done; + } + res = (memcmp(utf8, str, (size_t)len) == 0); +#else + PyObject *bytes = PyUnicode_AsUTF8String(unicode); + if (bytes == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + +#if PY_VERSION_HEX >= 0x03000000 + len = PyBytes_GET_SIZE(bytes); + utf8 = PyBytes_AS_STRING(bytes); +#else + len = PyString_GET_SIZE(bytes); + utf8 = PyString_AS_STRING(bytes); +#endif + if (len != str_len) { + Py_DECREF(bytes); + res = 0; + goto done; + } + + res = (memcmp(utf8, str, (size_t)len) == 0); + Py_DECREF(bytes); +#endif + +done: + PyErr_Restore(exc_type, exc_value, exc_tb); + return res; +} + +static inline int +PyUnicode_EqualToUTF8(PyObject *unicode, const char *str) +{ + return PyUnicode_EqualToUTF8AndSize(unicode, str, (Py_ssize_t)strlen(str)); +} +#endif + + +// gh-111138 added PyList_Extend() and PyList_Clear() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyList_Extend(PyObject *list, PyObject *iterable) +{ + return PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable); +} + +static inline int +PyList_Clear(PyObject *list) +{ + return PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL); +} +#endif + +// gh-111262 added PyDict_Pop() and PyDict_PopString() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result) +{ + PyObject *value; + + if (!PyDict_Check(dict)) { + PyErr_BadInternalCall(); + if (result) { + *result = NULL; + } + return -1; + } + + // bpo-16991 added _PyDict_Pop() to Python 3.5.0b2. + // Python 3.6.0b3 changed _PyDict_Pop() first argument type to PyObject*. + // Python 3.13.0a1 removed _PyDict_Pop(). +#if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x030500b2 || PY_VERSION_HEX >= 0x030D0000 + value = PyObject_CallMethod(dict, "pop", "O", key); +#elif PY_VERSION_HEX < 0x030600b3 + value = _PyDict_Pop(_Py_CAST(PyDictObject*, dict), key, NULL); +#else + value = _PyDict_Pop(dict, key, NULL); +#endif + if (value == NULL) { + if (result) { + *result = NULL; + } + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; + } + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; +} + +static inline int +PyDict_PopString(PyObject *dict, const char *key, PyObject **result) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + if (result != NULL) { + *result = NULL; + } + return -1; + } + + int res = PyDict_Pop(dict, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +#if PY_VERSION_HEX < 0x030200A4 +// Python 3.2.0a4 added Py_hash_t type +typedef Py_ssize_t Py_hash_t; +#endif + + +// gh-111545 added Py_HashPointer() to Python 3.13.0a3 +#if PY_VERSION_HEX < 0x030D00A3 +static inline Py_hash_t Py_HashPointer(const void *ptr) +{ +#if PY_VERSION_HEX >= 0x030900A4 && !defined(PYPY_VERSION) + return _Py_HashPointer(ptr); +#else + return _Py_HashPointer(_Py_CAST(void*, ptr)); +#endif +} +#endif + + +// Python 3.13a4 added a PyTime API. +// Use the private API added to Python 3.5. +#if PY_VERSION_HEX < 0x030D00A4 && PY_VERSION_HEX >= 0x03050000 +typedef _PyTime_t PyTime_t; +#define PyTime_MIN _PyTime_MIN +#define PyTime_MAX _PyTime_MAX + +static inline double PyTime_AsSecondsDouble(PyTime_t t) +{ return _PyTime_AsSecondsDouble(t); } + +static inline int PyTime_Monotonic(PyTime_t *result) +{ return _PyTime_GetMonotonicClockWithInfo(result, NULL); } + +static inline int PyTime_Time(PyTime_t *result) +{ return _PyTime_GetSystemClockWithInfo(result, NULL); } + +static inline int PyTime_PerfCounter(PyTime_t *result) +{ +#if PY_VERSION_HEX >= 0x03070000 && !defined(PYPY_VERSION) + return _PyTime_GetPerfCounterWithInfo(result, NULL); +#elif PY_VERSION_HEX >= 0x03070000 + // Call time.perf_counter_ns() and convert Python int object to PyTime_t. + // Cache time.perf_counter_ns() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter_ns"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + long long value = PyLong_AsLongLong(res); + Py_DECREF(res); + + if (value == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(value) >= sizeof(PyTime_t)); + *result = (PyTime_t)value; + return 0; +#else + // Call time.perf_counter() and convert C double to PyTime_t. + // Cache time.perf_counter() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + double d = PyFloat_AsDouble(res); + Py_DECREF(res); + + if (d == -1.0 && PyErr_Occurred()) { + return -1; + } + + // Avoid floor() to avoid having to link to libm + *result = (PyTime_t)(d * 1e9); + return 0; +#endif +} + +#endif + +// gh-111389 added hash constants to Python 3.13.0a5. These constants were +// added first as private macros to Python 3.4.0b1 and PyPy 7.3.8. +#if (!defined(PyHASH_BITS) \ + && ((!defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x030400B1) \ + || (defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x03070000 \ + && PYPY_VERSION_NUM >= 0x07030800))) +# define PyHASH_BITS _PyHASH_BITS +# define PyHASH_MODULUS _PyHASH_MODULUS +# define PyHASH_INF _PyHASH_INF +# define PyHASH_IMAG _PyHASH_IMAG +#endif + + +// gh-111545 added Py_GetConstant() and Py_GetConstantBorrowed() +// to Python 3.13.0a6 +#if PY_VERSION_HEX < 0x030D00A6 && !defined(Py_CONSTANT_NONE) + +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 + +static inline PyObject* Py_GetConstant(unsigned int constant_id) +{ + static PyObject* constants[Py_CONSTANT_EMPTY_TUPLE + 1] = {NULL}; + + if (constants[Py_CONSTANT_NONE] == NULL) { + constants[Py_CONSTANT_NONE] = Py_None; + constants[Py_CONSTANT_FALSE] = Py_False; + constants[Py_CONSTANT_TRUE] = Py_True; + constants[Py_CONSTANT_ELLIPSIS] = Py_Ellipsis; + constants[Py_CONSTANT_NOT_IMPLEMENTED] = Py_NotImplemented; + + constants[Py_CONSTANT_ZERO] = PyLong_FromLong(0); + if (constants[Py_CONSTANT_ZERO] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_ONE] = PyLong_FromLong(1); + if (constants[Py_CONSTANT_ONE] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_STR] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_BYTES] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); + if (constants[Py_CONSTANT_EMPTY_TUPLE] == NULL) { + goto fatal_error; + } + // goto dance to avoid compiler warnings about Py_FatalError() + goto init_done; + +fatal_error: + // This case should never happen + Py_FatalError("Py_GetConstant() failed to get constants"); + } + +init_done: + if (constant_id <= Py_CONSTANT_EMPTY_TUPLE) { + return Py_NewRef(constants[constant_id]); + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + +static inline PyObject* Py_GetConstantBorrowed(unsigned int constant_id) +{ + PyObject *obj = Py_GetConstant(constant_id); + Py_XDECREF(obj); + return obj; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t index) +{ + PyObject *item = PyList_GetItem(op, index); + Py_XINCREF(item); + return item; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + PyObject *value; + if (PyDict_GetItemRef(d, key, &value) < 0) { + // get error + if (result) { + *result = NULL; + } + return -1; + } + if (value != NULL) { + // present + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; + } + + // missing: set the item + if (PyDict_SetItem(d, key, default_value) < 0) { + // set error + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = Py_NewRef(default_value); + } + return 0; +} +#endif + +#if PY_VERSION_HEX < 0x030D00B3 +# define Py_BEGIN_CRITICAL_SECTION(op) { +# define Py_END_CRITICAL_SECTION() } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) { +# define Py_END_CRITICAL_SECTION2() } +#endif + +#if PY_VERSION_HEX < 0x030E0000 && PY_VERSION_HEX >= 0x03060000 && !defined(PYPY_VERSION) +typedef struct PyUnicodeWriter PyUnicodeWriter; + +static inline void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + PyMem_Free(writer); +} + +static inline PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == _Py_NULL) { + PyErr_NoMemory(); + return _Py_NULL; + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + return pub_writer; +} + +static inline PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + PyMem_Free(writer); + return str; +} + +static inline int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > 0x10ffff) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + +static inline int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Repr(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)strlen(str); + } + + PyObject *str_obj = PyUnicode_FromStringAndSize(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, + const wchar_t *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)wcslen(str); + } + + PyObject *str_obj = PyUnicode_FromWideChar(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + +static inline int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + PyObject *str = PyUnicode_FromFormatV(format, vargs); + va_end(vargs); + if (str == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} +#endif // PY_VERSION_HEX < 0x030E0000 + +// gh-116560 added PyLong_GetSign() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyLong_GetSign(PyObject *obj, int *sign) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + + *sign = _PyLong_Sign(obj); + return 0; +} +#endif + +// gh-126061 added PyLong_IsPositive/Negative/Zero() to Python in 3.14.0a2 +#if PY_VERSION_HEX < 0x030E00A2 +static inline int PyLong_IsPositive(PyObject *obj) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + return _PyLong_Sign(obj) == 1; +} + +static inline int PyLong_IsNegative(PyObject *obj) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + return _PyLong_Sign(obj) == -1; +} + +static inline int PyLong_IsZero(PyObject *obj) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + return _PyLong_Sign(obj) == 0; +} +#endif + + +// gh-124502 added PyUnicode_Equal() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyUnicode_Equal(PyObject *str1, PyObject *str2) +{ + if (!PyUnicode_Check(str1)) { + PyErr_Format(PyExc_TypeError, "first argument must be str, not %s", + Py_TYPE(str1)->tp_name); + return -1; + } + if (!PyUnicode_Check(str2)) { + PyErr_Format(PyExc_TypeError, "second argument must be str, not %s", + Py_TYPE(str2)->tp_name); + return -1; + } + +#if PY_VERSION_HEX >= 0x030d0000 && !defined(PYPY_VERSION) + PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *str1, PyObject *str2); + + return _PyUnicode_Equal(str1, str2); +#elif PY_VERSION_HEX >= 0x03060000 && !defined(PYPY_VERSION) + return _PyUnicode_EQ(str1, str2); +#elif PY_VERSION_HEX >= 0x03090000 && defined(PYPY_VERSION) + return _PyUnicode_EQ(str1, str2); +#else + return (PyUnicode_Compare(str1, str2) == 0); +#endif +} +#endif + + +// gh-121645 added PyBytes_Join() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) +{ + return _PyBytes_Join(sep, iterable); +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline Py_hash_t Py_HashBuffer(const void *ptr, Py_ssize_t len) +{ +#if PY_VERSION_HEX >= 0x03000000 && !defined(PYPY_VERSION) + PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void *src, Py_ssize_t len); + + return _Py_HashBytes(ptr, len); +#else + Py_hash_t hash; + PyObject *bytes = PyBytes_FromStringAndSize((const char*)ptr, len); + if (bytes == NULL) { + return -1; + } + hash = PyObject_Hash(bytes); + Py_DECREF(bytes); + return hash; +#endif +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyIter_NextItem(PyObject *iter, PyObject **item) +{ + iternextfunc tp_iternext; + + assert(iter != NULL); + assert(item != NULL); + + tp_iternext = Py_TYPE(iter)->tp_iternext; + if (tp_iternext == NULL) { + *item = NULL; + PyErr_Format(PyExc_TypeError, "expected an iterator, got '%s'", + Py_TYPE(iter)->tp_name); + return -1; + } + + if ((*item = tp_iternext(iter))) { + return 1; + } + if (!PyErr_Occurred()) { + return 0; + } + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + return 0; + } + return -1; +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline PyObject* PyLong_FromInt32(int32_t value) +{ + Py_BUILD_ASSERT(sizeof(long) >= 4); + return PyLong_FromLong(value); +} + +static inline PyObject* PyLong_FromInt64(int64_t value) +{ + Py_BUILD_ASSERT(sizeof(long long) >= 8); + return PyLong_FromLongLong(value); +} + +static inline PyObject* PyLong_FromUInt32(uint32_t value) +{ + Py_BUILD_ASSERT(sizeof(unsigned long) >= 4); + return PyLong_FromUnsignedLong(value); +} + +static inline PyObject* PyLong_FromUInt64(uint64_t value) +{ + Py_BUILD_ASSERT(sizeof(unsigned long long) >= 8); + return PyLong_FromUnsignedLongLong(value); +} + +static inline int PyLong_AsInt32(PyObject *obj, int32_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(int) == 4); + int value = PyLong_AsInt(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (int32_t)value; + return 0; +} + +static inline int PyLong_AsInt64(PyObject *obj, int64_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long long) == 8); + long long value = PyLong_AsLongLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (int64_t)value; + return 0; +} + +static inline int PyLong_AsUInt32(PyObject *obj, uint32_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long) >= 4); + unsigned long value = PyLong_AsUnsignedLong(obj); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } +#if SIZEOF_LONG > 4 + if ((unsigned long)UINT32_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C uint32_t"); + return -1; + } +#endif + *pvalue = (uint32_t)value; + return 0; +} + +static inline int PyLong_AsUInt64(PyObject *obj, uint64_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long long) == 8); + unsigned long long value = PyLong_AsUnsignedLongLong(obj); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (uint64_t)value; + return 0; +} +#endif + + +// gh-102471 added import and export API for integers to 3.14.0a2. +#if PY_VERSION_HEX < 0x030E00A2 && PY_VERSION_HEX >= 0x03000000 && !defined(PYPY_VERSION) +// Helpers to access PyLongObject internals. +static inline void +_PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) +{ +#if PY_VERSION_HEX >= 0x030C0000 + op->long_value.lv_tag = (uintptr_t)(1 - sign) | ((uintptr_t)(size) << 3); +#elif PY_VERSION_HEX >= 0x030900A4 + Py_SET_SIZE(op, sign * size); +#else + Py_SIZE(op) = sign * size; +#endif +} + +static inline Py_ssize_t +_PyLong_DigitCount(const PyLongObject *op) +{ +#if PY_VERSION_HEX >= 0x030C0000 + return (Py_ssize_t)(op->long_value.lv_tag >> 3); +#else + return _PyLong_Sign((PyObject*)op) < 0 ? -Py_SIZE(op) : Py_SIZE(op); +#endif +} + +static inline digit* +_PyLong_GetDigits(const PyLongObject *op) +{ +#if PY_VERSION_HEX >= 0x030C0000 + return (digit*)(op->long_value.ob_digit); +#else + return (digit*)(op->ob_digit); +#endif +} + +typedef struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + int8_t digit_endianness; +} PyLongLayout; + +typedef struct PyLongExport { + int64_t value; + uint8_t negative; + Py_ssize_t ndigits; + const void *digits; + Py_uintptr_t _reserved; +} PyLongExport; + +typedef struct PyLongWriter PyLongWriter; + +static inline const PyLongLayout* +PyLong_GetNativeLayout(void) +{ + static const PyLongLayout PyLong_LAYOUT = { + PyLong_SHIFT, + sizeof(digit), + -1, // least significant first + PY_LITTLE_ENDIAN ? -1 : 1, + }; + + return &PyLong_LAYOUT; +} + +static inline int +PyLong_Export(PyObject *obj, PyLongExport *export_long) +{ + if (!PyLong_Check(obj)) { + memset(export_long, 0, sizeof(*export_long)); + PyErr_Format(PyExc_TypeError, "expected int, got %s", + Py_TYPE(obj)->tp_name); + return -1; + } + + // Fast-path: try to convert to a int64_t + PyLongObject *self = (PyLongObject*)obj; + int overflow; +#if SIZEOF_LONG == 8 + long value = PyLong_AsLongAndOverflow(obj, &overflow); +#else + // Windows has 32-bit long, so use 64-bit long long instead + long long value = PyLong_AsLongLongAndOverflow(obj, &overflow); +#endif + Py_BUILD_ASSERT(sizeof(value) == sizeof(int64_t)); + // the function cannot fail since obj is a PyLongObject + assert(!(value == -1 && PyErr_Occurred())); + + if (!overflow) { + export_long->value = value; + export_long->negative = 0; + export_long->ndigits = 0; + export_long->digits = 0; + export_long->_reserved = 0; + } + else { + export_long->value = 0; + export_long->negative = _PyLong_Sign(obj) < 0; + export_long->ndigits = _PyLong_DigitCount(self); + if (export_long->ndigits == 0) { + export_long->ndigits = 1; + } + export_long->digits = _PyLong_GetDigits(self); + export_long->_reserved = (Py_uintptr_t)Py_NewRef(obj); + } + return 0; +} + +static inline void +PyLong_FreeExport(PyLongExport *export_long) +{ + PyObject *obj = (PyObject*)export_long->_reserved; + + if (obj) { + export_long->_reserved = 0; + Py_DECREF(obj); + } +} + +static inline PyLongWriter* +PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) +{ + if (ndigits <= 0) { + PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); + return NULL; + } + assert(digits != NULL); + + PyLongObject *obj = _PyLong_New(ndigits); + if (obj == NULL) { + return NULL; + } + _PyLong_SetSignAndDigitCount(obj, negative?-1:1, ndigits); + + *digits = _PyLong_GetDigits(obj); + return (PyLongWriter*)obj; +} + +static inline void +PyLongWriter_Discard(PyLongWriter *writer) +{ + PyLongObject *obj = (PyLongObject *)writer; + + assert(Py_REFCNT(obj) == 1); + Py_DECREF(obj); +} + +static inline PyObject* +PyLongWriter_Finish(PyLongWriter *writer) +{ + PyObject *obj = (PyObject *)writer; + PyLongObject *self = (PyLongObject*)obj; + Py_ssize_t j = _PyLong_DigitCount(self); + Py_ssize_t i = j; + int sign = _PyLong_Sign(obj); + + assert(Py_REFCNT(obj) == 1); + + // Normalize and get singleton if possible + while (i > 0 && _PyLong_GetDigits(self)[i-1] == 0) { + --i; + } + if (i != j) { + if (i == 0) { + sign = 0; + } + _PyLong_SetSignAndDigitCount(self, sign, i); + } + if (i <= 1) { + long val = sign * (long)(_PyLong_GetDigits(self)[0]); + Py_DECREF(obj); + return PyLong_FromLong(val); + } + + return obj; +} +#endif + + +#if PY_VERSION_HEX < 0x030C00A3 +# define Py_T_SHORT T_SHORT +# define Py_T_INT T_INT +# define Py_T_LONG T_LONG +# define Py_T_FLOAT T_FLOAT +# define Py_T_DOUBLE T_DOUBLE +# define Py_T_STRING T_STRING +# define _Py_T_OBJECT T_OBJECT +# define Py_T_CHAR T_CHAR +# define Py_T_BYTE T_BYTE +# define Py_T_UBYTE T_UBYTE +# define Py_T_USHORT T_USHORT +# define Py_T_UINT T_UINT +# define Py_T_ULONG T_ULONG +# define Py_T_STRING_INPLACE T_STRING_INPLACE +# define Py_T_BOOL T_BOOL +# define Py_T_OBJECT_EX T_OBJECT_EX +# define Py_T_LONGLONG T_LONGLONG +# define Py_T_ULONGLONG T_ULONGLONG +# define Py_T_PYSSIZET T_PYSSIZET + +# if PY_VERSION_HEX >= 0x03000000 && !defined(PYPY_VERSION) +# define _Py_T_NONE T_NONE +# endif + +# define Py_READONLY READONLY +# define Py_AUDIT_READ READ_RESTRICTED +# define _Py_WRITE_RESTRICTED PY_WRITE_RESTRICTED +#endif + + +// gh-127350 added Py_fopen() and Py_fclose() to Python 3.14a4 +#if PY_VERSION_HEX < 0x030E00A4 +static inline FILE* Py_fopen(PyObject *path, const char *mode) +{ +#if 0x030400A2 <= PY_VERSION_HEX && !defined(PYPY_VERSION) + PyAPI_FUNC(FILE*) _Py_fopen_obj(PyObject *path, const char *mode); + + return _Py_fopen_obj(path, mode); +#else + FILE *f; + PyObject *bytes; +#if PY_VERSION_HEX >= 0x03000000 + if (!PyUnicode_FSConverter(path, &bytes)) { + return NULL; + } +#else + if (!PyString_Check(path)) { + PyErr_SetString(PyExc_TypeError, "except str"); + return NULL; + } + bytes = Py_NewRef(path); +#endif + const char *path_bytes = PyBytes_AS_STRING(bytes); + + f = fopen(path_bytes, mode); + Py_DECREF(bytes); + + if (f == NULL) { + PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path); + return NULL; + } + return f; +#endif +} + +static inline int Py_fclose(FILE *file) +{ + return fclose(file); +} +#endif + + +#if 0x03090000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030E0000 && !defined(PYPY_VERSION) +static inline PyObject* +PyConfig_Get(const char *name) +{ + typedef enum { + _PyConfig_MEMBER_INT, + _PyConfig_MEMBER_UINT, + _PyConfig_MEMBER_ULONG, + _PyConfig_MEMBER_BOOL, + _PyConfig_MEMBER_WSTR, + _PyConfig_MEMBER_WSTR_OPT, + _PyConfig_MEMBER_WSTR_LIST, + } PyConfigMemberType; + + typedef struct { + const char *name; + size_t offset; + PyConfigMemberType type; + const char *sys_attr; + } PyConfigSpec; + +#define PYTHONCAPI_COMPAT_SPEC(MEMBER, TYPE, sys_attr) \ + {#MEMBER, offsetof(PyConfig, MEMBER), \ + _PyConfig_MEMBER_##TYPE, sys_attr} + + static const PyConfigSpec config_spec[] = { + PYTHONCAPI_COMPAT_SPEC(argv, WSTR_LIST, "argv"), + PYTHONCAPI_COMPAT_SPEC(base_exec_prefix, WSTR_OPT, "base_exec_prefix"), + PYTHONCAPI_COMPAT_SPEC(base_executable, WSTR_OPT, "_base_executable"), + PYTHONCAPI_COMPAT_SPEC(base_prefix, WSTR_OPT, "base_prefix"), + PYTHONCAPI_COMPAT_SPEC(bytes_warning, UINT, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(exec_prefix, WSTR_OPT, "exec_prefix"), + PYTHONCAPI_COMPAT_SPEC(executable, WSTR_OPT, "executable"), + PYTHONCAPI_COMPAT_SPEC(inspect, BOOL, _Py_NULL), +#if 0x030C0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(int_max_str_digits, UINT, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(interactive, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(module_search_paths, WSTR_LIST, "path"), + PYTHONCAPI_COMPAT_SPEC(optimization_level, UINT, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(parser_debug, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(platlibdir, WSTR, "platlibdir"), + PYTHONCAPI_COMPAT_SPEC(prefix, WSTR_OPT, "prefix"), + PYTHONCAPI_COMPAT_SPEC(pycache_prefix, WSTR_OPT, "pycache_prefix"), + PYTHONCAPI_COMPAT_SPEC(quiet, BOOL, _Py_NULL), +#if 0x030B0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(stdlib_dir, WSTR_OPT, "_stdlib_dir"), +#endif + PYTHONCAPI_COMPAT_SPEC(use_environment, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(verbose, UINT, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(warnoptions, WSTR_LIST, "warnoptions"), + PYTHONCAPI_COMPAT_SPEC(write_bytecode, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(xoptions, WSTR_LIST, "_xoptions"), + PYTHONCAPI_COMPAT_SPEC(buffered_stdio, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(check_hash_pycs_mode, WSTR, _Py_NULL), +#if 0x030B0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(code_debug_ranges, BOOL, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(configure_c_stdio, BOOL, _Py_NULL), +#if 0x030D0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(cpu_count, INT, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(dev_mode, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(dump_refs, BOOL, _Py_NULL), +#if 0x030B0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(dump_refs_file, WSTR_OPT, _Py_NULL), +#endif +#ifdef Py_GIL_DISABLED + PYTHONCAPI_COMPAT_SPEC(enable_gil, INT, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(faulthandler, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(filesystem_encoding, WSTR, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(filesystem_errors, WSTR, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(hash_seed, ULONG, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(home, WSTR_OPT, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(import_time, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(install_signal_handlers, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(isolated, BOOL, _Py_NULL), +#ifdef MS_WINDOWS + PYTHONCAPI_COMPAT_SPEC(legacy_windows_stdio, BOOL, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(malloc_stats, BOOL, _Py_NULL), +#if 0x030A0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(orig_argv, WSTR_LIST, "orig_argv"), +#endif + PYTHONCAPI_COMPAT_SPEC(parse_argv, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(pathconfig_warnings, BOOL, _Py_NULL), +#if 0x030C0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(perf_profiling, UINT, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(program_name, WSTR, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(run_command, WSTR_OPT, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(run_filename, WSTR_OPT, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(run_module, WSTR_OPT, _Py_NULL), +#if 0x030B0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(safe_path, BOOL, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(show_ref_count, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(site_import, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(skip_source_first_line, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(stdio_encoding, WSTR, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(stdio_errors, WSTR, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(tracemalloc, UINT, _Py_NULL), +#if 0x030B0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(use_frozen_modules, BOOL, _Py_NULL), +#endif + PYTHONCAPI_COMPAT_SPEC(use_hash_seed, BOOL, _Py_NULL), + PYTHONCAPI_COMPAT_SPEC(user_site_directory, BOOL, _Py_NULL), +#if 0x030A0000 <= PY_VERSION_HEX + PYTHONCAPI_COMPAT_SPEC(warn_default_encoding, BOOL, _Py_NULL), +#endif + }; + +#undef PYTHONCAPI_COMPAT_SPEC + + const PyConfigSpec *spec; + int found = 0; + for (size_t i=0; i < sizeof(config_spec) / sizeof(config_spec[0]); i++) { + spec = &config_spec[i]; + if (strcmp(spec->name, name) == 0) { + found = 1; + break; + } + } + if (found) { + if (spec->sys_attr != NULL) { + PyObject *value = PySys_GetObject(spec->sys_attr); + if (value == NULL) { + PyErr_Format(PyExc_RuntimeError, "lost sys.%s", spec->sys_attr); + return NULL; + } + return Py_NewRef(value); + } + + PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); + + const PyConfig *config = _Py_GetConfig(); + void *member = (char *)config + spec->offset; + switch (spec->type) { + case _PyConfig_MEMBER_INT: + case _PyConfig_MEMBER_UINT: + { + int value = *(int *)member; + return PyLong_FromLong(value); + } + case _PyConfig_MEMBER_BOOL: + { + int value = *(int *)member; + return PyBool_FromLong(value != 0); + } + case _PyConfig_MEMBER_ULONG: + { + unsigned long value = *(unsigned long *)member; + return PyLong_FromUnsignedLong(value); + } + case _PyConfig_MEMBER_WSTR: + case _PyConfig_MEMBER_WSTR_OPT: + { + wchar_t *wstr = *(wchar_t **)member; + if (wstr != NULL) { + return PyUnicode_FromWideChar(wstr, -1); + } + else { + return Py_NewRef(Py_None); + } + } + case _PyConfig_MEMBER_WSTR_LIST: + { + const PyWideStringList *list = (const PyWideStringList *)member; + PyObject *tuple = PyTuple_New(list->length); + if (tuple == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < list->length; i++) { + PyObject *item = PyUnicode_FromWideChar(list->items[i], -1); + if (item == NULL) { + Py_DECREF(tuple); + return NULL; + } + PyTuple_SET_ITEM(tuple, i, item); + } + return tuple; + } + default: + Py_UNREACHABLE(); + } + } + + PyErr_Format(PyExc_ValueError, "unknown config option name: %s", name); + return NULL; +} + +static inline int +PyConfig_GetInt(const char *name, int *value) +{ + PyObject *obj = PyConfig_Get(name); + if (obj == NULL) { + return -1; + } + + if (!PyLong_Check(obj)) { + Py_DECREF(obj); + PyErr_Format(PyExc_TypeError, "config option %s is not an int", name); + return -1; + } + + int as_int = PyLong_AsInt(obj); + Py_DECREF(obj); + if (as_int == -1 && PyErr_Occurred()) { + PyErr_Format(PyExc_OverflowError, + "config option %s value does not fit into a C int", name); + return -1; + } + + *value = as_int; + return 0; +} +#endif // PY_VERSION_HEX > 0x03090000 && !defined(PYPY_VERSION) + + #ifdef __cplusplus } #endif Binary files 2.9.2-1/bitarray/test_150.pickle and 3.6.1-1/bitarray/test_150.pickle differ diff -pruN 2.9.2-1/bitarray/test_bitarray.py 3.6.1-1/bitarray/test_bitarray.py --- 2.9.2-1/bitarray/test_bitarray.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/test_bitarray.py 2025-08-12 08:35:42.000000000 +0000 @@ -12,7 +12,8 @@ import platform import unittest import shutil import tempfile -from random import getrandbits, randrange, randint, shuffle +from io import BytesIO, UnsupportedOperation +from random import choice, getrandbits, randrange, randint, shuffle # imports needed inside tests import array @@ -24,16 +25,9 @@ import shelve import weakref -is_py3k = bool(sys.version_info[0] == 3) pyodide = bool(platform.machine() == 'wasm32') is_pypy = bool(platform.python_implementation() == 'PyPy') -if is_py3k: - from io import BytesIO -else: - from cStringIO import StringIO as BytesIO # type: ignore - range = xrange # type: ignore - from bitarray import (bitarray, frozenbitarray, bits2bytes, decodetree, get_default_endian, _set_default_endian, @@ -46,14 +40,16 @@ def skipIf(condition): return lambda f: f SYSINFO = _sysinfo() +PTRSIZE = SYSINFO[0] # pointer size in bytes DEBUG = SYSINFO[6] + def buffer_info(a, key=None): fields = ( "address", # 0. address of byte buffer "size", # 1. buffer size in bytes "endian", # 2. bit-endianness - "padding", # 3. number of pad bits + "padbits", # 3. number of pad bits "allocated", # 4. allocated memory "readonly", # 5. memory is read-only "imported", # 6. buffer is imported @@ -72,9 +68,10 @@ def ones(n, endian=None): a.setall(1) return a -def urandom(n, endian=None): - a = bitarray(0, endian) - a.frombytes(os.urandom(bits2bytes(n))) +def urandom_2(n, endian=""): + if endian == "": + endian = choice(['little', 'big']) + a = bitarray(os.urandom(bits2bytes(n)), endian) del a[n:] return a @@ -85,123 +82,84 @@ class Util(object): @staticmethod def random_endian(): - return ['little', 'big'][getrandbits(1)] + return choice(['little', 'big']) - def randombitarrays(self, start=0): + @staticmethod + def randombitarrays(start=0): for n in range(start, 10): - yield urandom(n, self.random_endian()) + yield urandom_2(n) for _ in range(3): - yield urandom(randrange(start, 1000), self.random_endian()) + yield urandom_2(randrange(start, 1000)) def randomlists(self): for a in self.randombitarrays(): yield a.tolist() @staticmethod - def rndsliceidx(length): - if getrandbits(1): - return None - else: - return randint(-length - 5, length + 5) - - @staticmethod def opposite_endian(endian): t = {'little': 'big', 'big': 'little'} return t[endian] @staticmethod - def calc_slicelength(s, length): - assert isinstance(s, slice) - start, stop, step = s.indices(length) - assert step < 0 or (start >= 0 and stop >= 0) - assert step > 0 or (start >= -1 and stop >= -1) - - # This implementation works because Python's floor division (a // b) - # always rounds to the lowest integer, even when a or b are negative. - res1 = (stop - start + (1 if step < 0 else -1)) // step + 1 - if res1 < 0: - res1 = 0 - - # The above implementation is not used in C. - # In C's a / b, if either a or b is negative, the result depends on - # the compiler. Therefore, we use the implementation below (where - # both a and b are always positive). - res2 = 0 - if step < 0: - if stop < start: - res2 = (start - stop - 1) // (-step) + 1 - else: - if start < stop: - res2 = (stop - start - 1) // step + 1 - - assert res1 == res2 - return res1 + def random_slice(n, step=None): + return slice(randint(-n - 2, n + 2), + randint(-n - 2, n + 2), + step or randint(-5, 5) or 1) def check_obj(self, a): self.assertIsInstance(a, bitarray) - ptr, size, endian, padbits, alloc, readonly, buf, exports = \ + ptr, nbytes, endian, padbits, alloc, readonly, buf, exports = \ a.buffer_info() - - self.assertEqual(size, bits2bytes(len(a))) - self.assertEqual(padbits, 8 * size - len(a)) + self.assertEqual(nbytes, bits2bytes(len(a))) self.assertTrue(0 <= padbits < 8) - self.assertEqual(endian, a.endian()) + self.assertEqual(endian, a.endian) self.assertTrue(endian in ('little', 'big')) - - self.assertEqual(a.nbytes, size) + self.assertEqual(a.nbytes, nbytes) self.assertEqual(a.padbits, padbits) self.assertEqual(a.readonly, readonly) - self.assertEqual(len(a) + a.padbits, 8 * a.nbytes) + self.assertEqual(len(a) + padbits, 8 * nbytes) if buf: # imported buffer implies that no extra memory is allocated self.assertEqual(alloc, 0) # an imported buffer will always have a multiple of 8 bits self.assertEqual(len(a) % 8, 0) - self.assertEqual(len(a), 8 * size) + self.assertEqual(len(a), 8 * nbytes) self.assertEqual(padbits, 0) else: # the allocated memory is always larger than the buffer size - self.assertTrue(alloc >= size) + self.assertTrue(alloc >= nbytes) if ptr == 0: # the buffer being a NULL pointer implies that the buffer size # and the allocated memory size are 0 - self.assertEqual(size, 0) + self.assertEqual(nbytes, 0) self.assertEqual(alloc, 0) - if type(a).__name__ == 'frozenbitarray': + if type(a) == frozenbitarray: # frozenbitarray have read-only memory self.assertEqual(readonly, 1) if padbits: # ensure padbits are zero - b = bitarray(endian=endian) - b.frombytes(a.tobytes()[-1:]) - self.assertFalse(b[-padbits:].any()) + b = bitarray(bytes(a)[-1:], endian=endian)[-padbits:] + self.assertEqual(len(b), padbits) + self.assertEqual(b.count(), 0) elif not buf: + self.assertFalse(isinstance(a, frozenbitarray)) # otherwise, unless the buffer is imported, it is writable self.assertEqual(readonly, 0) def assertEQUAL(self, a, b): self.assertEqual(a, b) - self.assertEqual(a.endian(), b.endian()) + self.assertEqual(a.endian, b.endian) def assertIsType(self, a, b): self.assertEqual(type(a).__name__, b) - self.assertEqual( - repr(type(a)), "<%s 'bitarray.%s'>" % - ('class' if is_py3k or b == 'frozenbitarray' else 'type', b)) - - def assertBitEqual(self, x, y): - for z in x, y: - self.assertEqual('01'[z], repr(z)) - self.assertEqual(x, y) - - def assertStopIteration(self, it): - self.assertRaises(StopIteration, next, it) + self.assertEqual(repr(type(a)), "" % b) - def assertRaisesMessage(self, excClass, msg, callable, *args, **kwargs): + @staticmethod + def assertRaisesMessage(excClass, msg, callable, *args, **kwargs): try: callable(*args, **kwargs) raise AssertionError("%s not raised" % excClass.__name__) @@ -211,45 +169,52 @@ class Util(object): # --------------------------------------------------------------------------- -class TestsModuleFunctions(unittest.TestCase, Util): +class ModuleFunctionsTests(unittest.TestCase, Util): def test_version_string(self): # the version string is not a function, but test it here anyway - self.assertIsInstance(__version__, str) + self.assertEqual(type(__version__), str) def test_sysinfo(self): info = _sysinfo() - self.assertIsInstance(info, tuple) + self.assertEqual(info[0], PTRSIZE) + self.assertEqual(info[1], PTRSIZE) + + self.assertEqual(type(info), tuple) for x in info: - self.assertIsInstance(x, int) + self.assertEqual(type(x), int) - if not is_pypy: - self.assertEqual(info[0], tuple.__itemsize__) self.assertEqual(info[7], int(sys.byteorder == 'little')) self.assertEqual(info[8], int(sys.byteorder == 'big')) - self.assertEqual(info[7] + info[8], 1) + + @skipIf(is_pypy) # PyPy doesn't have tuple.__itemsize__ + def test_ptrsize(self): + self.assertEqual(PTRSIZE, tuple.__itemsize__) + + def test_maxsize(self): + self.assertEqual(sys.maxsize, 2 ** (PTRSIZE * 8 - 1) - 1) def test_set_default_endian(self): self.assertRaises(TypeError, _set_default_endian, 0) self.assertRaises(TypeError, _set_default_endian, 'little', 0) self.assertRaises(ValueError, _set_default_endian, 'foo') - for default_endian in 'big', 'little', u'big', u'little': + for default_endian in 'big', 'little': _set_default_endian(default_endian) a = bitarray() - self.assertEqual(a.endian(), default_endian) + self.assertEqual(a.endian, default_endian) for x in None, 0, 64, '10111', [1, 0]: a = bitarray(x) - self.assertEqual(a.endian(), default_endian) + self.assertEqual(a.endian, default_endian) for endian in 'big', 'little', None: a = bitarray(endian=endian) - self.assertEqual(a.endian(), + self.assertEqual(a.endian, default_endian if endian is None else endian) # make sure that calling _set_default_endian wrong does not # change the default endianness self.assertRaises(ValueError, _set_default_endian, 'foobar') - self.assertEqual(bitarray().endian(), default_endian) + self.assertEqual(bitarray().endian, default_endian) def test_get_default_endian(self): # takes no arguments @@ -258,7 +223,7 @@ class TestsModuleFunctions(unittest.Test _set_default_endian(default_endian) endian = get_default_endian() self.assertEqual(endian, default_endian) - self.assertIsInstance(endian, str) + self.assertEqual(type(endian), str) def test_bits2bytes(self): for arg in 'foo', [], None, {}, 187.0, -4.0: @@ -270,18 +235,16 @@ class TestsModuleFunctions(unittest.Test self.assertRaises(ValueError, bits2bytes, -1) self.assertRaises(ValueError, bits2bytes, -924) - self.assertEqual(bits2bytes(0), 0) - for n in range(1, 100): + for n, res in (0, 0), (1, 1), (7, 1), (8, 1), (9, 2): + self.assertEqual(bits2bytes(n), res) + + for n in range(1, 200): m = bits2bytes(n) self.assertEqual(m, (n - 1) // 8 + 1) - self.assertIsInstance(m, int) + self.assertEqual(type(m), int) - for n, m in [(0, 0), (1, 1), (2, 1), (7, 1), (8, 1), (9, 2), - (10, 2), (15, 2), (16, 2), (64, 8), (65, 9), - (2**31, 2**28), (2**32, 2**29), (2**34, 2**31), - (2**34+793, 2**31+100), (2**35-8, 2**32-1), - (2**62, 2**59), (2**63-8, 2**60-1)]: - self.assertEqual(bits2bytes(n), m) + k = (1 << n) + randrange(1000) + self.assertEqual(bits2bytes(k), (k - 1) // 8 + 1) # --------------------------------------------------------------------------- @@ -291,20 +254,18 @@ class CreateObjectTests(unittest.TestCas a = bitarray() self.assertEqual(len(a), 0) self.assertEqual(a.tolist(), []) - self.assertIsType(a, 'bitarray') + self.assertEqual(type(a), bitarray) self.check_obj(a) def test_endian(self): - a = bitarray(endian='little') - a.frombytes(b'ABC') - self.assertEqual(a.endian(), 'little') - self.assertIsInstance(a.endian(), str) + a = bitarray(b"ABC", endian='little') + self.assertEqual(a.endian, 'little') + self.assertEqual(type(a.endian), str) self.check_obj(a) - b = bitarray(endian='big') - b.frombytes(b'ABC') - self.assertEqual(b.endian(), 'big') - self.assertIsInstance(a.endian(), str) + b = bitarray(b"ABC", endian='big') + self.assertEqual(b.endian, 'big') + self.assertEqual(type(a.endian), str) self.check_obj(b) self.assertNotEqual(a, b) @@ -317,8 +278,8 @@ class CreateObjectTests(unittest.TestCas a_little = bitarray() _set_default_endian('big') - self.assertEqual(a_big.endian(), 'big') - self.assertEqual(a_little.endian(), 'little') + self.assertEqual(a_big.endian, 'big') + self.assertEqual(a_little.endian, 'little') def test_endian_wrong(self): self.assertRaises(TypeError, bitarray, endian=0) @@ -338,7 +299,7 @@ class CreateObjectTests(unittest.TestCas _set_default_endian(endian) a = bitarray(buffer=b'A') - self.assertEqual(a.endian(), endian) + self.assertEqual(a.endian, endian) self.assertEqual(len(a), 8) def test_buffer_readonly(self): @@ -350,7 +311,7 @@ class CreateObjectTests(unittest.TestCas self.check_obj(a) @skipIf(is_pypy) - def test_buffer_writeable(self): + def test_buffer_writable(self): a = bitarray(buffer=bytearray([65])) self.assertFalse(a.readonly) a[6] = 1 @@ -386,23 +347,14 @@ class CreateObjectTests(unittest.TestCas self.assertEqual(len(a), n) self.check_obj(a) - if not is_py3k: - a = bitarray(long(29)) - self.assertEqual(len(a), 29) - self.assertRaises(ValueError, bitarray, -1) self.assertRaises(ValueError, bitarray, -924) def test_list(self): - lst = [0, 1, False, True] - a = bitarray(lst) + a = bitarray([0, 1, False, True]) self.assertEqual(a, bitarray('0101')) self.check_obj(a) - if not is_py3k: - a = bitarray([long(1), long(0)]) - self.assertEqual(a, bitarray('10')) - self.assertRaises(ValueError, bitarray, [0, 1, 2]) self.assertRaises(TypeError, bitarray, [0, 1, None]) @@ -412,19 +364,12 @@ class CreateObjectTests(unittest.TestCas self.assertEqual(a.tolist(), lst) self.check_obj(a) - def test_tuple(self): - tup = (0, True, False, 1) - a = bitarray(tup) - self.assertEqual(a, bitarray('0101')) - self.check_obj(a) - - self.assertRaises(ValueError, bitarray, (0, 1, 2)) - self.assertRaises(TypeError, bitarray, (0, 1, None)) - - for n in range(50): - lst = [bool(getrandbits(1)) for d in range(n)] - a = bitarray(tuple(lst)) - self.assertEqual(a.tolist(), lst) + def test_sequence(self): + lst = [0, 1, 1, 1, 0] + for x in lst, tuple(lst), array.array('i', lst): + self.assertEqual(len(x), 5) # sequences have len, iterables not + a = bitarray(x) + self.assertEqual(a, bitarray('01110')) self.check_obj(a) def test_iter1(self): @@ -454,32 +399,25 @@ class CreateObjectTests(unittest.TestCas self.assertRaises(ValueError, bitarray, range(0, 3)) def test_string01(self): - for s in ('0010111', u'0010111', '0010 111', u'0010 111', - '0010_111', u'0010_111'): + for s in '0010111', '0010 111', '0010_111': a = bitarray(s) self.assertEqual(a.tolist(), [0, 0, 1, 0, 1, 1, 1]) self.check_obj(a) - for n in range(50): - lst = [bool(getrandbits(1)) for d in range(n)] + for lst in self.randomlists(): s = ''.join([['0', '1'][x] for x in lst]) a = bitarray(s) self.assertEqual(a.tolist(), lst) self.check_obj(a) - self.assertRaises(ValueError, bitarray, '01021') - self.assertRaises(UnicodeEncodeError, bitarray, u'1\u26050') + self.assertRaises(ValueError, bitarray, '01021') # UCS1 + self.assertRaises(ValueError, bitarray, '1\u26050') # UCS2 + self.assertRaises(ValueError, bitarray, '0\U00010348') # UCS4 def test_string01_whitespace(self): a = bitarray(WHITESPACE) self.assertEqual(a, bitarray()) - # For Python 2 (where strings are bytes), we are in the lucky - # position that none of the valid characters ('\t'=9, '\n'=10, - # '\v'=11, '\r'=13, ' '=32, '0'=48 and '1'=49) are valid pickle - # header bytes (0..7). - # Therefore a string of '0's and '1'a can start with any whitespace - # character, as well as '0' or '1' (obviously). for c in WHITESPACE: a = bitarray(c + '1101110001') self.assertEqual(a, bitarray('1101110001')) @@ -487,61 +425,28 @@ class CreateObjectTests(unittest.TestCas a = bitarray(' 0\n1\r0\t1\v0 ') self.assertEqual(a, bitarray('01010')) - def test_rawbytes(self): # representation used for pickling - for blob, endian, s in [ - (b'\x00', 'little', ''), - (b'\x07\x01', 'little', '1'), - (b'\x07\x80', 'big', '1'), - (b'\x03\xff', 'big', '11111'), - (b'\x00\x0f', 'little', '11110000'), - (b'\x00\xf0', 'big', '11110000'), - (b'\x02\x87\xda', 'big', '10000111 110110') - ]: - a = bitarray(blob, endian) - self.assertEqual(a, bitarray(s)) - self.assertEqual(a.endian(), endian) + def test_bytes_bytearray(self): + for x in b'\x80', bytearray(b'\x80'): + a = bitarray(x, 'little') + self.assertEqual(len(a), 8 * len(x)) + self.assertEqual(a.tobytes(), x) + self.assertEqual(a.to01(), '00000001') self.check_obj(a) - def test_rawbytes_invalid(self): - msg3 = ("cannot extend bitarray with 'bytes', " - "use .pack() or .frombytes() instead") - if is_py3k: - # no bytes will cause TypeError - self.assertRaisesMessage(TypeError, msg3, bitarray, b'') - else: - # no bytes are interpreted as an empty string on Python 2 - self.assertEqual(bitarray(b''), bitarray()) - - for blob in b'\x00', b'\x07\x80': - self.assertRaisesMessage(ValueError, - "endianness missing for pickle", - bitarray, blob) - - for i in range(1, 8): - b = bytes(bytearray([i])) - # this error is raised in newbitarray_from_pickle() - self.assertRaises(ValueError, bitarray, b, 'big') - # Python 2: PyErr_Format() seems to handle "0x%02x" - # incorrectly. E.g. instead of "0x01", I get "0x1" - if is_py3k: - self.assertRaisesMessage(ValueError, - "invalid pickle header byte: 0x%02x" % b[0], - bitarray, b, 'big') - - for i in range(8, 256): - b = bytes(bytearray([i])) - if is_py3k: - # we don't allow bitarrays being created from bytes - self.assertRaises(TypeError, bitarray, b) - continue + for n in range(100): + x = os.urandom(n) + a = bitarray(x) + self.assertEqual(len(a), 8 * n) + self.assertEqual(memoryview(a), x) - # on Python 2 - if b in WHITESPACE + '_01': - # character is valid - self.assertEqual(len(bitarray(b)), 1 if b in '01' else 0) - else: - # character is invalid - self.assertRaises(ValueError, bitarray, b) + def test_byte(self): + for byte, endian, res in [ + (b'\x01', 'little', '10000000'), + (b'\x80', 'little', '00000001'), + (b'\x80', 'big', '10000000'), + (b'\x01', 'big', '00000001')]: + a = bitarray(byte, endian) + self.assertEqual(a.to01(), res) def test_bitarray_simple(self): for n in range(10): @@ -553,7 +458,7 @@ class CreateObjectTests(unittest.TestCas def test_bitarray_endian(self): # Test creating a new bitarray with different endianness from an # existing bitarray. - for endian in 'little', 'big', u'little', u'big': + for endian in 'little', 'big': a = bitarray(endian=endian) b = bitarray(a) self.assertFalse(a is b) @@ -561,15 +466,15 @@ class CreateObjectTests(unittest.TestCas endian2 = self.opposite_endian(endian) b = bitarray(a, endian2) - self.assertEqual(b.endian(), endian2) + self.assertEqual(b.endian, endian2) self.assertEqual(a, b) for a in self.randombitarrays(): - endian2 = self.opposite_endian(a.endian()) + endian2 = self.opposite_endian(a.endian) b = bitarray(a, endian2) self.assertEqual(a, b) - self.assertEqual(b.endian(), endian2) - self.assertNotEqual(a.endian(), b.endian()) + self.assertEqual(b.endian, endian2) + self.assertNotEqual(a.endian, b.endian) def test_bitarray_endianness(self): a = bitarray('11100001', endian='little') @@ -587,35 +492,26 @@ class CreateObjectTests(unittest.TestCas def test_frozenbitarray(self): a = bitarray(frozenbitarray()) self.assertEQUAL(a, bitarray()) - self.assertIsType(a, 'bitarray') + self.assertEqual(type(a), bitarray) for endian in 'little', 'big': a = bitarray(frozenbitarray('011', endian=endian)) self.assertEQUAL(a, bitarray('011', endian)) - self.assertIsType(a, 'bitarray') + self.assertEqual(type(a), bitarray) def test_create_empty(self): - for x in (None, 0, '', list(), tuple(), set(), dict(), u'', - bitarray(), frozenbitarray()): + for x in (None, 0, '', list(), tuple(), set(), dict(), bytes(), + bytearray(), bitarray(), frozenbitarray()): a = bitarray(x) self.assertEqual(len(a), 0) self.assertEQUAL(a, bitarray()) - if is_py3k: - self.assertRaises(TypeError, bitarray, b'') - else: - self.assertEqual(bitarray(b''), bitarray()) - def test_wrong_args(self): # wrong types for x in False, True, Ellipsis, slice(0), 0.0, 0 + 0j: self.assertRaises(TypeError, bitarray, x) - if is_py3k: - self.assertRaises(TypeError, bitarray, b'10') - else: - self.assertEQUAL(bitarray(b'10'), bitarray('10')) # wrong values - for x in -1, 'A': + for x in -1, 'A', '\0', '010\0 11': self.assertRaises(ValueError, bitarray, x) # test second (endian) argument self.assertRaises(TypeError, bitarray, 0, 0) @@ -649,6 +545,17 @@ class ToObjectsTests(unittest.TestCase, for a in self.randombitarrays(): self.assertEqual(tuple(a), tuple(a.tolist())) + def test_bytes_bytearray(self): + for n in range(20): + a = urandom_2(8 * n) + self.assertEqual(a.padbits, 0) + self.assertEqual(bytes(a), a.tobytes()) + self.assertEqual(bytearray(a), a.tobytes()) + + def test_set(self): + for a in self.randombitarrays(): + self.assertEqual(set(a), set(a.tolist())) + # --------------------------------------------------------------------------- class MetaDataTests(unittest.TestCase): @@ -658,272 +565,58 @@ class MetaDataTests(unittest.TestCase): self.assertEqual(a.buffer_info()[1:4], (2, 'little', 3)) info = a.buffer_info() - self.assertIsInstance(info, tuple) + self.assertEqual(type(info), tuple) self.assertEqual(len(info), 8) for i, item in enumerate(info): if i == 2: - self.assertIsInstance(item, str) + self.assertEqual(type(item), str) continue - self.assertIsInstance(item, int) + self.assertEqual(type(item), int) def test_endian(self): for endian in 'big', 'little': a = bitarray(endian=endian) - self.assertEqual(a.endian(), endian) + self.assertEqual(a.endian, endian) - def test_len(self): - for n in range(100): - a = bitarray(n) - self.assertEqual(len(a), n) - -# --------------------------------------------------------------------------- - -@skipIf(not DEBUG) -class InternalTests(unittest.TestCase, Util): +# -------------------------- (Number) index tests --------------------------- - # Internal functionality exposed for the purpose of testing. - # This class will only be part of the test suite in debug mode. - - def test_shift_r8_empty(self): - a = bitarray() - a._shift_r8(0, 0, 3) - self.assertEqual(a, bitarray()) +class GetItemTests(unittest.TestCase, Util): - def test_shift_r8_explicit(self): - x = bitarray('11000100 11111111 11100111 10111111 00001000') - y = bitarray('11000100 00000111 11111111 00111101 00001000') - x._shift_r8(1, 4, 5) - self.assertEqual(x, y) - x._shift_r8(2, 1, 5) # start > stop -- do nothing - self.assertEqual(x, y) - x._shift_r8(0, 5, 0) # shift = 0 -- do nothing - self.assertEqual(x, y) - - x = bitarray('11000100 11110') - y = bitarray('00011000 10011') - x._shift_r8(0, 2, 3) - self.assertEqual(x, y) - - x = bitarray('1100011') - y = bitarray('0110001') - x._shift_r8(0, 1, 1) - self.assertEqual(x, y) - - def test_shift_r8_random(self): - for _ in range(5000): - N = randrange(200) - x = urandom(N, self.random_endian()) - a = randint(0, x.nbytes) - b = randint(a, x.nbytes) - n = randrange(8) - y = x.copy() - y[8 * a : 8 * b] >>= n - s = x.to01() - if a < b: - s = s[:8 * a] + n * "0" + s[8 * a : 8 * b - n] + s[8 * b:] - if 8 * b > N: - s = s[:N] - x._shift_r8(a, b, n) - self.assertEqual(x.to01(), s) - self.assertEqual(x, y) - self.assertEqual(x.endian(), y.endian()) - self.assertEqual(len(x), N) - - def test_copy_n_explicit(self): - x = bitarray('11000100 11110') - # ^^^^ ^ - y = bitarray('0101110001') - # ^^^^^ - x._copy_n(4, y, 1, 5) - self.assertEqual(x, bitarray('11001011 11110')) - # ^^^^ ^ - x = bitarray('10110111 101', 'little') - y = x.copy() - x._copy_n(3, x, 3, 7) # copy region of x onto x - self.assertEqual(x, y) - x._copy_n(3, bitarray(x, 'big'), 3, 7) # as before but other endian - self.assertEqual(x, y) - x._copy_n(5, bitarray(), 0, 0) # copy empty bitarray onto x - self.assertEqual(x, y) - - def test_copy_n_example(self): - # example given in examples/copy_n.py - y = bitarray( - '00101110 11111001 01011101 11001011 10110000 01011110 011') - x = bitarray( - '01011101 11100101 01110101 01011001 01110100 10001010 01111011') - x._copy_n(21, y, 6, 31) - self.assertEqual(x, bitarray( - '01011101 11100101 01110101 11110010 10111011 10010111 01101011')) - - def check_copy_n(self, N, M, a, b, n): - x = urandom(N, self.random_endian()) - x_lst = x.tolist() - y = x if M < 0 else urandom(M, self.random_endian()) - y_lst = y.tolist() - x_lst[a:a + n] = y_lst[b:b + n] - x._copy_n(a, y, b, n) - self.assertEqual(x, bitarray(x_lst)) - self.assertEqual(len(x), N) - self.check_obj(x) - - if M < 0: - return - self.assertEqual(y, bitarray(y_lst)) - self.assertEqual(len(y), M) - self.check_obj(y) - - def test_copy_n_random(self): - for repeat, max_size in (1000, 25), (100, 200): - for _ in range(repeat): - N = randrange(max_size) - n = randint(0, N) - a = randint(0, N - n) - b = randint(0, N - n) - self.check_copy_n(N, -1, a, b, n) - - M = randrange(max_size) - n = randint(0, min(N, M)) - a = randint(0, N - n) - b = randint(0, M - n) - self.check_copy_n(N, M, a, b, n) - - @staticmethod - def getslice(a, start, slicelength): - # this is the Python eqivalent of __getitem__ for slices with step=1 - b = bitarray(slicelength, a.endian()) - b._copy_n(0, a, start, slicelength) - return b - - def test_getslice(self): - for a in self.randombitarrays(): - a_lst = a.tolist() - n = len(a) - i = randint(0, n) - j = randint(i, n) - b = self.getslice(a, i, j - i) - self.assertEqual(b.tolist(), a_lst[i:j]) - self.assertEQUAL(b, a[i:j]) - - def check_overlap(self, a, b, res): - r1 = a._overlap(b) - r2 = b._overlap(a) - self.assertIsInstance(r1, bool) - self.assertTrue(r1 == r2 == res) - self.check_obj(a) - self.check_obj(b) - - def test_overlap_empty(self): - a = bitarray() - self.check_overlap(a, a, False) - b = bitarray() - self.check_overlap(a, b, False) - - def test_overlap_distinct(self): - for a in self.randombitarrays(): - # buffers overlaps with itself, unless buffer is NULL - self.check_overlap(a, a, bool(a)) - b = a.copy() - self.check_overlap(a, b, False) - - def test_overlap_shared(self): - a = bitarray(64) - b = bitarray(buffer=a) - self.check_overlap(b, a, True) - - c = bitarray(buffer=memoryview(a)[2:4]) - self.check_overlap(c, a, True) - - d = bitarray(buffer=memoryview(a)[5:]) - self.check_overlap(d, c, False) - self.check_overlap(d, b, True) - - e = bitarray(buffer=memoryview(a)[3:3]) - self.check_overlap(e, c, False) - self.check_overlap(e, d, False) - - def test_overlap_shared_random(self): - n = 100 # buffer size in bytes - a = bitarray(8 * n) - for _ in range(1000): - i1 = randint(0, n) - j1 = randint(i1, n) - b1 = bitarray(buffer=memoryview(a)[i1:j1]) - - i2 = randint(0, n) - j2 = randint(i2, n) - b2 = bitarray(buffer=memoryview(a)[i2:j2]) - - x1, x2 = zeros(n), zeros(n) - x1[i1:j1] = x2[i2:j2] = 1 - self.check_overlap(b1, b2, (x1 & x2).any()) - -# --------------------------------------------------------------------------- - -class SliceTests(unittest.TestCase, Util): - - def test_getitem_1(self): + def test_explicit(self): a = bitarray() self.assertRaises(IndexError, a.__getitem__, 0) a.append(True) - self.assertBitEqual(a[0], 1) - self.assertBitEqual(a[-1], 1) + self.assertEqual(a[0], 1) + self.assertEqual(a[-1], 1) self.assertRaises(IndexError, a.__getitem__, 1) self.assertRaises(IndexError, a.__getitem__, -2) a.append(False) - self.assertBitEqual(a[1], 0) - self.assertBitEqual(a[-1], 0) + self.assertEqual(a[1], 0) + self.assertEqual(a[-1], 0) self.assertRaises(IndexError, a.__getitem__, 2) self.assertRaises(IndexError, a.__getitem__, -3) self.assertRaises(TypeError, a.__getitem__, 1.5) self.assertRaises(TypeError, a.__getitem__, None) self.assertRaises(TypeError, a.__getitem__, 'A') - def test_getitem_2(self): + def test_basic(self): a = bitarray('1100010') - for i, b in enumerate(a): - self.assertBitEqual(a[i], b) - self.assertBitEqual(a[i - 7], b) + for i, v in enumerate(a): + self.assertEqual(a[i], v) + self.assertEqual(type(a[i]), int) + self.assertEqual(a[-7 + i], v) self.assertRaises(IndexError, a.__getitem__, 7) self.assertRaises(IndexError, a.__getitem__, -8) - def test_getslice(self): - a = bitarray('01001111 00001') - self.assertEQUAL(a[:], a) - self.assertFalse(a[:] is a) - self.assertEQUAL(a[13:2:-3], bitarray('1010')) - self.assertEQUAL(a[2:-1:4], bitarray('010')) - self.assertEQUAL(a[::2], bitarray('0011001')) - self.assertEQUAL(a[8:], bitarray('00001')) - self.assertEQUAL(a[7:], bitarray('100001')) - self.assertEQUAL(a[:8], bitarray('01001111')) - self.assertEQUAL(a[::-1], bitarray('10000111 10010')) - self.assertEQUAL(a[:8:-1], bitarray('1000')) - - self.assertRaises(ValueError, a.__getitem__, slice(None, None, 0)) - - def test_getslice_random(self): - for a in self.randombitarrays(start=1): + def test_range(self): + for a in self.randombitarrays(): aa = a.tolist() - la = len(a) - for _ in range(10): - step = self.rndsliceidx(la) or None - s = slice(self.rndsliceidx(la), self.rndsliceidx(la), step) - self.assertEQUAL(a[s], bitarray(aa[s], endian=a.endian())) + for i in range(len(a)): + self.assertEqual(a[i], aa[i]) - def test_getslice_random_step1(self): - for _ in range(1000): - n = randrange(200) - a = urandom(n, self.random_endian()) - sa = a.to01() - i = randint(0, n) - j = randint(0, n) - b = a[i:j] - self.assertEqual(b.to01(), sa[i:j]) - self.assertEqual(len(b), max(j - i, 0)) - self.assertEqual(b.endian(), a.endian()) +class SetItemTests(unittest.TestCase, Util): - def test_setitem_simple(self): + def test_explicit(self): a = bitarray('0') a[0] = 1 self.assertEqual(a, bitarray('1')) @@ -946,42 +639,126 @@ class SliceTests(unittest.TestCase, Util self.assertRaises(TypeError, a.__setitem__, 'a', True) self.assertEqual(a, bitarray('10')) - def test_setitem_random(self): + def test_random(self): for a in self.randombitarrays(start=1): i = randrange(len(a)) aa = a.tolist() - val = bool(getrandbits(1)) - a[i] = val - aa[i] = val + v = getrandbits(1) + a[i] = v + aa[i] = v self.assertEqual(a.tolist(), aa) self.check_obj(a) - def test_setslice_simple(self): + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([5, 1, 2, 3]) + b = bitarray(endian="little", buffer=a) + self.assertFalse(b.readonly) + # operate on imported (writable) buffer + b[7] = 1 + self.assertEqual(a, bytearray([0x85, 1, 2, 3])) + b[-2] = 1 + self.assertEqual(a, bytearray([0x85, 1, 2, 0x43])) + +class DelItemTests(unittest.TestCase, Util): + + def test_simple(self): + a = bitarray('100110') + del a[1] + self.assertEqual(len(a), 5) + del a[3], a[-2] + self.assertEqual(a, bitarray('100')) + self.assertRaises(IndexError, a.__delitem__, 3) + self.assertRaises(IndexError, a.__delitem__, -4) + + def test_random(self): for a in self.randombitarrays(start=1): - la = len(a) - b = bitarray(la) - b[0:la] = bitarray(a) + n = len(a) + b = a.copy() + i = randrange(n) + del b[i] + self.assertEQUAL(b, a[:i] + a[i + 1:]) + self.assertEqual(len(b), n - 1) + self.check_obj(b) + + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([5, 11, 7]) + b = bitarray(buffer=a) + self.assertFalse(b.readonly) + self.assertRaises(BufferError, b.__delitem__, 13) + +# -------------------------- Slice index tests ------------------------------ + +class GetSliceTests(unittest.TestCase, Util): + + def test_slice(self): + a = bitarray('01001111 00001') + self.assertEQUAL(a[:], a) + self.assertFalse(a[:] is a) + self.assertEQUAL(a[13:2:-3], bitarray('1010')) + self.assertEQUAL(a[2:-1:4], bitarray('010')) + self.assertEQUAL(a[::2], bitarray('0011001')) + self.assertEQUAL(a[8:], bitarray('00001')) + self.assertEQUAL(a[7:], bitarray('100001')) + self.assertEQUAL(a[:8], bitarray('01001111')) + self.assertEQUAL(a[::-1], bitarray('10000111 10010')) + self.assertEQUAL(a[:8:-1], bitarray('1000')) + self.assertRaises(ValueError, a.__getitem__, slice(None, None, 0)) + + def test_reverse(self): + for _ in range(20): + n = randrange(200) + a = urandom_2(n) + b = a.copy() + b.reverse() + self.assertEQUAL(a[::-1], b) + + def test_random_step1(self): + for n in range(200): + a = urandom_2(n) + i = randint(0, n) + j = randint(0, n) + b = a[i:j] + self.assertEqual(b.to01(), a.to01()[i:j]) + self.assertEqual(len(b), max(j - i, 0)) + self.assertEqual(b.endian, a.endian) + + def test_random(self): + for n in range(200): + a = urandom_2(n) + s = self.random_slice(n) + b = a[s] + self.assertEqual(len(b), len(range(n)[s])) # slicelength + self.assertEqual(list(b), a.tolist()[s]) + self.assertEqual(b.endian, a.endian) + +class SetSliceTests(unittest.TestCase, Util): + + def test_simple(self): + for a in self.randombitarrays(start=1): + n = len(a) + b = bitarray(n) + b[0:n] = bitarray(a) self.assertEqual(a, b) self.assertFalse(a is b) - b = bitarray(la) + b = bitarray(n) b[:] = bitarray(a) self.assertEqual(a, b) self.assertFalse(a is b) - b = bitarray(la) - b[::-1] = bitarray(a) - self.assertEqual(a.tolist()[::-1], b.tolist()) + b = bitarray(n) + b[::-1] = a + self.assertEqual(b.tolist(), a.tolist()[::-1]) - def test_setslice_random(self): + def test_random(self): for a in self.randombitarrays(start=1): - la = len(a) + len_a = len(a) for _ in range(10): - step = self.rndsliceidx(la) or None - s = slice(self.rndsliceidx(la), self.rndsliceidx(la), step) - lb = (randrange(10) if step is None else - self.calc_slicelength(s, la)) - b = bitarray(lb) + s = self.random_slice(len_a) + len_b = randrange(10) if s.step == 1 else len(range(len_a)[s]) + b = bitarray(len_b) c = bitarray(a) c[s] = b self.check_obj(c) @@ -989,20 +766,23 @@ class SliceTests(unittest.TestCase, Util cc[s] = b.tolist() self.assertEqual(c, bitarray(cc)) - def test_setslice_self_random(self): + def test_self_random(self): for a in self.randombitarrays(): + n = len(a) for step in -1, 1: s = slice(None, None, step) + # ensure slicelength equals len(a) + self.assertEqual(len(range(n)[s]), n) aa = a.tolist() a[s] = a aa[s] = aa self.assertEqual(a, bitarray(aa)) - def test_setslice_special(self): + def test_special(self): for n in 0, 1, 10, 87: - a = urandom(n) + a = urandom_2(n) for m in 0, 1, 10, 99: - x = urandom(m) + x = urandom_2(m) b = a.copy() b[n:n] = x # insert at end - extend self.assertEqual(b, a + x) @@ -1012,44 +792,42 @@ class SliceTests(unittest.TestCase, Util self.check_obj(b) self.assertEqual(len(b), len(a) + 2 * len(x)) - def test_setslice_range(self): + def test_range(self): # tests C function insert_n() - for endian in 'big', 'little': - for n in range(500): - a = urandom(n, endian) - p = randint(0, n) - m = randint(0, 500) - - x = urandom(m, self.random_endian()) - b = a.copy() - b[p:p] = x - self.assertEQUAL(b, a[:p] + x + a[p:]) - self.assertEqual(len(b), len(a) + m) - self.check_obj(b) + for _ in range(100): + n = randrange(200) + a = urandom_2(n) + p = randint(0, n) + m = randint(0, 500) + x = urandom_2(m) + b = a.copy() + b[p:p] = x + self.assertEQUAL(b, a[:p] + x + a[p:]) + self.assertEqual(len(b), len(a) + m) + self.check_obj(b) - def test_setslice_resize(self): - N, M = 200, 300 - for endian in 'big', 'little': - for n in 0, randint(0, N), N: - a = urandom(n, endian) - for p1 in 0, randint(0, n), n: - for p2 in 0, randint(0, p1), p1, randint(0, n), n: - for m in 0, randint(0, M), M: - x = urandom(m, self.random_endian()) - b = a.copy() - b[p1:p2] = x - b_lst = a.tolist() - b_lst[p1:p2] = x.tolist() - self.assertEqual(b.tolist(), b_lst) - if p1 <= p2: - self.assertEQUAL(b, a[:p1] + x + a[p2:]) - self.assertEqual(len(b), n + p1 - p2 + len(x)) - else: - self.assertEqual(b, a[:p1] + x + a[p1:]) - self.assertEqual(len(b), n + len(x)) - self.check_obj(b) + def test_resize(self): + for _ in range(100): + n = randint(0, 200) + a = urandom_2(n) + p1 = randint(0, n) + p2 = randint(0, n) + m = randint(0, 300) + x = urandom_2(m) + b = a.copy() + b[p1:p2] = x + b_lst = a.tolist() + b_lst[p1:p2] = x.tolist() + self.assertEqual(b.tolist(), b_lst) + if p1 <= p2: + self.assertEQUAL(b, a[:p1] + x + a[p2:]) + self.assertEqual(len(b), n + p1 - p2 + len(x)) + else: + self.assertEqual(b, a[:p1] + x + a[p1:]) + self.assertEqual(len(b), n + len(x)) + self.check_obj(b) - def test_setslice_self(self): + def test_self(self): a = bitarray('1100111') a[::-1] = a self.assertEqual(a, bitarray('1110011')) @@ -1072,7 +850,7 @@ class SliceTests(unittest.TestCase, Util a[:] = a self.assertEqual(a, bitarray('010111')) - def test_setslice_self_shared_buffer(self): + def test_self_shared_buffer(self): # This is a special case. We have two bitarrays which share the # same buffer, and then do a slice assignment. The bitarray is # copied onto itself in reverse order. So we need to make a copy @@ -1085,7 +863,7 @@ class SliceTests(unittest.TestCase, Util self.assertEqual(a, b) self.assertEqual(a, bitarray('00000111')) - def test_setslice_self_shared_buffer_2(self): + def test_self_shared_buffer_2(self): # This is an even more special case. We have a bitarrays which # shares part of anothers bitarray buffer. So in setslice_bitarray(), # we need to make a copy of other if: @@ -1101,7 +879,7 @@ class SliceTests(unittest.TestCase, Util self.assertEqual(a, bitarray('11111111 00000011 00000000')) @skipIf(is_pypy) - def test_setslice_self_shared_buffer_3(self): + def test_self_shared_buffer_3(self): # Requires to check for (in setslice_bitarray()): # # other->ob_item <= self->ob_item <= other->ob_item + Py_SIZE(other) @@ -1137,7 +915,7 @@ class SliceTests(unittest.TestCase, Util a[:-6:-1] = bitarray('10111') self.assertEqual(a, bitarray('00000001 1101')) - def test_setslice_bitarray_2(self): + def test_bitarray_2(self): a = bitarray('1111') a[3:3] = bitarray('000') # insert self.assertEqual(a, bitarray('1110001')) @@ -1157,62 +935,59 @@ class SliceTests(unittest.TestCase, Util a[2:5] = bitarray('0') # remove self.assertEqual(a, bitarray('11011')) - def test_setslice_frozenbitarray(self): + def test_frozenbitarray(self): a = bitarray('11111111 1111') b = frozenbitarray('0000') a[2:6] = b self.assertEqual(a, bitarray('11000011 1111')) - self.assertIsType(b, 'frozenbitarray') + self.assertEqual(type(b), frozenbitarray) self.assertEqual(b, bitarray('0000')) b = frozenbitarray('011100') a[::2] = b self.assertEqual(a, bitarray('01101011 0101')) self.check_obj(a) - self.assertIsType(b, 'frozenbitarray') + self.assertEqual(type(b), frozenbitarray) self.assertEqual(b, bitarray('011100')) def test_setslice_bitarray_random_same_length(self): - for endian in 'little', 'big': - for _ in range(100): - n = randrange(200) - a = urandom(n, endian) - lst_a = a.tolist() - b = urandom(randint(0, n), self.random_endian()) - lst_b = b.tolist() - i = randint(0, n - len(b)) - j = i + len(b) - self.assertEqual(j - i, len(b)) - a[i:j] = b - lst_a[i:j] = lst_b - self.assertEqual(a.tolist(), lst_a) - # a didn't change length - self.assertEqual(len(a), n) - self.assertEqual(a.endian(), endian) - self.check_obj(a) + for _ in range(100): + n = randrange(200) + a = urandom_2(n) + lst_a = a.tolist() + b = urandom_2(randint(0, n)) + lst_b = b.tolist() + i = randint(0, n - len(b)) + j = i + len(b) + self.assertEqual(j - i, len(b)) + a[i:j] = b + lst_a[i:j] = lst_b + self.assertEqual(a.tolist(), lst_a) + # a didn't change length + self.assertEqual(len(a), n) + self.check_obj(a) - def test_setslice_bitarray_random_step_1(self): + def test_bitarray_random_step1(self): for _ in range(50): n = randrange(300) - a = urandom(n, self.random_endian()) + a = urandom_2(n) lst_a = a.tolist() - b = urandom(randint(0, 100), self.random_endian()) + b = urandom_2(randrange(100)) lst_b = b.tolist() - s = slice(self.rndsliceidx(n), self.rndsliceidx(n), None) + s = self.random_slice(n, step=1) a[s] = b lst_a[s] = lst_b self.assertEqual(a.tolist(), lst_a) self.check_obj(a) - def test_setslice_bitarray_random(self): + def test_bitarray_random(self): for _ in range(100): n = randrange(50) - a = urandom(n, self.random_endian()) + a = urandom_2(n) lst_a = a.tolist() - b = urandom(randrange(50), self.random_endian()) + b = urandom_2(randrange(50)) lst_b = b.tolist() - s = slice(self.rndsliceidx(n), self.rndsliceidx(n), - randint(-3, 3) or None) + s = self.random_slice(n) try: a[s] = b except ValueError: @@ -1229,7 +1004,7 @@ class SliceTests(unittest.TestCase, Util self.assertEqual(a.tolist(), lst_a) self.check_obj(a) - def test_setslice_bool_explicit(self): + def test_bool_explicit(self): a = bitarray('11111111') a[::2] = False self.assertEqual(a, bitarray('01010101')) @@ -1254,57 +1029,53 @@ class SliceTests(unittest.TestCase, Util a[-2:2:-1] = 1 # ^^^^ self.assertEqual(a, bitarray('00011110')) - def test_setslice_bool_simple(self): + def test_bool_step1(self): for _ in range(100): - N = randint(100, 2000) - s = slice(randint(0, 20), randint(N - 20, N), randint(1, 20)) - a = zeros(N) - a[s] = 1 - b = zeros(N) - b[list(range(s.start, s.stop, s.step))] = 1 + n = randrange(1000) + start = randint(0, n) + stop = randint(start, n) + a = bitarray(n) + a[start:stop] = 1 + self.assertEqual(a.count(1), stop - start) + b = bitarray(n) + b[range(start, stop)] = 1 self.assertEqual(a, b) - def test_setslice_bool_range(self): - N = 200 - a = bitarray(N, self.random_endian()) - b = bitarray(N) - for step in range(-N - 1, N): - if step == 0: - continue + def test_setslice_bool_random_slice(self): + for _ in range(200): + n = randrange(100) + a = urandom_2(n) + aa = a.tolist() + s = self.random_slice(n) + slicelength = len(range(n)[s]) v = getrandbits(1) - a.setall(not v) - a[::step] = v - - b.setall(not v) - b[list(range(0, N, abs(step)))] = v - if step < 0: - b.reverse() - self.assertEqual(a, b) + a[s] = v + aa[s] = slicelength * [v] + self.assertEqual(a.tolist(), aa) - def test_setslice_bool_random(self): - N = 100 - a = bitarray(N) - for _ in range(100): a.setall(0) - aa = a.tolist() - step = self.rndsliceidx(N) or None - s = slice(self.rndsliceidx(N), self.rndsliceidx(N), step) a[s] = 1 - aa[s] = self.calc_slicelength(s, N) * [1] - self.assertEqual(a.tolist(), aa) + self.assertEqual(a.count(1), slicelength) - def test_setslice_bool_random2(self): - for a in self.randombitarrays(): - n = len(a) + def test_setslice_bool_step(self): + # this test exercises set_range() when stop is much larger than start + cnt = 0 + for _ in range(500): + n = randrange(3000, 4000) + a = urandom_2(n) aa = a.tolist() - step = self.rndsliceidx(n) or None - s = slice(self.rndsliceidx(n), self.rndsliceidx(n), step) + s = slice(randrange(1000), randrange(1000, n), randint(1, 100)) + self.assertTrue(s.stop - s.start >= 0) + cnt += s.stop - s.start >= 1024 + slicelength = len(range(n)[s]) + self.assertTrue(slicelength > 0) v = getrandbits(1) a[s] = v - aa[s] = self.calc_slicelength(s, n) * [v] + aa[s] = slicelength * [v] self.assertEqual(a.tolist(), aa) + self.assertTrue(cnt > 300) - def test_setslice_to_int(self): + def test_to_int(self): a = bitarray('11111111') a[::2] = 0 # ^ ^ ^ ^ self.assertEqual(a, bitarray('01010101')) @@ -1317,7 +1088,7 @@ class SliceTests(unittest.TestCase, Util # a[:2:] = '0' self.assertRaises(TypeError, a.__setitem__, slice(None, 2, None), '0') - def test_setslice_to_invalid(self): + def test_invalid(self): a = bitarray('11111111') s = slice(2, 6, None) self.assertRaises(TypeError, a.__setitem__, s, 1.2) @@ -1340,26 +1111,22 @@ class SliceTests(unittest.TestCase, Util a.__setitem__, slice(None, None, -1), bitarray('0001000')) self.assertEqual(a, bitarray('11000011')) - def test_delitem_simple(self): - a = bitarray('100110') - del a[1] - self.assertEqual(len(a), 5) - del a[3], a[-2] - self.assertEqual(a, bitarray('100')) - self.assertRaises(IndexError, a.__delitem__, 3) - self.assertRaises(IndexError, a.__delitem__, -4) + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([5, 1, 2, 3]) + b = bitarray(endian="little", buffer=a) + self.assertFalse(b.readonly) + # operate on imported (writable) buffer + b[8:-8] = 1 + self.assertEqual(a, bytearray([5, 0xff, 0xff, 3])) + b[8:-8:2] = 0 + self.assertEqual(a, bytearray([5, 0xaa, 0xaa, 3])) + b[8:20] = bitarray('11110000 0011') + self.assertEqual(a, bytearray([5, 0x0f, 0xac, 3])) - def test_delitem_random(self): - for a in self.randombitarrays(start=1): - n = len(a) - b = a.copy() - i = randrange(n) - del b[i] - self.assertEQUAL(b, a[:i] + a[i + 1:]) - self.assertEqual(len(b), n - 1) - self.check_obj(b) +class DelSliceTests(unittest.TestCase, Util): - def test_delslice_explicit(self): + def test_explicit(self): a = bitarray('10101100 10110') del a[3:9] # ^^^^^ ^ self.assertEqual(a, bitarray('1010110')) @@ -1378,127 +1145,280 @@ class SliceTests(unittest.TestCase, Util del a[:] self.assertEqual(a, bitarray()) - def test_delslice_special(self): + def test_special(self): for n in 0, 1, 10, 73: - a = urandom(n) + a = urandom_2(n) b = a.copy() del b[:0] del b[n:] self.assertEqual(b, a) - del b[10:] # delete at end + del b[10:] # delete everything after 10th item self.assertEqual(b, a[:10]) del b[:] # clear self.assertEqual(len(b), 0) self.check_obj(b) - def test_delslice_random(self): - for a in self.randombitarrays(): - la = len(a) - for _ in range(10): - step = self.rndsliceidx(la) or None - s = slice(self.rndsliceidx(la), self.rndsliceidx(la), step) - c = a.copy() - del c[s] - self.check_obj(c) - c_lst = a.tolist() - del c_lst[s] - self.assertEQUAL(c, bitarray(c_lst, endian=c.endian())) + def test_random(self): + for _ in range(100): + n = randrange(500) + a = urandom_2(n) + s = self.random_slice(n) + slicelength = len(range(n)[s]) + c = a.copy() + del c[s] + self.assertEqual(len(c), n - slicelength) + self.check_obj(c) + c_lst = a.tolist() + del c_lst[s] + self.assertEQUAL(c, bitarray(c_lst, endian=c.endian)) - def test_delslice_range(self): + def test_range(self): # tests C function delete_n() - for n in range(500): - a = urandom(n, self.random_endian()) + for n in range(200): + a = urandom_2(n) p = randint(0, n) - m = randint(0, 500) + m = randint(0, 200) b = a.copy() del b[p:p + m] self.assertEQUAL(b, a[:p] + a[p + m:]) self.check_obj(b) - def test_delslice_range_step(self): - N = 200 - for step in range(-N - 1, N): + def test_range_step(self): + n = 200 + for step in range(-n - 1, n): if step == 0: continue - a = urandom(N, self.random_endian()) + a = urandom_2(n) lst = a.tolist() del a[::step] del lst[::step] self.assertEqual(a.tolist(), lst) -# --------------------------------------------------------------------------- + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([5, 1, 2, 3]) + b = bitarray(buffer=a) + self.assertFalse(b.readonly) + self.assertRaises(BufferError, b.__delitem__, slice(3, 21)) + # even though we don't delete anything, raise error + self.assertRaises(BufferError, b.__delitem__, slice(3, 3)) + +# ----------------------------- Masked index tests -------------------------- -class MaskedIndexTests(unittest.TestCase, Util): +class GetMaskedIndexTests(unittest.TestCase, Util): - def test_get_basic(self): + def test_basic(self): a = bitarray('1001001') mask = bitarray('1010111') self.assertEqual(a[mask], bitarray('10001')) self.assertRaises(IndexError, a.__getitem__, bitarray('1011')) - def test_get_random(self): + def test_random(self): for a in self.randombitarrays(): n = len(a) + # select items from a wehre a is 1 -> all 1 items self.assertEqual(a[a], a.count() * bitarray('1')) mask = zeros(n) self.assertEqual(a[mask], bitarray()) - mask.setall(1) + mask = ones(n) self.assertEqual(a[mask], a) - mask = urandom(n) - res = bitarray(a[i] for i in range(n) if mask[i]) - self.assertEqual(a[mask], res) + mask = urandom_2(n) + self.assertEqual(list(a[mask]), + [a[i] for i in range(n) if mask[i]]) + + def test_random_slice_mask(self): + for n in range(100): + s = self.random_slice(n, step=randint(1, 5)) + a = urandom_2(n) + mask = zeros(n) + mask[s] = 1 + self.assertEQUAL(a[mask], a[s]) + +class SetMaskedIndexTests(unittest.TestCase, Util): - def test_set_basic(self): + def test_basic(self): a = bitarray('1001001') mask = bitarray('1010111') + val = bitarray("0 1 110") + res = bitarray("0011110") self.assertRaises(NotImplementedError, a.__setitem__, mask, 1) + self.assertRaises(ValueError, a.__setitem__, mask, 2) + a[mask] = val + self.assertEqual(a, res) + b = bitarray('0111') + self.assertRaisesMessage( + IndexError, + "attempt to assign mask of size 5 to bitarray of size 4", + a.__setitem__, mask, b) + + def test_issue225(self): + # example from issue #225 + a = bitarray('0000000') + b = bitarray('1100110') + c = bitarray('10 10 ') + a[b] = c + self.assertEqual(a, + bitarray('1000100')) + + def test_zeros_mask(self): + for a in self.randombitarrays(): + b = a.copy() + mask = zeros(len(a)) + a[mask] = bitarray() + self.assertEqual(a, b) + + def test_ones_mask(self): + for a in self.randombitarrays(): + n = len(a) + mask = ones(n) + c = urandom_2(n) + a[mask] = c + self.assertEqual(a, c) + + def test_random_mask_set_random(self): + for a in self.randombitarrays(): + b = a.copy() + mask = urandom_2(len(a)) + other = urandom_2(mask.count()) + a[mask] = other + b[list(mask.search(1))] = other + self.assertEqual(a, b) + + def test_random_slice_mask(self): + for n in range(100): + s = self.random_slice(n, randint(1, 5)) + slicelength = len(range(n)[s]) + a = urandom_2(n) + b = a.copy() + mask = zeros(n) + mask[s] = 1 + other = urandom_2(slicelength) + a[mask] = b[s] = other + self.assertEQUAL(a, b) - def test_del_basic(self): + def test_random_mask_set_zeros(self): + for a in self.randombitarrays(): + mask = urandom_2(len(a), endian=a.endian) + b = a.copy() + self.assertRaisesMessage( + NotImplementedError, + "mask assignment to bool not implemented;\n" + "`a[mask] = 0` equivalent to `a &= ~mask`", + a.__setitem__, mask, 0) + a[mask] = zeros(mask.count()) + b &= ~mask + self.assertEqual(a, b) + + def test_random_mask_set_ones(self): + for a in self.randombitarrays(): + mask = urandom_2(len(a), endian=a.endian) + b = a.copy() + self.assertRaisesMessage( + NotImplementedError, + "mask assignment to bool not implemented;\n" + "`a[mask] = 1` equivalent to `a |= mask`", + a.__setitem__, mask, 1) + a[mask] = ones(mask.count()) + b |= mask + self.assertEqual(a, b) + + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0, 0xff]) + # 00000000 11111111 + b = bitarray(endian="big", buffer=a) + c = bitarray('00001111 00110011') + b[c] = bitarray(' 1001 01 10') + self.assertEqual(a, bytearray([0b00001001, 0b11011110])) + +class DelMaskedIndexTests(unittest.TestCase, Util): + + def test_basic(self): a = bitarray('1001001') mask = bitarray('1010111') del a[mask] self.assertEqual(a, bitarray('01')) self.assertRaises(IndexError, a.__delitem__, bitarray('101')) - def test_del_random(self): + def test_zeros_mask(self): for a in self.randombitarrays(): - n = len(a) b = a.copy() # mask has only zeros - nothing will be removed - mask = zeros(n) + mask = zeros(len(a)) del b[mask] self.assertEqual(b, a) - b = a.copy() + def test_ones_mask(self): + for a in self.randombitarrays(): # mask has only ones - everything will be removed - mask.setall(1) - del b[mask] - self.assertEqual(b, bitarray()) + mask = ones(len(a)) + del a[mask] + self.assertEqual(a, bitarray()) - b = a.copy() + def test_self_mask(self): + for a in self.randombitarrays(): + cnt0 = a.count(0) # mask is bitarray itself - all 1 items are removed - # only all the 0's remain - del b[b] - self.assertEqual(b, zeros(a.count(0))) + del a[a] + self.assertEqual(a, zeros(cnt0)) + def test_random_mask(self): + for a in self.randombitarrays(): + n = len(a) b = a.copy() - mask = urandom(n) - res = bitarray(a[i] for i in range(n) if not mask[i]) + mask = urandom_2(n) del b[mask] - self.assertEqual(b, res) + self.assertEqual(b, + bitarray(a[i] for i in range(n) if not mask[i])) # `del a[mask]` is equivalent to the in-place version of - # selecting the reverse mask `a = a[~mask]` - self.assertEqual(a[~mask], b) + # selecting the inverted mask `a = a[~mask]` + self.assertEqual(b, a[~mask]) -# --------------------------------------------------------------------------- + def test_random_slice_mask(self): + for n in range(100): + s = self.random_slice(n) + a = urandom_2(n) + b = a.copy() + mask = zeros(n) + mask[s] = 1 + del a[mask], b[s] + self.assertEQUAL(a, b) -class SequenceIndexTests(unittest.TestCase, Util): + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([5, 3]) + b = bitarray(buffer=a) + self.assertFalse(b.readonly) + self.assertRaises(BufferError, b.__delitem__, + bitarray('00001111 00110011')) + # even though we don't delete anything, raise error + self.assertRaises(BufferError, b.__delitem__, bitarray(16)) - def test_get_basic(self): +# ------------------------- Sequence index tests ---------------------------- + +class CommonSequenceIndexTests(unittest.TestCase, Util): + + def test_type_messages(self): + for item, msg in [ + (tuple([1, 2]), "multiple dimensions not supported"), + (None, "bitarray indices must be integers, slices or " + "sequences, not 'NoneType'"), + (0.12, "bitarray indices must be integers, slices or " + "sequences, not 'float'"), + ]: + a = bitarray('10111') + self.assertRaisesMessage(TypeError, msg, a.__getitem__, item) + self.assertRaisesMessage(TypeError, msg, a.__setitem__, item, 1) + self.assertRaisesMessage(TypeError, msg, a.__delitem__, item) + +class GetSequenceIndexTests(unittest.TestCase, Util): + + def test_basic(self): a = bitarray('00110101 00') self.assertEqual(a[[2, 4, -3, 9]], bitarray('1010')) self.assertEqual(a[71 * [2, 4, 7]], 71 * bitarray('101')) @@ -1507,29 +1427,37 @@ class SequenceIndexTests(unittest.TestCa self.assertRaises(IndexError, a.__getitem__, [1, 10]) self.assertRaises(IndexError, a.__getitem__, [-11]) - def test_get_types(self): + def test_types(self): a = bitarray('11001101 01') lst = [1, 3, -2] - for b in [lst, array.array('i', lst)]: + for b in lst, array.array('i', lst): self.assertEqual(a[b], bitarray('100')) lst[2] += len(a) self.assertEqual(a[bytearray(lst)], bitarray('100')) - if is_py3k: - self.assertEqual(a[bytes(lst)], bitarray('100')) + self.assertEqual(a[bytes(lst)], bitarray('100')) self.assertRaises(TypeError, a.__getitem__, [2, "B"]) self.assertRaises(TypeError, a.__getitem__, [2, 1.2]) self.assertRaises(TypeError, a.__getitem__, tuple(lst)) - def test_get_random(self): + def test_random(self): for a in self.randombitarrays(): n = len(a) lst = [randrange(n) for _ in range(n // 2)] b = a[lst] self.assertEqual(b, bitarray(a[i] for i in lst)) - self.assertEqual(b.endian(), a.endian()) + self.assertEqual(b.endian, a.endian) - def test_set_bool_basic(self): + def test_range(self): + for n in range(100): + s = self.random_slice(n) + r = range(n)[s] + a = urandom_2(n) + self.assertEQUAL(a[r], a[s]) + +class SetSequenceIndexTests(unittest.TestCase, Util): + + def test_bool_basic(self): a = zeros(10) a[[2, 3, 5, 7]] = 1 self.assertEqual(a, bitarray('00110101 00')) @@ -1545,18 +1473,27 @@ class SequenceIndexTests(unittest.TestCa self.assertRaises(TypeError, a.__setitem__, (3, -1)) self.assertRaises(TypeError, a.__setitem__, a) - def test_set_bool_random(self): + def test_bool_random(self): for a in self.randombitarrays(): n = len(a) lst = [randrange(n) for _ in range(n // 2)] b = a.copy() - for v in 0, 1: - a[lst] = v - for i in lst: - b[i] = v - self.assertEqual(a, b) + v = getrandbits(1) + a[lst] = v + for i in lst: + b[i] = v + self.assertEqual(a, b) - def test_set_bitarray_basic(self): + def test_bool_range(self): + for n in range(100): + s = self.random_slice(n) + r = range(n)[s] + a = urandom_2(n) + b = a.copy() + a[s] = b[r] = getrandbits(1) + self.assertEQUAL(a, b) + + def test_bitarray_basic(self): a = zeros(10) a[[2, 3, 5, 7]] = bitarray('1101') self.assertEqual(a, bitarray('00110001 00')) @@ -1570,11 +1507,11 @@ class SequenceIndexTests(unittest.TestCa self.assertRaisesMessage(ValueError, msg, a.__setitem__, [1, 2], bitarray('001')) - def test_set_bitarray_random(self): + def test_bitarray_random(self): for a in self.randombitarrays(): n = len(a) lst = [randrange(n) for _ in range(n // 2)] - c = urandom(len(lst)) + c = urandom_2(len(lst)) b = a.copy() a[lst] = c @@ -1582,7 +1519,17 @@ class SequenceIndexTests(unittest.TestCa b[j] = c[i] self.assertEqual(a, b) - def test_set_bitarray_random_self(self): + def test_bitarray_range(self): + for n in range(100): + s = self.random_slice(n) + r = range(n)[s] + a = urandom_2(n) + b = a.copy() + # note that len(r) is slicelength + a[s] = b[r] = urandom_2(len(r)) + self.assertEQUAL(a, b) + + def test_bitarray_random_self(self): for a in self.randombitarrays(): lst = list(range(len(a))) shuffle(lst) @@ -1594,47 +1541,79 @@ class SequenceIndexTests(unittest.TestCa b[j] = c[i] self.assertEqual(a, b) - def test_del_basic(self): + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0, 1, 2, 3]) + b = bitarray(endian="big", buffer=a) + self.assertFalse(b.readonly) + # operate on imported (writable) buffer + b[range(0, 32, 8)] = 1 + self.assertEqual(a, bytearray([0x80, 0x81, 0x82, 0x83])) + b[range(0, 10)] = bitarray("00001111 01", "little") + self.assertEqual(a, bytearray([0x0f, 0x41, 0x82, 0x83])) + +class DelSequenceIndexTests(unittest.TestCase, Util): + + def test_basic(self): a = bitarray('00110101 00') # ^ ^ ^ ^ del a[[2, 4, 7, 9]] self.assertEqual(a, bitarray('001100')) del a[[]] # delete nothing self.assertEqual(a, bitarray('001100')) + del a[[2]] + self.assertEqual(a, bitarray('00100')) a = bitarray('00110101 00') - del a[71 * [2, 4, 7, 9]] + # same as earlier, but list is not ordered and has repeated indices + del a[[7, 9, 2, 2, 4, 7]] self.assertEqual(a, bitarray('001100')) self.assertRaises(IndexError, a.__delitem__, [1, 10]) + self.assertRaises(IndexError, a.__delitem__, [10]) self.assertRaises(TypeError, a.__delitem__, (1, 3)) - def test_delitems_random(self): - for a in self.randombitarrays(): - n = len(a) - lst = [randrange(n) for _ in range(n // 2)] + def test_delete_one(self): + for a in self.randombitarrays(start=1): + b = a.copy() + i = randrange(len(a)) + del a[i], b[[i]] + self.assertEqual(a, b) + + def test_random(self): + for n in range(100): + a = urandom_2(n) + lst = [randrange(n) for _ in range(randint(0, n))] b = a.copy() - c = a.copy() del a[lst] + self.assertEqual(len(a), n - len(set(lst))) for i in sorted(set(lst), reverse=True): del b[i] self.assertEqual(a, b) - lst = list(range(n)) + def test_shuffle(self): + for a in self.randombitarrays(): + lst = list(range(len(a))) shuffle(lst) - del c[lst] - self.assertEqual(len(c), 0) + del a[lst] + self.assertEqual(len(a), 0) - def test_type_messages(self): - for item, msg in [ - (tuple([1, 2]), "multiple dimensions not supported"), - (None, "bitarray indices must be integers, slices or " - "sequences, not 'NoneType'"), - (0.12, "bitarray indices must be integers, slices or " - "sequences, not 'float'"), - ]: - a = bitarray('10111') - self.assertRaisesMessage(TypeError, msg, a.__getitem__, item) - self.assertRaisesMessage(TypeError, msg, a.__setitem__, item, 1) - self.assertRaisesMessage(TypeError, msg, a.__delitem__, item) + def test_range(self): + for n in range(100): + s = self.random_slice(n) + r = range(n)[s] + a = urandom_2(n) + b = a.copy() + del a[s], b[r] + self.assertEQUAL(a, b) + + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0, 1, 2, 3]) + b = bitarray(buffer=a) + self.assertFalse(b.readonly) + # operate on imported (writable) buffer + self.assertRaises(BufferError, b.__delitem__, range(0, 32, 8)) + # even though we don't delete anything, raise error + self.assertRaises(BufferError, b.__delitem__, range(0)) # --------------------------------------------------------------------------- @@ -1650,15 +1629,6 @@ class MiscTests(unittest.TestCase, Util) self.assertEqual(bool(bitarray('0')), True) self.assertEqual(bool(bitarray('1')), True) - def test_to01(self): - a = bitarray() - self.assertEqual(a.to01(), '') - self.assertIsInstance(a.to01(), str) - - a = bitarray('101') - self.assertEqual(a.to01(), '101') - self.assertIsInstance(a.to01(), str) - def test_iterate(self): for lst in self.randomlists(): acc = [] @@ -1669,10 +1639,11 @@ class MiscTests(unittest.TestCase, Util) def test_iter1(self): it = iter(bitarray('011')) self.assertIsType(it, 'bitarrayiterator') - self.assertBitEqual(next(it), 0) - self.assertBitEqual(next(it), 1) - self.assertBitEqual(next(it), 1) - self.assertStopIteration(it) + for res in 0, 1, 1: + item = next(it) + self.assertEqual(type(item), int) + self.assertEqual(item, res) + self.assertRaises(StopIteration, next, it) def test_iter2(self): for a in self.randombitarrays(): @@ -1704,96 +1675,44 @@ class MiscTests(unittest.TestCase, Util) for i in range(len(a)): self.assertEqual(a[i], b[i + 1234]) - def test_endianness1(self): - a = bitarray(endian='little') - a.frombytes(b'\x01') - self.assertEqual(a.to01(), '10000000') - - b = bitarray(endian='little') - b.frombytes(b'\x80') - self.assertEqual(b.to01(), '00000001') - - c = bitarray(endian='big') - c.frombytes(b'\x80') - self.assertEqual(c.to01(), '10000000') - - d = bitarray(endian='big') - d.frombytes(b'\x01') - self.assertEqual(d.to01(), '00000001') - - self.assertEqual(a, c) - self.assertEqual(b, d) - - def test_endianness2(self): - a = zeros(8, endian='little') - a[0] = True - self.assertEqual(a.tobytes(), b'\x01') - a[1] = True - self.assertEqual(a.tobytes(), b'\x03') - a.frombytes(b' ') - self.assertEqual(a.tobytes(), b'\x03 ') - self.assertEqual(a.to01(), '1100000000000100') - - def test_endianness3(self): - a = zeros(8, endian='big') - a[7] = True - self.assertEqual(a.tobytes(), b'\x01') - a[6] = True - self.assertEqual(a.tobytes(), b'\x03') - a.frombytes(b' ') - self.assertEqual(a.tobytes(), b'\x03 ') - self.assertEqual(a.to01(), '0000001100100000') - - def test_endianness4(self): - a = bitarray('00100000', endian='big') - self.assertEqual(a.tobytes(), b' ') - b = bitarray('00000100', endian='little') - self.assertEqual(b.tobytes(), b' ') - self.assertNotEqual(a, b) - @skipIf(is_pypy) def test_overflow(self): a = bitarray(1) - for i in -7, -1, 0, 1: - n = 2 ** 63 + i + for i in 0, 1: + n = (1 << 63) + i self.assertRaises(OverflowError, a.__imul__, n) self.assertRaises(OverflowError, bitarray, n) - a = bitarray(2 ** 10) - self.assertRaises(OverflowError, a.__imul__, 2 ** 53) - - if SYSINFO[0] == 8: - return + a = bitarray(1 << 10) + self.assertRaises(OverflowError, a.__imul__, 1 << 53) - a = bitarray(10 ** 6) + @skipIf(PTRSIZE != 4 or is_pypy) + def test_overflow_32bit(self): + a = bitarray(1000_000) self.assertRaises(OverflowError, a.__imul__, 17180) - for i in -7, -1, 0, 1: - self.assertRaises(OverflowError, bitarray, 2 ** 31 + i) + for i in 0, 1: + self.assertRaises(OverflowError, bitarray, (1 << 31) + i) try: - a = bitarray(2 ** 31 - 8); + a = bitarray((1 << 31) - 1); except MemoryError: return self.assertRaises(OverflowError, bitarray.append, a, True) - def test_unicode_create(self): - a = bitarray(u'') - self.assertEqual(a, bitarray()) - - a = bitarray(u'111001') - self.assertEqual(a, bitarray('111001')) - def test_unhashable(self): a = bitarray() self.assertRaises(TypeError, hash, a) self.assertRaises(TypeError, dict, [(a, 'foo')]) - @skipIf(sys.version_info[0] == 2) def test_abc(self): from collections import abc a = bitarray('001') + self.assertIsInstance(a, abc.Iterable) + self.assertIsInstance(a, abc.Sized) self.assertIsInstance(a, abc.Sequence) self.assertIsInstance(a, abc.MutableSequence) + if sys.version_info[:2] >= (3, 12): + self.assertIsInstance(a, abc.Buffer) if sys.platform != "win32": self.assertFalse(isinstance(a, abc.Hashable)) @@ -1816,17 +1735,17 @@ class PickleTests(unittest.TestCase, Uti a = bitarray(buffer=b'A') # readonly (because buffer is readonly), but not frozenbitarray self.assertTrue(a.readonly) - self.assertIsType(a, 'bitarray') + self.assertEqual(type(a), bitarray) b = pickle.loads(pickle.dumps(a)) self.assertTrue(b.readonly) - self.assertIsType(b, 'bitarray') + self.assertEqual(type(b), bitarray) def test_endian(self): for endian in 'little', 'big': a = bitarray(endian=endian) b = pickle.loads(pickle.dumps(a)) - self.assertEqual(b.endian(), endian) + self.assertEqual(b.endian, endian) def test_reduce_explicit(self): a = frozenbitarray('11001111 01001', 'little') @@ -1847,7 +1766,7 @@ class PickleTests(unittest.TestCase, Uti ( type(a), # type object a.tobytes(), # buffer - a.endian(), # endianness + a.endian, # endianness a.padbits, # number of pad bits int(a.readonly) # readonly ), @@ -1857,7 +1776,7 @@ class PickleTests(unittest.TestCase, Uti b = _bitarray_reconstructor(*res[1]) self.assertEqual(a, b) self.assertEqual(type(a), type(b)) - self.assertEqual(a.endian(), b.endian()) + self.assertEqual(a.endian, b.endian) self.assertEqual(a.readonly, b.readonly) self.check_obj(b) @@ -1873,12 +1792,12 @@ class PickleTests(unittest.TestCase, Uti def test_reconstructor_explicit(self): a = _bitarray_reconstructor(bitarray, b'', 'little', 0, 0) self.assertEqual(len(a), 0) - self.assertEqual(a.endian(), 'little') + self.assertEqual(a.endian, 'little') self.check_obj(a) a = _bitarray_reconstructor(bitarray, b'\x0f', 'big', 1, 0) self.assertEqual(a, bitarray("0000111")) - self.assertEqual(a.endian(), 'big') + self.assertEqual(a.endian, 'big') self.check_obj(a) def test_reconstructor_invalid_args(self): @@ -1908,11 +1827,11 @@ class PickleTests(unittest.TestCase, Uti self.assertRaises(TypeError, _bitarray_reconstructor, bitarray, b'\x0f', 'big', 0.0, 0) self.assertRaisesMessage( - ValueError, "invalid number of padbits: 8", + ValueError, "invalid number of pad bits: 8", _bitarray_reconstructor, bitarray, b"A", "big", 8, 0) self.assertRaisesMessage( - # the number of bytes is 0 zero, so padbits cannot be 1 - ValueError, "invalid number of padbits: 1", + # the number of bytes is 0 zero, so pad bits cannot be 1 + ValueError, "invalid number of pad bits: 1", _bitarray_reconstructor, bitarray, b"", "big", 1, 0) # argument 5 - readonly @@ -1936,22 +1855,19 @@ class PickleTests(unittest.TestCase, Uti ]): b = d['b%d' % i] self.assertEqual(b.to01(), s) - self.assertEqual(b.endian(), end) - self.assertIsType(b, 'bitarray') + self.assertEqual(b.endian, end) + self.assertEqual(type(b), bitarray) self.assertFalse(b.readonly) self.check_obj(b) f = d['f%d' % i] self.assertEqual(f.to01(), s) - self.assertEqual(f.endian(), end) - self.assertIsType(f, 'frozenbitarray') + self.assertEqual(f.endian, end) + self.assertEqual(type(f), frozenbitarray) self.assertTrue(f.readonly) self.check_obj(f) - @skipIf(sys.version_info[0] == 2) def test_load(self): - # test data file was created using bitarray 1.5.0 / Python 3.5.5 - self.check_file('test_150.pickle') # using bitarray 2.8.1 / Python 3.5.5 (_bitarray_reconstructor) self.check_file('test_281.pickle') @@ -1963,7 +1879,7 @@ class PickleTests(unittest.TestCase, Uti self.assertEQUAL(a, b) self.check_obj(b) -# --------------------------------------------------------------------------- +# ---------------------------- Richcompare tests ---------------------------- class RichCompareTests(unittest.TestCase, Util): @@ -1993,26 +1909,25 @@ class RichCompareTests(unittest.TestCase ]: a = bitarray(sa, self.random_endian()) b = bitarray(sb, self.random_endian()) - self.assertEqual(a == b, int(res[0])) - self.assertEqual(a != b, int(res[1])) - self.assertEqual(a >= b, int(res[2])) - self.assertEqual(a > b, int(res[3])) - self.assertEqual(a <= b, int(res[4])) - self.assertEqual(a < b, int(res[5])) + r = bitarray(res) + self.assertEqual(a == b, r[0]) + self.assertEqual(a != b, r[1]) + self.assertEqual(a >= b, r[2]) + self.assertEqual(a > b, r[3]) + self.assertEqual(a <= b, r[4]) + self.assertEqual(a < b, r[5]) def test_eq_ne(self): - for _ in range(10): - self.assertTrue(bitarray(0, self.random_endian()) == - bitarray(0, self.random_endian())) - self.assertFalse(bitarray(0, self.random_endian()) != - bitarray(0, self.random_endian())) + for _ in range(5): + self.assertTrue(urandom_2(0) == urandom_2(0)) + self.assertFalse(urandom_2(0) != urandom_2(0)) for n in range(1, 20): a = ones(n, self.random_endian()) b = bitarray(a, self.random_endian()) self.assertTrue(a == b) self.assertFalse(a != b) - b[n - 1] = 0 + b[-1] = 0 self.assertTrue(a != b) self.assertFalse(a == b) @@ -2042,9 +1957,9 @@ class RichCompareTests(unittest.TestCase self.check(a, b, a[i], b[i]) def test_size(self): - for _ in range(100): - a = zeros(randint(1, 20), self.random_endian()) - b = zeros(randint(1, 20), self.random_endian()) + for _ in range(10): + a = bitarray(randrange(20), self.random_endian()) + b = bitarray(randrange(20), self.random_endian()) self.check(a, b, len(a), len(b)) def test_random(self): @@ -2063,46 +1978,17 @@ class RichCompareTests(unittest.TestCase class SpecialMethodTests(unittest.TestCase, Util): - def test_all(self): - a = bitarray() - self.assertTrue(a.all()) - for s, r in ('0', False), ('1', True), ('01', False): - self.assertTrue(bitarray(s).all() is r) - - for a in self.randombitarrays(): - self.assertTrue(a.all() is all(a)) - - N = randint(1000, 2000) - a = ones(N) - self.assertTrue(a.all()) - a[N - 1] = 0 - self.assertFalse(a.all()) - - def test_any(self): - a = bitarray() - self.assertFalse(a.any()) - for s, r in ('0', False), ('1', True), ('01', True): - self.assertTrue(bitarray(s).any() is r) - - for a in self.randombitarrays(): - self.assertTrue(a.any() is any(a)) - - N = randint(1000, 2000) - a = zeros(N) - self.assertFalse(a.any()) - a[N - 1] = 1 - self.assertTrue(a.any()) - def test_repr(self): r = repr(bitarray()) self.assertEqual(r, "bitarray()") - self.assertIsInstance(r, str) + self.assertEqual(type(r), str) r = repr(bitarray('10111')) self.assertEqual(r, "bitarray('10111')") - self.assertIsInstance(r, str) + self.assertEqual(type(r), str) for a in self.randombitarrays(): + self.assertEqual(repr(a), str(a)) b = eval(repr(a)) self.assertFalse(b is a) self.assertEqual(a, b) @@ -2122,63 +2008,24 @@ class SpecialMethodTests(unittest.TestCa self.assertFalse(b is a) self.assertEQUAL(a, b) - def assertReallyEqual(self, a, b): - # assertEqual first, because it will have a good message if the - # assertion fails. - self.assertEqual(a, b) - self.assertEqual(b, a) - self.assertTrue(a == b) - self.assertTrue(b == a) - self.assertFalse(a != b) - self.assertFalse(b != a) - if not is_py3k: - self.assertEqual(0, cmp(a, b)) - self.assertEqual(0, cmp(b, a)) - - def assertReallyNotEqual(self, a, b): - # assertNotEqual first, because it will have a good message if the - # assertion fails. - self.assertNotEqual(a, b) - self.assertNotEqual(b, a) - self.assertFalse(a == b) - self.assertFalse(b == a) - self.assertTrue(a != b) - self.assertTrue(b != a) - if not is_py3k: - self.assertNotEqual(0, cmp(a, b)) - self.assertNotEqual(0, cmp(b, a)) - - def test_equality(self): - self.assertReallyEqual(bitarray(''), bitarray('')) - self.assertReallyEqual(bitarray('0'), bitarray('0')) - self.assertReallyEqual(bitarray('1'), bitarray('1')) - - def test_not_equality(self): - self.assertReallyNotEqual(bitarray(''), bitarray('1')) - self.assertReallyNotEqual(bitarray(''), bitarray('0')) - self.assertReallyNotEqual(bitarray('0'), bitarray('1')) - - def test_equality_random(self): - for a in self.randombitarrays(start=1): - b = a.copy() - self.assertReallyEqual(a, b) - n = len(a) - b.invert(n - 1) # flip last bit - self.assertReallyNotEqual(a, b) - @skipIf(is_pypy) def test_sizeof(self): a = bitarray() size = sys.getsizeof(a) self.assertEqual(size, a.__sizeof__()) - self.assertIsInstance(size, int if is_py3k else (int, long)) + self.assertEqual(type(size), int) self.assertTrue(size < 200) a = bitarray(8000) self.assertTrue(sys.getsizeof(a) > 1000) -# --------------------------------------------------------------------------- +# -------------------------- Sequence methods tests ------------------------- + +class SequenceTests(unittest.TestCase, Util): -class SequenceMethodsTests(unittest.TestCase, Util): + def test_len(self): + for n in range(100): + a = bitarray(n) + self.assertEqual(len(a), n) def test_concat(self): a = bitarray('001') @@ -2194,10 +2041,7 @@ class SequenceMethodsTests(unittest.Test self.assertEQUAL(a, bitarray('001')) self.assertRaises(TypeError, a.__add__, 42) - if is_py3k: - self.assertRaises(TypeError, a.__add__, b'1101') - else: - self.assertEqual(a + b'10', bitarray('00110')) + self.assertRaises(ValueError, a.__add__, b'1101') for a in self.randombitarrays(): aa = a.copy() @@ -2205,7 +2049,7 @@ class SequenceMethodsTests(unittest.Test bb = b.copy() c = a + b self.assertEqual(c, bitarray(a.tolist() + b.tolist())) - self.assertEqual(c.endian(), a.endian()) + self.assertEqual(c.endian, a.endian) self.check_obj(c) self.assertEQUAL(a, aa) @@ -2227,12 +2071,7 @@ class SequenceMethodsTests(unittest.Test self.assertEqual(a, bitarray('110')) self.assertRaises(TypeError, a.__iadd__, 42) - b = b'101' - if is_py3k: - self.assertRaises(TypeError, a.__iadd__, b) - else: - a += b - self.assertEqual(a, bitarray('110101')) + self.assertRaises(ValueError, a.__iadd__, b'101') for a in self.randombitarrays(): for b in self.randombitarrays(): @@ -2242,7 +2081,7 @@ class SequenceMethodsTests(unittest.Test self.assertEqual(d, a + b) self.assertTrue(c is d) self.assertEQUAL(c, d) - self.assertEqual(d.endian(), a.endian()) + self.assertEqual(d.endian, a.endian) self.check_obj(d) def test_repeat_explicit(self): @@ -2272,14 +2111,12 @@ class SequenceMethodsTests(unittest.Test def test_repeat_random(self): for a in self.randombitarrays(): b = a.copy() - for m in list(range(-3, 5)) + [randint(100, 200)]: - res = bitarray(m * a.to01(), endian=a.endian()) + for m in list(range(-3, 5)) + [randint(5, 200)]: + res = bitarray(m * a.to01(), endian=a.endian) self.assertEqual(len(res), len(a) * max(0, m)) - c = a * m - self.assertEQUAL(c, res) - c = m * a - self.assertEQUAL(c, res) + self.assertEQUAL(a * m, res) + self.assertEQUAL(m * a, res) c = a.copy() c *= m @@ -2290,21 +2127,18 @@ class SequenceMethodsTests(unittest.Test def test_contains_simple(self): a = bitarray() - self.assertFalse(False in a) - self.assertFalse(True in a) + self.assertFalse(0 in a) + self.assertFalse(1 in a) self.assertTrue(bitarray() in a) - a.append(True) - self.assertTrue(True in a) - self.assertFalse(False in a) - a = bitarray([False]) - self.assertTrue(False in a) - self.assertFalse(True in a) - a.append(True) + a.append(1) + self.assertTrue(1 in a) + self.assertFalse(0 in a) + a = bitarray([0]) + self.assertTrue(0 in a) + self.assertFalse(1 in a) + a.append(1) self.assertTrue(0 in a) self.assertTrue(1 in a) - if not is_py3k: - self.assertTrue(long(0) in a) - self.assertTrue(long(1) in a) def test_contains_errors(self): a = bitarray() @@ -2317,17 +2151,15 @@ class SequenceMethodsTests(unittest.Test self.assertRaises(TypeError, a.__contains__, 'asdf') self.assertRaises(ValueError, a.__contains__, 2) self.assertRaises(ValueError, a.__contains__, -1) - if not is_py3k: - self.assertRaises(ValueError, a.__contains__, long(2)) def test_contains_range(self): for n in range(2, 50): a = zeros(n) - self.assertTrue(False in a) - self.assertFalse(True in a) + self.assertTrue(0 in a) + self.assertFalse(1 in a) a[randrange(n)] = 1 - self.assertTrue(True in a) - self.assertTrue(False in a) + self.assertTrue(1 in a) + self.assertTrue(0 in a) a.setall(1) self.assertTrue(True in a) self.assertFalse(False in a) @@ -2340,9 +2172,10 @@ class SequenceMethodsTests(unittest.Test for s, r in [('', True), # every bitarray contains an empty one ('1', True), ('11', True), ('111', False), ('011', True), ('0001', True), ('00011', False)]: - self.assertEqual(bitarray(s) in a, r) + c = bitarray(s) in a + self.assertTrue(c is r) -# --------------------------------------------------------------------------- +# -------------------------- Number methods tests --------------------------- class NumberTests(unittest.TestCase, Util): @@ -2464,7 +2297,7 @@ class NumberTests(unittest.TestCase, Uti aa = a.copy() self.assertEQUAL(a & a, aa) self.assertEQUAL(a | a, aa) - self.assertEQUAL(a ^ a, zeros(len(aa), aa.endian())) + self.assertEQUAL(a ^ a, zeros(len(aa), aa.endian)) self.assertEQUAL(a, aa) def test_bitwise_inplace_self(self): @@ -2475,7 +2308,7 @@ class NumberTests(unittest.TestCase, Uti a |= a self.assertEQUAL(a, aa) a ^= a - self.assertEqual(a, zeros(len(aa), aa.endian())) + self.assertEqual(a, zeros(len(aa), aa.endian)) def test_invert(self): a = bitarray('11011') @@ -2496,12 +2329,12 @@ class NumberTests(unittest.TestCase, Uti @staticmethod def shift(a, n, direction): if n >= len(a): - return zeros(len(a), a.endian()) + return zeros(len(a), a.endian) if direction == 'right': - return zeros(n, a.endian()) + a[:len(a)-n] + return zeros(n, a.endian) + a[:len(a)-n] elif direction == 'left': - return a[n:] + zeros(n, a.endian()) + return a[n:] + zeros(n, a.endian) else: raise ValueError("invalid direction: %s" % direction) @@ -2512,7 +2345,7 @@ class NumberTests(unittest.TestCase, Uti self.assertRaises(TypeError, lambda: a << 1.2) self.assertRaises(TypeError, a.__lshift__, 1.2) self.assertRaises(ValueError, lambda: a << -1) - self.assertRaises(OverflowError, a.__lshift__, 2 ** 63) + self.assertRaises(OverflowError, a.__lshift__, 1 << 63) for a in self.randombitarrays(): c = a.copy() @@ -2567,7 +2400,7 @@ class NumberTests(unittest.TestCase, Uti self.assertEQUAL(b, self.shift(a, n, 'right')) def check_random(self, n, endian, n_shift, direction): - a = urandom(n, endian) + a = urandom_2(n, endian) self.assertEqual(len(a), n) b = a.copy() @@ -2600,7 +2433,7 @@ class NumberTests(unittest.TestCase, Uti # ensure shifts with len(a) (or larger) result in all zero bitarrays for a in self.randombitarrays(): c = a.copy() - z = zeros(len(a), a.endian()) + z = zeros(len(a), a.endian) n = randint(len(a), len(a) + 10) self.assertEQUAL(a << n, z) self.assertEQUAL(a >> n, z) @@ -2619,20 +2452,40 @@ class NumberTests(unittest.TestCase, Uti def test_frozenbitarray(self): a = frozenbitarray('0010011') - self.assertEqual(a << 3, bitarray('0011000')) + b = a << 3 + self.assertEqual(b, bitarray('0011000')) + self.assertEqual(type(b), frozenbitarray) self.assertRaises(TypeError, a.__ilshift__, 4) -# --------------------------------------------------------------------------- + @skipIf(is_pypy) + def test_imported(self): + _set_default_endian("big") + a = bytearray([0xf0, 0x01, 0x02, 0x0f]) + b = bitarray(buffer=a) + self.assertFalse(b.readonly) + # operate on imported (writable) buffer + b[8:24] <<= 3 + self.assertEqual(a, bytearray([0xf0, 0x08, 0x10, 0x0f])) + b[0:9] |= bitarray("0000 1100 1") + self.assertEqual(a, bytearray([0xfc, 0x88, 0x10, 0x0f])) + b[23:] ^= bitarray("1 1110 1110") + self.assertEqual(a, bytearray([0xfc, 0x88, 0x11, 0xe1])) + b[16:] &= bitarray("1111 0000 1111 0000") + self.assertEqual(a, bytearray([0xfc, 0x88, 0x10, 0xe0])) + b >>= 8 + self.assertEqual(a, bytearray([0x00, 0xfc, 0x88, 0x10])) + +# -------------------------------- .extend() ---------------------------- class ExtendTests(unittest.TestCase, Util): - def test_wrongArgs(self): + def test_wrong_args(self): a = bitarray() self.assertRaises(TypeError, a.extend) - self.assertRaises(TypeError, a.extend, None) - self.assertRaises(TypeError, a.extend, True) - self.assertRaises(TypeError, a.extend, 24) - self.assertRaises(TypeError, a.extend, 1.0) + for x in None, 1, True, 24, 1.0: + self.assertRaises(TypeError, a.extend, x) + self.assertEqual(len(a), 0) + self.check_obj(a) def test_bitarray(self): a = bitarray() @@ -2641,7 +2494,7 @@ class ExtendTests(unittest.TestCase, Uti a.extend(bitarray('110')) self.assertEqual(a, bitarray('110')) a.extend(bitarray('1110')) - self.assertEqual(a, bitarray('1101110')) + self.assertEqual(a, bitarray('110 1110')) a = bitarray('00001111', endian='little') a.extend(bitarray('00100111', endian='big')) @@ -2655,12 +2508,11 @@ class ExtendTests(unittest.TestCase, Uti c = bitarray(a) c.extend(b) self.assertEqual(c.to01(), sa + bb.to01()) - self.assertEqual(c.endian(), a.endian()) + self.assertEqual(c.endian, a.endian) self.assertEqual(len(c), len(a) + len(b)) self.check_obj(c) # ensure b hasn't changed self.assertEQUAL(b, bb) - self.check_obj(b) def test_list(self): a = bitarray() @@ -2679,22 +2531,30 @@ class ExtendTests(unittest.TestCase, Uti self.assertEqual(c.tolist(), a + b) self.check_obj(c) - def test_tuple(self): + def test_range(self): a = bitarray() - a.extend(tuple()) - self.assertEqual(a, bitarray()) - a.extend((0, 1, True, 0, False)) - self.assertEqual(a, bitarray('01100')) - self.assertRaises(ValueError, a.extend, (0, 1, 2)) - self.assertRaises(TypeError, a.extend, (0, 1, 'a')) - self.assertEqual(a, bitarray('01100')) + a.extend(range(2)) + self.assertEqual(a, bitarray('01')) + self.check_obj(a) - for a in self.randomlists(): - for b in self.randomlists(): - c = bitarray(a) - c.extend(tuple(b)) - self.assertEqual(c.tolist(), a + b) - self.check_obj(c) + def test_sequence(self): + lst = [0, 1, 0, 1, 1] + for x in [lst, tuple(lst), bytes(lst), bytearray(lst), + array.array('b', lst)]: + self.assertEqual(len(x), 5) # sequences have len, iterables not + a = bitarray() + a.extend(x) + self.assertEqual(a, bitarray("01011")) + self.check_obj(a) + + lst.append(2) # will raise ValueError + for x in [lst, tuple(lst), bytes(lst), bytearray(lst), + array.array('b', lst)]: + self.assertEqual(len(x), 6) + a = bitarray() + self.assertRaises(ValueError, a.extend, x) + self.assertEqual(len(a), 0) + self.check_obj(a) def test_generator_1(self): def gen(lst): @@ -2730,12 +2590,9 @@ class ExtendTests(unittest.TestCase, Uti raise KeyError yield i % 2 - a = bitarray() - self.assertRaises(KeyError, a.extend, gen()) - self.assertEqual(a, bitarray('0101')) - a = [] - self.assertRaises(KeyError, a.extend, gen()) - self.assertEqual(a, [0, 1, 0, 1]) + for a in bitarray(), []: + self.assertRaises(KeyError, a.extend, gen()) + self.assertEqual(list(a), [0, 1, 0, 1]) def test_iterator_1(self): a = bitarray() @@ -2781,13 +2638,14 @@ class ExtendTests(unittest.TestCase, Uti self.assertEqual(a, bitarray('0110111')) a = bitarray() - self.assertRaises(ValueError, a.extend, 1000 * '01' + '.') + self.assertRaises(ValueError, a.extend, 100 * '01' + '.') + self.assertRaises(ValueError, a.extend, 100 * '01' + '\0') self.assertEqual(a, bitarray()) for a in self.randomlists(): for b in self.randomlists(): c = bitarray(a) - c.extend(''.join(['0', '1'][x] for x in b)) + c.extend(''.join(str(x) for x in b)) self.assertEqual(c, bitarray(a + b)) self.check_obj(c) @@ -2801,32 +2659,6 @@ class ExtendTests(unittest.TestCase, Uti self.assertEqual(a, bitarray('010101 1010')) self.check_obj(a) - def test_unicode(self): - a = bitarray() - a.extend(u'') - self.assertEqual(a, bitarray()) - self.assertRaises(ValueError, a.extend, u'0011201') - # ensure no bits got added after error was raised - self.assertEqual(a, bitarray()) - self.check_obj(a) - - a = bitarray() - a.extend(u'001 011_') - self.assertEqual(a, bitarray('001011')) - self.assertRaises(UnicodeEncodeError, a.extend, u'1\u2605 0') - self.assertEqual(a, bitarray('001011')) - self.check_obj(a) - - def test_bytes(self): - a = bitarray() - b = b'10110' - if is_py3k: - self.assertRaises(TypeError, a.extend, b) - else: - a.extend(b) - self.assertEqual(a, bitarray('10110')) - self.check_obj(a) - def test_self(self): for s in '', '1', '110', '00110111': a = bitarray(s) @@ -2834,19 +2666,51 @@ class ExtendTests(unittest.TestCase, Uti self.assertEqual(a, bitarray(2 * s)) for a in self.randombitarrays(): - endian = a.endian() + endian = a.endian s = a.to01() a.extend(a) self.assertEqual(a.to01(), 2 * s) - self.assertEqual(a.endian(), endian) + self.assertEqual(a.endian, endian) self.assertEqual(len(a), 2 * len(s)) self.check_obj(a) -# --------------------------------------------------------------------------- +# ------------------------ Tests for bitarray methods ----------------------- + +class AllAnyTests(unittest.TestCase, Util): + + def test_all(self): + a = bitarray() + self.assertTrue(a.all()) + for s, r in ('0', False), ('1', True), ('01', False): + self.assertTrue(bitarray(s).all() is r) + + for a in self.randombitarrays(): + self.assertTrue(a.all() is all(a)) -class MethodTests(unittest.TestCase, Util): + N = randint(1000, 2000) + a = ones(N) + self.assertTrue(a.all()) + a[N - 1] = 0 + self.assertFalse(a.all()) - def test_append_simple(self): + def test_any(self): + a = bitarray() + self.assertFalse(a.any()) + for s, r in ('0', False), ('1', True), ('01', True): + self.assertTrue(bitarray(s).any() is r) + + for a in self.randombitarrays(): + self.assertTrue(a.any() is any(a)) + + N = randint(1000, 2000) + a = zeros(N) + self.assertFalse(a.any()) + a[N - 1] = 1 + self.assertTrue(a.any()) + +class AppendTests(unittest.TestCase, Util): + + def test_simple(self): a = bitarray() a.append(True) a.append(False) @@ -2855,81 +2719,91 @@ class MethodTests(unittest.TestCase, Uti a.append(0) a.append(1) self.assertEQUAL(a, bitarray('10001')) + self.check_obj(a) + + def test_wrong_args(self): + a = bitarray("10001") self.assertRaises(ValueError, a.append, 2) self.assertRaises(TypeError, a.append, None) self.assertRaises(TypeError, a.append, '') self.assertEQUAL(a, bitarray('10001')) self.check_obj(a) - def test_append_random(self): - for a in self.randombitarrays(): - aa = a.tolist() - a.append(1) - self.assertEQUAL(a, bitarray(aa + [1], endian=a.endian())) - a.append(0) - self.assertEQUAL(a, bitarray(aa + [1, 0], endian=a.endian())) - self.check_obj(a) + def test_random(self): + a = urandom_2(1000) + b = bitarray(endian=a.endian) + for i in range(len(a)): + b.append(a[i]) + self.assertEQUAL(b, a[:i+1]) + self.check_obj(b) - def test_insert(self): +class InsertTests(unittest.TestCase, Util): + + def test_basic(self): + a = bitarray('00111') + a.insert(0, 1) + self.assertEqual(a, bitarray('1 00111')) + a.insert(0, 0) + self.assertEqual(a, bitarray('01 00111')) + a.insert(2, 1) + self.assertEqual(a, bitarray('011 00111')) + + def test_errors(self): a = bitarray('111100') - a.insert(3, False) - self.assertEqual(a, bitarray('1110100')) self.assertRaises(ValueError, a.insert, 0, 2) self.assertRaises(TypeError, a.insert, 0, None) self.assertRaises(TypeError, a.insert) self.assertRaises(TypeError, a.insert, None) - self.assertEqual(a, bitarray('1110100')) + self.assertEqual(a, bitarray('111100')) self.check_obj(a) - def test_insert_random(self): + def test_random(self): for a in self.randombitarrays(): aa = a.tolist() for _ in range(20): item = getrandbits(1) - pos = randint(-len(a) - 2, len(a) + 2) + pos = randint(-len(a) - 5, len(a) + 5) a.insert(pos, item) aa.insert(pos, item) self.assertEqual(a.tolist(), aa) self.check_obj(a) - def test_fill_simple(self): - for endian in 'little', 'big': - a = bitarray(endian=endian) - self.assertEqual(a.fill(), 0) - self.assertEqual(len(a), 0) +class FillTests(unittest.TestCase, Util): - a = bitarray('101', endian) - self.assertEqual(a.fill(), 5) - self.assertEqual(a, bitarray('10100000')) - self.assertEqual(a.fill(), 0) - self.assertEqual(a, bitarray('10100000')) - self.check_obj(a) + def test_simple(self): + a = bitarray(endian=self.random_endian()) + self.assertEqual(a.fill(), 0) + self.assertEqual(len(a), 0) + + a = bitarray('101', self.random_endian()) + self.assertEqual(a.fill(), 5) + self.assertEqual(a, bitarray('10100000')) + self.assertEqual(a.fill(), 0) + self.assertEqual(a, bitarray('10100000')) + self.check_obj(a) - def test_fill_exported(self): + def test_exported(self): a = bitarray('11101') b = bitarray(buffer=a) v = memoryview(a) self.assertEqual(a.fill(), 3) self.assertEqual(a, b) - if is_py3k: - self.assertEqual(v.nbytes, 1) + self.assertEqual(v.nbytes, 1) - def test_fill_random(self): + def test_random(self): for a in self.randombitarrays(): b = a.copy() res = b.fill() self.assertTrue(0 <= res < 8) - self.assertEqual(b.endian(), a.endian()) + self.assertTrue(b.padbits == 0) + self.assertEqual(len(b) % 8, 0) + self.assertEqual(b, a + zeros(res)) + self.assertEqual(b.endian, a.endian) self.check_obj(b) - if len(a) % 8 == 0: - self.assertEqual(b, a) - else: - self.assertEqual(len(b) % 8, 0) - self.assertNotEqual(b, a) - self.assertEqual(b[:len(a)], a) - self.assertEqual(b[len(a):], zeros(len(b) - len(a))) - def test_invert_simple(self): +class InvertTests(unittest.TestCase, Util): + + def test_simple(self): a = bitarray() a.invert() self.assertEQUAL(a, bitarray()) @@ -2938,27 +2812,75 @@ class MethodTests(unittest.TestCase, Uti a = bitarray('11011') a.invert() self.assertEQUAL(a, bitarray('00100')) - a.invert(2) - self.assertEQUAL(a, bitarray('00000')) - a.invert(-1) - self.assertEQUAL(a, bitarray('00001')) + for i, res in [( 0, '10100'), + ( 4, '10101'), + ( 2, '10001'), + (-1, '10000'), + (-5, '00000')]: + a.invert(i) + self.assertEqual(a.to01(), res) - def test_invert_errors(self): + def test_errors(self): a = bitarray(5) self.assertRaises(IndexError, a.invert, 5) self.assertRaises(IndexError, a.invert, -6) self.assertRaises(TypeError, a.invert, "A") self.assertRaises(TypeError, a.invert, 0, 1) + self.assertFalse(a.any()) + self.check_obj(a) - def test_invert_random(self): + def test_random(self): for a in self.randombitarrays(start=1): + n = len(a) b = a.copy() - i = randrange(len(a)) - b.invert(i) + i = randint(-n, n - 1) a[i] = not a[i] - self.assertEQUAL(a, b) + b.invert(i) + self.assertEQUAL(b, a) + self.check_obj(b) + + def test_all(self): + for a in self.randombitarrays(): + b = a.copy() + a.invert() + self.assertEqual(a, bitarray([not v for v in b])) + self.assertEqual(a.endian, b.endian) + self.check_obj(a) + self.assertEQUAL(b, ~a) + + def test_span(self): + for a in self.randombitarrays(): + n = len(a) + b = a.copy() + for _ in range(10): + i = randint(0, n) + j = randint(i, n) + a.invert(slice(i, j)) + b[i:j] = ~b[i:j] + self.assertEqual(a, b) + + def test_random_slice(self): + for a in self.randombitarrays(): + n = len(a) + b = a.copy() + for _ in range(10): + s = self.random_slice(n) + a.invert(s) + b[s] = ~b[s] + self.assertEQUAL(a, b) + + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0, 1, 2, 3, 32, 255]) + b = bitarray(buffer=a) + # operate on imported (writable) buffer + self.assertFalse(b.readonly) + b.invert() + self.assertEqual(a, bytearray([255, 254, 253, 252, 223, 0])) - def test_sort_simple(self): +class SortTests(unittest.TestCase, Util): + + def test_simple(self): a = bitarray('1101000') a.sort() self.assertEqual(a, bitarray('0000111')) @@ -2976,7 +2898,7 @@ class MethodTests(unittest.TestCase, Uti self.assertRaises(TypeError, a.sort, 'A') - def test_sort_random(self): + def test_random(self): for rev in False, True, 0, 1, 7, -1, -7, None: for a in self.randombitarrays(): lst = a.tolist() @@ -2989,44 +2911,206 @@ class MethodTests(unittest.TestCase, Uti self.assertEqual(a, bitarray(lst)) self.check_obj(a) - def test_reverse_explicit(self): + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0x6f, 0xa5]) + b = bitarray(endian="big", buffer=a) + self.assertFalse(b.readonly) + # operate on imported (writable) buffer + b.sort() + self.assertEqual(b.count(), 10) + self.assertEqual(a, bytearray([0x03, 0xff])) + +# ----------------------- .pack() .unpack() --------------------------- + +class PackTests(unittest.TestCase, Util): + + def test_pack_simple(self): + for endian in 'little', 'big': + _set_default_endian(endian) + a = bitarray() + a.pack(bytes()) + self.assertEQUAL(a, bitarray()) + a.pack(b'\x00') + self.assertEQUAL(a, bitarray('0')) + a.pack(b'\xff') + self.assertEQUAL(a, bitarray('01')) + a.pack(b'\x01\x00\x7a') + self.assertEQUAL(a, bitarray('01101')) + a.pack(bytearray([0x01, 0x00, 0xff, 0xa7])) + self.assertEQUAL(a, bitarray('01101 1011')) + self.check_obj(a) + + def test_pack_types(self): + a = bitarray() + a.pack(b'\0\x01') # bytes + self.assertEqual(a, bitarray('01')) + a.pack(bytearray([0, 2])) # bytearray + self.assertEqual(a, bitarray('01 01')) + a.pack(memoryview(b'\x02\0')) # memoryview + self.assertEqual(a, bitarray('01 01 10')) + + a.pack(array.array('B', [0, 255, 192])) + self.assertEqual(a, bitarray('01 01 10 011')) + self.check_obj(a) + + def test_pack_bitarray(self): + b = bitarray("00000000 00000001 10000000 11111111 00000000") + a = bitarray() + a.pack(bitarray(b)) + self.assertEqual(a, bitarray('01110')) + self.check_obj(a) + + def test_pack_self(self): + a = bitarray() + self.assertRaisesMessage( + BufferError, + "cannot resize bitarray that is exporting buffers", + a.pack, a) + + def test_pack_allbytes(self): + a = bitarray() + a.pack(bytearray(range(256))) + self.assertEqual(a.to01(), '0' + 255 * '1') + self.check_obj(a) + + def test_pack_errors(self): + a = bitarray() + self.assertRaises(TypeError, a.pack, 0) + self.assertRaises(TypeError, a.pack, '1') + self.assertRaises(TypeError, a.pack, [1, 3]) + + def test_unpack_simple(self): + a = bitarray('01') + self.assertEqual(type(a.unpack()), bytes) + self.assertEqual(a.unpack(), b'\x00\x01') + self.assertEqual(a.unpack(b'A'), b'A\x01') + self.assertEqual(a.unpack(b'0', b'1'), b'01') + self.assertEqual(a.unpack(one=b'\xff'), b'\x00\xff') + self.assertEqual(a.unpack(zero=b'A'), b'A\x01') + self.assertEqual(a.unpack(one=b't', zero=b'f'), b'ft') + + def test_unpack_random(self): + for a in self.randombitarrays(): + self.assertEqual(a.unpack(b'0', b'1'), a.to01().encode()) + # round trip + b = bitarray() + b.pack(a.unpack()) + self.assertEqual(b, a) + # round trip with invert + b = bitarray() + b.pack(a.unpack(b'\x01', b'\x00')) + b.invert() + self.assertEqual(b, a) + # use .extend() to pack + b = bitarray() + b.extend(a.unpack()) + self.assertEqual(b, a) + + def test_unpack_errors(self): + a = bitarray('01') + self.assertRaises(TypeError, a.unpack, b'') + self.assertRaises(TypeError, a.unpack, b'0', b'') + self.assertRaises(TypeError, a.unpack, b'a', zero=b'b') + self.assertRaises(TypeError, a.unpack, foo=b'b') + self.assertRaises(TypeError, a.unpack, one=b'aa', zero=b'b') + self.assertRaises(TypeError, a.unpack, '0') + self.assertRaises(TypeError, a.unpack, one='a') + self.assertRaises(TypeError, a.unpack, b'0', '1') + +class PopTests(unittest.TestCase, Util): + + def test_basic(self): + a = bitarray('01') + self.assertRaisesMessage(IndexError, "pop index out of range", + a.pop, 2) + self.assertEqual(a.pop(), True) + self.assertEqual(a.pop(), False) + self.assertEqual(a, bitarray()) + # pop from empty bitarray + self.assertRaisesMessage(IndexError, "pop from empty bitarray", a.pop) + + def test_simple(self): + for x, n, r, y in [('1', 0, 1, ''), + ('0', -1, 0, ''), + ('0011100', 3, 1, '001100')]: + a = bitarray(x) + self.assertTrue(a.pop(n) is r) + self.assertEqual(a, bitarray(y)) + self.check_obj(a) + + def test_reverse(self): + for a in self.randombitarrays(): + c = a.copy() + b = bitarray() + while a: + b.append(a.pop()) + self.assertEqual(a, bitarray()) + b.reverse() + self.assertEqual(b, c) + + def test_random_1(self): + for a in self.randombitarrays(): + self.assertRaises(IndexError, a.pop, len(a)) + self.assertRaises(IndexError, a.pop, -len(a) - 1) + if len(a) == 0: + continue + aa = a.tolist() + enda = a.endian + self.assertEqual(a.pop(), aa[-1]) + self.check_obj(a) + self.assertEqual(a.endian, enda) + + def test_random_2(self): + for a in self.randombitarrays(start=1): + n = randrange(-len(a), len(a)) + aa = a.tolist() + x = a.pop(n) + self.assertEqual(x, aa[n]) + self.assertEqual(type(x), int) + y = aa.pop(n) + self.assertEqual(a, bitarray(aa)) + self.assertEqual(x, y) + self.check_obj(a) + +class ReverseTests(unittest.TestCase, Util): + + def test_explicit(self): for x, y in [('', ''), ('1', '1'), ('10', '01'), ('001', '100'), ('1110', '0111'), ('11100', '00111'), ('011000', '000110'), ('1101100', '0011011'), ('11110000', '00001111'), - ('11111000011', '11000011111'), - ('11011111 00100000 000111', - '111000 00000100 11111011')]: + ('11111000011', '11000011111')]: a = bitarray(x) a.reverse() self.assertEQUAL(a, bitarray(y)) self.check_obj(a) - self.assertRaises(TypeError, bitarray().reverse, 42) + def test_argument(self): + a = bitarray(3) + self.assertRaises(TypeError, a.reverse, 42) - def test_reverse_random(self): + def test_random(self): for a in self.randombitarrays(): b = a.copy() a.reverse() - self.assertEqual(a.tolist(), b.tolist()[::-1]) - self.assertEQUAL(a, bitarray(reversed(b), endian=a.endian())) + self.assertEqual(a.to01(), b.to01()[::-1]) + self.assertEQUAL(a, bitarray(reversed(b), endian=a.endian)) self.assertEQUAL(a, b[::-1]) self.check_obj(a) - def test_tolist(self): - a = bitarray() - self.assertEqual(a.tolist(), []) - - a = bitarray('110') - lst = a.tolist() - self.assertIsInstance(lst, list) - self.assertEqual(repr(lst), '[1, 1, 0]') + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0, 1, 2, 3, 255]) + b = bitarray(buffer=a) + # reversing an imported (writable) buffer + self.assertFalse(b.readonly) + b.reverse() + self.assertEqual(a, bytearray([255, 192, 64, 128, 0])) - for lst in self.randomlists(): - a = bitarray(lst) - self.assertEqual(a.tolist(), lst) +class RemoveTests(unittest.TestCase, Util): - def test_remove(self): + def test_explicit(self): a = bitarray('1010110') for val, res in [(False, '110110'), (True, '10110'), (1, '0110'), (1, '010'), (0, '10'), @@ -3035,13 +3119,14 @@ class MethodTests(unittest.TestCase, Uti self.assertEQUAL(a, bitarray(res)) self.check_obj(a) + def test_errors(self): a = bitarray('0010011') a.remove(1) self.assertEQUAL(a, bitarray('000011')) self.assertRaises(TypeError, a.remove, 'A') self.assertRaises(ValueError, a.remove, 21) + self.assertEQUAL(a, bitarray('000011')) - def test_remove_errors(self): a = bitarray() for i in (True, False, 1, 0): self.assertRaises(ValueError, a.remove, i) @@ -3051,54 +3136,21 @@ class MethodTests(unittest.TestCase, Uti a.setall(1) self.assertRaises(ValueError, a.remove, 0) - def test_pop_simple(self): - for x, n, r, y in [('1', 0, 1, ''), - ('0', -1, 0, ''), - ('0011100', 3, 1, '001100')]: - a = bitarray(x) - self.assertTrue(a.pop(n) is r) - self.assertEqual(a, bitarray(y)) - self.check_obj(a) - - a = bitarray('01') - self.assertEqual(a.pop(), True) - self.assertEqual(a.pop(), False) - # pop from empty bitarray - self.assertRaises(IndexError, a.pop) - - def test_pop_random_1(self): + def test_random(self): for a in self.randombitarrays(): - self.assertRaises(IndexError, a.pop, len(a)) - self.assertRaises(IndexError, a.pop, -len(a) - 1) - if len(a) == 0: + b = a.tolist() + v = getrandbits(1) + if v not in a: continue - aa = a.tolist() - enda = a.endian() - self.assertEqual(a.pop(), aa[-1]) - self.check_obj(a) - self.assertEqual(a.endian(), enda) - - def test_pop_random_2(self): - for a in self.randombitarrays(start=1): - n = randrange(-len(a), len(a)) - aa = a.tolist() - x = a.pop(n) - self.assertBitEqual(x, aa[n]) - y = aa.pop(n) - self.assertEqual(a, bitarray(aa)) - self.assertBitEqual(x, y) + a.remove(v) + b.remove(v) + self.assertEqual(a.tolist(), b) self.check_obj(a) - def test_clear(self): - for a in self.randombitarrays(): - endian = a.endian() - a.clear() - self.assertEqual(len(a), 0) - self.assertEqual(a.endian(), endian) - self.check_obj(a) +class SetAllTests(unittest.TestCase, Util): - def test_setall(self): - a = urandom(5) + def test_explicit(self): + a = urandom_2(5) a.setall(True) self.assertRaises(ValueError, a.setall, -1) self.assertRaises(TypeError, a.setall, None) @@ -3107,23 +3159,86 @@ class MethodTests(unittest.TestCase, Uti self.assertEqual(a.to01(), '00000') self.check_obj(a) - def test_setall_empty(self): + def test_empty(self): a = bitarray() for v in 0, 1: a.setall(v) self.assertEqual(len(a), 0) self.check_obj(a) - def test_setall_random(self): + def test_random(self): for a in self.randombitarrays(): - end = a.endian() + endian = a.endian val = getrandbits(1) a.setall(val) self.assertEqual(a.to01(), len(a) * str(val)) - self.assertEqual(a.endian(), end) + self.assertEqual(a.endian, endian) self.check_obj(a) -# --------------------------------------------------------------------------- + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0, 1, 2, 3]) + b = bitarray(buffer=a) + self.assertFalse(b.readonly) + # operate on imported (writable) buffer + b.setall(1) + self.assertEqual(a, bytearray([0xff, 0xff, 0xff, 0xff])) + +class To01Tests(unittest.TestCase, Util): + + def test_no_grouping(self): + a = bitarray() + self.assertEqual(a.to01(1), "") + + a = bitarray("100011110") + for s in [a.to01(), a.to01(0), a.to01(0, "X"), a.to01(1, ""), + a.to01(group=0), a.to01(sep="X"), a.to01(group=2, sep="")]: + self.assertEqual(type(s), str) + self.assertEqual(len(s), len(a)) + self.assertEqual(s, "100011110") + + def test_examples(self): + a = bitarray("0000 1111 0011 0101") + self.assertEqual(a.to01(1, "-"), "0-0-0-0-1-1-1-1-0-0-1-1-0-1-0-1") + self.assertEqual(a.to01(2, sep='+'), "00+00+11+11+00+11+01+01") + self.assertEqual(a.to01(3), "000 011 110 011 010 1") + self.assertEqual(a.to01(group=4, sep="_"), "0000_1111_0011_0101") + self.assertEqual(a.to01(group=5, sep='.'), "00001.11100.11010.1") + self.assertEqual(a.to01(group=6), "000011 110011 0101") + self.assertEqual(a.to01(7), "0000111 1001101 01") + self.assertEqual(a.to01(8, ", "), "00001111, 00110101") + self.assertEqual(a.to01(9, "ABC"), "000011110ABC0110101") + + def test_wrong_args(self): + a = bitarray("1101100") + self.assertRaises(TypeError, a.to01, None) + self.assertRaises(ValueError, a.to01, -1) + self.assertRaises(TypeError, a.to01, foo=4) + self.assertRaises(TypeError, a.to01, 2, None) + self.assertRaises(TypeError, a.to01, 4, b"_") + + def test_sep(self): + for a in self.randombitarrays(): + sep = "".join(chr(randint(32, 126)) + for _ in range(randrange(10))) + self.assertEqual(a.to01(1, sep), sep.join(str(v) for v in a)) + + a = bitarray("11100111") + # use unicode character black star as separator + s = a.to01(3, "\u2605") + self.assertEqual(s, "111\u2605001\u260511") + + def test_random(self): + for a in self.randombitarrays(): + n = len(a) + group = randrange(10) + nsep = randrange(6) + s = a.to01(group, nsep * " ") + self.assertEqual(a, bitarray(s)) + nspace = s.count(" ") + self.assertEqual(len(s), n + nspace) + self.assertEqual(nspace, + nsep * ((n - 1) // group) if group and n else 0) class ByteReverseTests(unittest.TestCase, Util): @@ -3161,8 +3276,7 @@ class ByteReverseTests(unittest.TestCase def test_byte(self): for i in range(256): - a = bitarray() - a.frombytes(bytearray([i])) + a = bitarray(bytearray([i])) self.assertEqual(len(a), 8) b = a.copy() b.bytereverse() @@ -3181,14 +3295,13 @@ class ByteReverseTests(unittest.TestCase self.assertEQUAL(a, b) def test_random(self): - t = bitarray(endian=self.random_endian()) - t.frombytes(bytearray(range(256))) + t = bitarray(bytearray(range(256)), self.random_endian()) t.bytereverse() table = t.tobytes() # translation table self.assertEqual(table[:9], b'\x00\x80\x40\xc0\x20\xa0\x60\xe0\x10') for n in range(100): - a = urandom(8 * n, self.random_endian()) + a = urandom_2(8 * n) i = randint(0, n) # start j = randint(0, n) # stop b = a.copy() @@ -3199,13 +3312,58 @@ class ByteReverseTests(unittest.TestCase def test_endian(self): for n in range(20): - a = urandom(8 * n, self.random_endian()) + a = urandom_2(8 * n) b = a.copy() a.bytereverse() - a = bitarray(a, self.opposite_endian(a.endian())) + a = bitarray(a, self.opposite_endian(a.endian)) self.assertEqual(a.tobytes(), b.tobytes()) -# --------------------------------------------------------------------------- + @skipIf(is_pypy) + def test_imported(self): + a = bytearray([0, 1, 2, 3, 255]) + b = bitarray(buffer=a) + # operate on imported (writable) buffer + self.assertFalse(b.readonly) + b.bytereverse() + self.assertEqual(a, bytearray([0, 128, 64, 192, 255])) + +class ToListTests(unittest.TestCase, Util): + + def test_empty(self): + a = bitarray() + self.assertEqual(a.tolist(), []) + + def test_simple(self): + a = bitarray('110') + lst = a.tolist() + self.assertEqual(type(lst), list) + self.assertEqual(lst, [1, 1, 0]) + for item in lst: + self.assertEqual(type(item), int) + + def test_random(self): + for a in self.randombitarrays(): + res = a.tolist() + self.assertEqual(res, list(a)) + self.assertEqual(res, [int(v) for v in a.to01()]) + +class ClearTests(unittest.TestCase, Util): + + def test_simple(self): + a = bitarray("1110000001001000011111") + a.clear() + self.assertEqual(len(a), 0) + + def test_random(self): + for a in self.randombitarrays(): + endian = a.endian + a.clear() + self.assertFalse(a) + self.assertEqual(len(a), 0) + self.assertEqual(a.endian, endian) + self.check_obj(a) + +# -------------------------------- .count() --------------------------------- class CountTests(unittest.TestCase, Util): @@ -3248,9 +3406,9 @@ class CountTests(unittest.TestCase, Util def test_random_sub(self): for _ in range(1000): n = randrange(100) - a = urandom(n) + a = urandom_2(n) s = a.to01() - b = urandom(randrange(8)) + b = urandom_2(randrange(8)) t = b.to01() i = randint(-n - 10, n + 10) j = randint(-n - 10, n + 10) @@ -3258,14 +3416,13 @@ class CountTests(unittest.TestCase, Util def test_byte(self): for i in range(256): - a = bitarray() - a.frombytes(bytearray([i])) + a = bitarray(bytearray([i])) self.assertEqual(len(a), 8) self.assertEqual(a.count(), bin(i)[2:].count('1')) def test_whole_range(self): for n in range(500): - a = urandom(n, self.random_endian()) + a = urandom_2(n) s = a.to01() for v in 0, 1: ref = s.count(str(v)) @@ -3273,40 +3430,27 @@ class CountTests(unittest.TestCase, Util self.assertEqual(a.count(v, n, -n - 1, -1), ref) def test_sparse(self): - N = 65536 - a = zeros(N) - indices = set(randrange(N) for _ in range(256)) + n = 65536 + a = bitarray(n) + indices = set(randrange(n) for _ in range(256)) a[list(indices)] = 1 self.assertEqual(a.count(1), len(indices)) - self.assertEqual(a.count(0), N - len(indices)) + self.assertEqual(a.count(0), n - len(indices)) for _ in range(100): - i = randrange(N) - j = randrange(i, N) + i = randrange(n) + j = randrange(i, n) cnt = sum(1 for k in indices if i <= k < j) self.assertEqual(a.count(1, i, j), cnt) self.assertEqual(a.count(0, i, j), j - i - cnt) - def test_zeros(self): - N = 30 - a = zeros(N, self.random_endian()) - for _ in range(10): - i = randrange(N) - j = randrange(i, N) - self.assertEqual(a.count(0, i, j), j - i) - - for step in range(-N - 3, N + 3): - if step == 0: - continue - self.assertEqual(a.count(0, i, i, step), 0) - - def test_range(self): - N = 300 - a = urandom(N, self.random_endian()) + def test_step1(self): + n = 300 + a = urandom_2(n) s = a.to01() for _ in range(1000): - i = randrange(N) - j = randrange(i, N) + i = randrange(n) + j = randrange(i, n) t = s[i:j] c0 = t.count('0') @@ -3320,27 +3464,6 @@ class CountTests(unittest.TestCase, Util self.assertEqual(b.count(0), c0) self.assertEqual(b.count(1), c1) - def test_slicelength(self): - for N in range(100): - step = randint(-N - 1, N) - if step == 0: - continue - - a = zeros(N, self.random_endian()) - i = randint(-N - 1, N) - j = randint(-N - 1, N) - slicelength = self.calc_slicelength(slice(i, j, step), N) - self.assertEqual(len(a[i:j:step]), slicelength) - - self.assertEqual(a.count(0, i, j, step), slicelength) - self.assertEqual(a.count(1, i, j, step), 0) - a[i:j:step] = 1 - self.assertEqual(a.count(0), N - slicelength) - self.assertEqual(a.count(1), slicelength) - del a[i:j:step] - self.assertEqual(len(a), N - slicelength) - self.assertFalse(a.any()) - def test_explicit(self): a = bitarray('01001100 01110011 01') self.assertEqual(a.count(), 9) @@ -3355,27 +3478,23 @@ class CountTests(unittest.TestCase, Util self.assertEqual(a.count(1, 1, -1), 8) self.assertEqual(a.count(1, 17, 14), 0) - def test_random(self): - for _ in range(1000): - n = randrange(200) - a = urandom(n, self.random_endian()) + def test_random_slice(self): + for n in range(500): + a = urandom_2(n) v = randrange(2) - i = randint(-n - 3, n + 3) - j = randint(-n - 3, n + 3) - step = randint(-n - 3, n + 3) - if step == 0: - continue - self.assertEqual(a.count(v, i, j, step), a[i:j:step].count(v)) + s = self.random_slice(n) + self.assertEqual(a.count(v, s.start, s.stop, s.step), + a[s].count(v)) def test_offest_buffer(self): # this tests if words are aligned in popcnt_words() N = 1 << 16 for i in range(20): - a = urandom(N, 'little') + a = urandom_2(N, 'little') b = bitarray(buffer=memoryview(a)[i:], endian='little') self.assertEqual(b.count(), a.count(1, 8 * i)) -# --------------------------------------------------------------------------- +# -------------------------- .find() and .index() --------------------------- class IndexTests(unittest.TestCase, Util): @@ -3433,16 +3552,17 @@ class IndexTests(unittest.TestCase, Util return -1 s = slice(start, stop, 1) start, stop, stride = s.indices(n) - stop += 1 - i = stop - 1 if right else start - return i if start <= i < stop else -1 + if start > stop: + return -1 + return stop if right else start def test_find_empty(self): # test staticmethod .find_empty() against Python builtins for x in bytearray([0]), b"\0", "A": empty = 0 * x # empty sequence self.assertEqual(len(empty), 0) - for n in range(5): + for _ in range(50): + n = randint(0, 5) z = n * x # sequence of length n self.assertEqual(len(z), n) self.assertTrue(type(x) == type(empty) == type(z)) @@ -3450,35 +3570,36 @@ class IndexTests(unittest.TestCase, Util self.assertEqual(z.find(empty), self.find_empty(n)) self.assertEqual(z.rfind(empty), self.find_empty(n, right=1)) - for start in range(-5, 5): - self.assertEqual(z.find(empty, start), - self.find_empty(n, start)) - self.assertEqual(z.rfind(empty, start), - self.find_empty(n, start, right=1)) - - for stop in range(-5, 5): - self.assertEqual(z.find(empty, start, stop), - self.find_empty(n, start, stop)) - self.assertEqual(z.rfind(empty, start, stop), - self.find_empty(n, start, stop, 1)) + start = randint(-5, 5) + self.assertEqual(z.find(empty, start), + self.find_empty(n, start)) + self.assertEqual(z.rfind(empty, start), + self.find_empty(n, start, right=1)) + + stop = randint(-5, 5) + self.assertEqual(z.find(empty, start, stop), + self.find_empty(n, start, stop)) + self.assertEqual(z.rfind(empty, start, stop), + self.find_empty(n, start, stop, 1)) def test_empty(self): - # now that we have the tested staticmethod .find_empty(), we use it - # to test .find() with an empty bitarray + # now that we have the established .find_empty(), we use it to + # test .find() with an empty bitarray empty = bitarray() - for n in range(5): + for _ in range(50): + n = randint(0, 5) z = bitarray(n) - for r in 0, 1: - self.assertEqual(z.find(empty, right=r), - self.find_empty(n, right=r)) - - for start in range(-5, 5): - self.assertEqual(z.find(empty, start, right=r), - self.find_empty(n, start, right=r)) - - for stop in range(-5, 5): - self.assertEqual(z.find(empty, start, stop, r), - self.find_empty(n, start, stop, r)) + right = getrandbits(1) + self.assertEqual(z.find(empty, right=right), + self.find_empty(n, right=right)) + + start = randint(-5, 5) + self.assertEqual(z.find(empty, start, right=right), + self.find_empty(n, start, right=right)) + + stop = randint(-5, 5) + self.assertEqual(z.find(empty, start, stop, right), + self.find_empty(n, start, stop, right)) def test_range_explicit(self): n = 150 @@ -3515,18 +3636,18 @@ class IndexTests(unittest.TestCase, Util self.assertEqual(a.find(1, start, stop, 0), plst2[0]) self.assertEqual(a.find(1, start, stop, 1), plst2[-1]) else: - for right in 0, 1: - self.assertEqual(a.find(1, start, stop, right), -1) + right = getrandbits(1) + self.assertEqual(a.find(1, start, stop, right), -1) def test_random_sub(self): # test finding sub_bitarray - for _ in range(500): + for _ in range(200): n = randrange(1, 100) - a = urandom(n, self.random_endian()) + a = urandom_2(n) s = a.to01() self.assertEqual(a.find(a), 0) n = len(a) - b = bitarray(randrange(0, 10), self.random_endian()) + b = bitarray(randrange(10), self.random_endian()) t = b.to01() self.assertEqual(a.find(b), s.find(t)) @@ -3540,52 +3661,53 @@ class IndexTests(unittest.TestCase, Util self.assertEqual(a.find(b, i, j, 1), ref_r) if len(b) == 1: # test finding int - self.assertEqual(a.find(b[0], i, j, 0), ref_l) - self.assertEqual(a.find(b[0], i, j, 1), ref_r) + v = b[0] + self.assertTrue(v in range(2)) + self.assertEqual(a.find(v, i, j, 0), ref_l) + self.assertEqual(a.find(v, i, j, 1), ref_r) -# --------------------------------------------------------------------------- +# ----------------------------- .search() ----------------------------------- class SearchTests(unittest.TestCase, Util): + def test_no_itersearch(self): + a = bitarray() + # removed in bitarray 3.0 + self.assertRaises(AttributeError, a.__getattribute__, 'itersearch') + def test_simple(self): a = bitarray() for s in 0, 1, False, True, bitarray('0'), bitarray('1'): - self.assertEqual(a.search(s), []) + self.assertEqual(list(a.search(s)), []) a = bitarray('00100') for s in 1, True, bitarray('1'), bitarray('10'): - self.assertEqual(a.search(s), [2]) + self.assertEqual(list(a.search(s)), [2]) a = 100 * bitarray('1') - self.assertEqual(a.search(0), []) - self.assertEqual(a.search(1), list(range(100))) + self.assertEqual(list(a.search(0)), []) + self.assertEqual(list(a.search(1)), list(range(100))) - a = bitarray('10010101110011111001011') - for limit in range(10): - self.assertEqual(a.search(bitarray('011'), limit), - [6, 11, 20][:limit]) - - self.assertRaises(ValueError, a.search, bitarray()) self.assertRaises(TypeError, a.search, '010') - def test_itersearch_next(self): + def test_search_next(self): a = bitarray('10011') - self.assertRaises(TypeError, a.itersearch, '') - it = a.itersearch(1) + self.assertRaises(TypeError, a.search, '') + it = a.search(1) self.assertIsType(it, 'searchiterator') self.assertEqual(next(it), 0) self.assertEqual(next(it), 3) self.assertEqual(next(it), 4) - self.assertStopIteration(it) + self.assertRaises(StopIteration, next, it) x = bitarray('11') - it = a.itersearch(x) + it = a.search(x) del a, x self.assertEqual(next(it), 3) - def test_itersearch_empty(self): + def test_search_empty(self): a = bitarray('10011') empty = bitarray() - self.assertEqual(list(a.itersearch(empty)), [0, 1, 2, 3, 4, 5]) + self.assertEqual(list(a.search(empty)), [0, 1, 2, 3, 4, 5]) for start, stop, right, res in [ (-9, 9, 0, [0, 1, 2, 3, 4, 5]), ( 1, 4, 0, [1, 2, 3, 4]), @@ -3596,7 +3718,7 @@ class SearchTests(unittest.TestCase, Uti ( 2, 2, 1, [2]), ( 2, 1, 1, []), ]: - self.assertEqual(list(a.itersearch(empty, start, stop, right)), + self.assertEqual(list(a.search(empty, start, stop, right)), res) def test_explicit_1(self): @@ -3607,8 +3729,7 @@ class SearchTests(unittest.TestCase, Uti ('011', [2]), ('0011', [1]), ('10011', [0]), ('100111', [])]: b = bitarray(s, self.random_endian()) - self.assertEqual(a.search(b), res) - self.assertEqual(list(a.itersearch(b)), res) + self.assertEqual(list(a.search(b)), res) def test_explicit_2(self): a = bitarray('10010101 11001111 1001011') @@ -3617,49 +3738,47 @@ class SearchTests(unittest.TestCase, Uti ('1011', [5, 19]), ('100', [0, 9, 16])]: b = bitarray(s) - self.assertEqual(a.search(b), res) - self.assertEqual(list(a.itersearch(b)), res) + self.assertEqual(list(a.search(b)), res) def test_bool_random(self): for a in self.randombitarrays(): b = a.copy() b.setall(0) - b[list(a.itersearch(1))] = 1 + b[list(a.search(1))] = 1 self.assertEQUAL(b, a) b.setall(1) - b[list(a.itersearch(0))] = 0 + b[list(a.search(0))] = 0 self.assertEQUAL(b, a) - s = set(a.search(0) + a.search(1)) + s = set(a.search(0)) | set(a.search(1)) self.assertEqual(len(s), len(a)) def test_random(self): for a in self.randombitarrays(): if a: # search for a in itself - self.assertEqual(a.search(a), [0]) - self.assertEqual(list(a.itersearch(a)), [0]) - self.assertEqual(list(a.itersearch(a, right=1)), [0]) + self.assertEqual(list(a.search(a)), [0]) + self.assertEqual(list(a.search(a, right=1)), [0]) for sub in '0', '1', '01', '01', '11', '101', '1101', '01100': b = bitarray(sub, self.random_endian()) plst = [i for i in range(len(a)) if a[i:i + len(b)] == b] - self.assertEqual(a.search(b), plst) + self.assertEqual(list(a.search(b)), plst) - for p in a.itersearch(b): + for p in a.search(b): self.assertEqual(a[p:p + len(b)], b) - self.assertEqual(list(a.itersearch(b)), plst) + self.assertEqual(list(a.search(b)), plst) - for p in a.itersearch(b, right=1): + for p in a.search(b, right=1): self.assertEqual(a[p:p + len(b)], b) - self.assertEqual(list(a.itersearch(b, right=1)), plst[::-1]) + self.assertEqual(list(a.search(b, right=1)), plst[::-1]) - def test_itersearch_random(self): + def test_search_random(self): for _ in range(500): n = randrange(1, 50) - a = urandom(n, self.random_endian()) - b = urandom(randrange(0, 10), self.random_endian()) + a = urandom_2(n) + b = urandom_2(randrange(10)) i = randrange(n) j = randrange(n) aa = a[i:j] @@ -3671,34 +3790,34 @@ class SearchTests(unittest.TestCase, Uti plst = list(range(i, j + 1)) self.assertEqual(sorted(plst), plst) - self.assertEqual(list(a.itersearch(b, i, j)), plst) + self.assertEqual(list(a.search(b, i, j)), plst) if len(b) == 1: # test sub-bitarray being int - self.assertEqual(list(a.itersearch(b[0], i, j)), plst) + self.assertEqual(list(a.search(b[0], i, j)), plst) if plst: # test first and last using .find() self.assertEqual(a.find(b, i, j, 0), plst[0]) self.assertEqual(a.find(b, i, j, 1), plst[-1]) plst.reverse() - self.assertEqual(list(a.itersearch(b, i, j, 1)), plst) + self.assertEqual(list(a.search(b, i, j, 1)), plst) if len(b) == 1: # test sub-bitarray being int - self.assertEqual(list(a.itersearch(b[0], i, j, 1)), plst) + self.assertEqual(list(a.search(b[0], i, j, 1)), plst) # test contains self.assertEqual(b in aa, bool(plst) if b else True) if not plst: # test .find() not found - for right in 0, 1: - self.assertEqual(a.find(b, i, j, right), -1) + right = getrandbits(1) + self.assertEqual(a.find(b, i, j, right), -1) def test_iterator_change(self): for right in 0, 1: a = zeros(100) b = zeros(10) c = 0 - for i, x in enumerate(a.itersearch(b, right=right)): + for i, x in enumerate(a.search(b, right=right)): if i == 40: a.clear() c += 1 @@ -3709,31 +3828,24 @@ class SearchTests(unittest.TestCase, Uti a = zeros(100) b = zeros(0) c = 0 - for i, x in enumerate(a.itersearch(b, right=right)): + for i, x in enumerate(a.search(b, right=right)): if i == 20: b.append(1) c += 1 self.assertEqual(c, 21) -# --------------------------------------------------------------------------- +# ------------------------ .frombytes() and .tobytes() ---------------------- class BytesTests(unittest.TestCase, Util): - @staticmethod - def randombytes(): - for n in range(1, 20): - yield os.urandom(n) - def test_frombytes_simple(self): - a = bitarray(endian='big') + a = bitarray("110", "big") a.frombytes(b'A') - self.assertEqual(a, bitarray('01000001')) + self.assertEqual(a, bitarray('110 01000001')) - b = a - b.frombytes(b'BC') - self.assertEQUAL(b, bitarray('01000001 01000010 01000011', + a.frombytes(b'BC') + self.assertEQUAL(a, bitarray('110 01000001 01000010 01000011', endian='big')) - self.assertTrue(b is a) def test_frombytes_types(self): a = bitarray(endian='big') @@ -3745,33 +3857,33 @@ class BytesTests(unittest.TestCase, Util self.assertEqual(a, bitarray('01000001 11111110 01000011')) a.clear() - if is_py3k: # Python 2's array cannot be used as buffer - a.frombytes(array.array('B', [5, 255, 192])) - self.assertEqual(a, bitarray('00000101 11111111 11000000')) - + arr = array.array('H', [0x010f, 0xff]) # array + self.assertEqual(arr.itemsize, 2) + if sys.byteorder == "big": + arr.byteswap() + a.frombytes(arr) + self.assertEqual(a, bitarray("00001111 00000001 11111111 00000000")) self.check_obj(a) - for x in u'', 0, 1, False, True, None, []: + for x in '', 0, 1, False, True, None, []: self.assertRaises(TypeError, a.frombytes, x) def test_frombytes_bitarray(self): - for endian in 'little', 'big': - # endianness doesn't matter here as we're writting the buffer - # from bytes, and then getting the memoryview - b = bitarray(0, endian) - b.frombytes(b'ABC') - - a = bitarray(0, 'big') - a.frombytes(bitarray(b)) - self.assertEqual(a.endian(), 'big') - self.assertEqual(a, bitarray('01000001 01000010 01000011')) - self.check_obj(a) + # endianness doesn't matter here as we're writting the buffer + # from bytes, and then extend from buffer bytes again + b = bitarray(0, self.random_endian()) + b.frombytes(b'ABC') + + a = bitarray(0, 'big') + a.frombytes(bitarray(b)) # get bytes from bitarray buffer + self.assertEqual(a.endian, 'big') + self.assertEqual(a.tobytes(), b'ABC') + self.check_obj(a) def test_frombytes_self(self): a = bitarray() self.assertRaisesMessage( - BufferError, - "cannot resize bitarray that is exporting buffers", + BufferError, "cannot resize bitarray that is exporting buffers", a.frombytes, a) def test_frombytes_empty(self): @@ -3791,32 +3903,33 @@ class BytesTests(unittest.TestCase, Util self.check_obj(a) def test_frombytes_random(self): - for b in self.randombitarrays(): - for s in self.randombytes(): - a = bitarray(endian=b.endian()) - a.frombytes(s) - c = b.copy() - b.frombytes(s) - self.assertEQUAL(b[-len(a):], a) - self.assertEQUAL(b[:-len(a)], c) - self.assertEQUAL(b, c + a) - self.check_obj(a) + for n in range(20): + s = os.urandom(n) + b = bitarray(0, self.random_endian()) + b.frombytes(s) + self.assertEqual(len(b), 8 * n) + for a in self.randombitarrays(): + c = bitarray(a, b.endian) + c.frombytes(s) + self.assertEqual(len(c), len(a) + 8 * n) + self.assertEqual(c, a + b) + self.check_obj(c) def test_tobytes_empty(self): a = bitarray() self.assertEqual(a.tobytes(), b'') def test_tobytes_endian(self): - for end in ('big', 'little'): - a = bitarray(endian=end) - a.frombytes(b'foo') - self.assertEqual(a.tobytes(), b'foo') - - for s in self.randombytes(): - a = bitarray(endian=end) - a.frombytes(s) - self.assertEqual(a.tobytes(), s) - self.check_obj(a) + a = bitarray(endian=self.random_endian()) + a.frombytes(b'foo') + self.assertEqual(a.tobytes(), b'foo') + + for n in range(20): + s = os.urandom(n) + a = bitarray(s, endian=self.random_endian()) + self.assertEqual(len(a), 8 * n) + self.assertEqual(a.tobytes(), s) + self.check_obj(a) def test_tobytes_explicit_ones(self): for n, s in [(1, b'\x01'), (2, b'\x03'), (3, b'\x07'), (4, b'\x0f'), @@ -3826,101 +3939,7 @@ class BytesTests(unittest.TestCase, Util a = ones(n, endian='little') self.assertEqual(a.tobytes(), s) - def test_unpack_simple(self): - a = bitarray('01') - self.assertIsInstance(a.unpack(), bytes) - self.assertEqual(a.unpack(), b'\x00\x01') - self.assertEqual(a.unpack(b'A'), b'A\x01') - self.assertEqual(a.unpack(b'0', b'1'), b'01') - self.assertEqual(a.unpack(one=b'\xff'), b'\x00\xff') - self.assertEqual(a.unpack(zero=b'A'), b'A\x01') - self.assertEqual(a.unpack(one=b't', zero=b'f'), b'ft') - - def test_unpack_random(self): - for a in self.randombitarrays(): - self.assertEqual(a.unpack(b'0', b'1'), - a.to01().encode()) - # round trip - b = bitarray() - b.pack(a.unpack()) - self.assertEqual(b, a) - # round trip with invert - b = bitarray() - b.pack(a.unpack(b'\x01', b'\x00')) - b.invert() - self.assertEqual(b, a) - - def test_unpack_errors(self): - a = bitarray('01') - self.assertRaises(TypeError, a.unpack, b'') - self.assertRaises(TypeError, a.unpack, b'0', b'') - self.assertRaises(TypeError, a.unpack, b'a', zero=b'b') - self.assertRaises(TypeError, a.unpack, foo=b'b') - self.assertRaises(TypeError, a.unpack, one=b'aa', zero=b'b') - if is_py3k: - self.assertRaises(TypeError, a.unpack, '0') - self.assertRaises(TypeError, a.unpack, one='a') - self.assertRaises(TypeError, a.unpack, b'0', '1') - - def test_pack_simple(self): - for endian in 'little', 'big': - _set_default_endian(endian) - a = bitarray() - a.pack(bytes()) - self.assertEQUAL(a, bitarray()) - a.pack(b'\x00') - self.assertEQUAL(a, bitarray('0')) - a.pack(b'\xff') - self.assertEQUAL(a, bitarray('01')) - a.pack(b'\x01\x00\x7a') - self.assertEQUAL(a, bitarray('01101')) - a.pack(bytearray([0x01, 0x00, 0xff, 0xa7])) - self.assertEQUAL(a, bitarray('01101 1011')) - self.check_obj(a) - - def test_pack_types(self): - a = bitarray() - a.pack(b'\0\x01') # bytes - self.assertEqual(a, bitarray('01')) - a.pack(bytearray([0, 2])) # bytearray - self.assertEqual(a, bitarray('01 01')) - a.pack(memoryview(b'\x02\0')) # memoryview - self.assertEqual(a, bitarray('01 01 10')) - - if is_py3k: # Python 2's array cannot be used as buffer - a.pack(array.array('B', [0, 255, 192])) - self.assertEqual(a, bitarray('01 01 10 011')) - - self.check_obj(a) - - def test_pack_bitarray(self): - b = bitarray("00000000 00000001 10000000 11111111 00000000") - a = bitarray() - a.pack(bitarray(b)) - self.assertEqual(a, bitarray('01110')) - self.check_obj(a) - - def test_pack_self(self): - a = bitarray() - self.assertRaisesMessage( - BufferError, - "cannot resize bitarray that is exporting buffers", - a.pack, a) - - def test_pack_allbytes(self): - a = bitarray() - a.pack(bytearray(range(256))) - self.assertEqual(a, bitarray('0' + 255 * '1')) - self.check_obj(a) - - def test_pack_errors(self): - a = bitarray() - self.assertRaises(TypeError, a.pack, 0) - if is_py3k: - self.assertRaises(TypeError, a.pack, '1') - self.assertRaises(TypeError, a.pack, [1, 3]) - -# --------------------------------------------------------------------------- +# -------------------------- test attributes -------------------------------- class DescriptorTests(unittest.TestCase, Util): @@ -3928,18 +3947,18 @@ class DescriptorTests(unittest.TestCase, for a in self.randombitarrays(): self.assertEqual(a.nbytes, bits2bytes(len(a))) self.assertEqual(a.padbits, 8 * a.nbytes - len(a)) - if is_py3k: - self.assertIsInstance(a.nbytes, int) - self.assertIsInstance(a.padbits, int) + self.assertTrue(0 <= a.padbits < 8) + self.assertEqual(type(a.nbytes), int) + self.assertEqual(type(a.padbits), int) def test_readonly(self): a = bitarray('110') self.assertFalse(a.readonly) - self.assertIsInstance(a.readonly, bool) + self.assertEqual(type(a.readonly), bool) b = frozenbitarray(a) self.assertTrue(b.readonly) - self.assertIsInstance(b.readonly, bool) + self.assertEqual(type(b.readonly), bool) # --------------------------------------------------------------------------- @@ -4013,11 +4032,11 @@ class FileTests(unittest.TestCase, Util) def test_fromfile_wrong_args(self): a = bitarray() self.assertRaises(TypeError, a.fromfile) - self.assertRaises(Exception, a.fromfile, 42) - self.assertRaises(Exception, a.fromfile, 'bar') + self.assertRaises(AttributeError, a.fromfile, 42) + self.assertRaises(AttributeError, a.fromfile, 'bar') with open(self.tmpfname, 'wb') as fo: - pass + fo.write(b"ABC") with open(self.tmpfname, 'rb') as fi: self.assertRaises(TypeError, a.fromfile, fi, None) @@ -4028,11 +4047,21 @@ class FileTests(unittest.TestCase, Util) a = bitarray() with open(self.tmpfname, 'wb') as fi: - self.assertRaises(Exception, a.fromfile, fi) + self.assertRaisesMessage(UnsupportedOperation, "read", + a.fromfile, fi) - if is_py3k: - with open(self.tmpfname, 'r') as fi: - self.assertRaises(TypeError, a.fromfile, fi) + with open(self.tmpfname, 'r') as fi: + self.assertRaisesMessage(TypeError, ".read() did not return " + "'bytes', got 'str'", a.fromfile, fi) + + def test_frombytes_invalid_reader(self): + class Reader: + def read(self, n): + return 12 + a = bitarray() + f = Reader() + self.assertRaisesMessage(TypeError, ".read() did not return " + "'bytes', got 'int'", a.fromfile, f) def test_from_large_files(self): for N in range(65534, 65538): @@ -4062,8 +4091,7 @@ class FileTests(unittest.TestCase, Util) self.check_obj(a) def test_fromfile_n(self): - a = bitarray() - a.frombytes(b'ABCDEFGHIJ') + a = bitarray(b'ABCDEFGHIJ') with open(self.tmpfname, 'wb') as fo: a.tofile(fo) self.assertFileSize(10) @@ -4145,15 +4173,14 @@ class FileTests(unittest.TestCase, Util) # write to closed file self.assertRaises(ValueError, a.tofile, f) - if is_py3k: - with open(self.tmpfname, 'w') as f: - self.assertRaises(TypeError, a.tofile, f) + with open(self.tmpfname, 'w') as f: + self.assertRaises(TypeError, a.tofile, f) with open(self.tmpfname, 'rb') as f: self.assertRaises(Exception, a.tofile, f) def test_tofile_large(self): - n = 100 * 1000 + n = 100_000 a = zeros(8 * n) a[2::37] = 1 with open(self.tmpfname, 'wb') as f: @@ -4172,23 +4199,21 @@ class FileTests(unittest.TestCase, Util) raw = self.read_file() self.assertEqual(len(raw), a.nbytes) - # when we fill the padbits in a, we can compare + # when we fill the pad bits in a, we can compare a.fill() - b = bitarray(endian='little') - b.frombytes(raw) + b = bitarray(raw, endian='little') self.assertEqual(a, b) def test_tofile_BytesIO(self): for n in list(range(10)) + list(range(65534, 65538)): data = os.urandom(n) - a = bitarray(0, 'big') - a.frombytes(data) + a = bitarray(data, 'big') self.assertEqual(a.nbytes, n) f = BytesIO() a.tofile(f) self.assertEqual(f.getvalue(), data) - @skipIf(sys.version_info[0] == 2 or is_pypy) + @skipIf(is_pypy) def test_mmap(self): with open(self.tmpfname, 'wb') as fo: fo.write(1000 * b'\0') @@ -4208,7 +4233,7 @@ class FileTests(unittest.TestCase, Util) self.assertEqual(self.read_file(), 1000 * b'\x55') # pyodide hits emscripten mmap bug - @skipIf(sys.version_info[0] == 2 or pyodide or is_pypy) + @skipIf(pyodide or is_pypy) def test_mmap_2(self): with open(self.tmpfname, 'wb') as fo: fo.write(1000 * b'\x22') @@ -4223,7 +4248,7 @@ class FileTests(unittest.TestCase, Util) self.assertEqual(self.read_file(), 1000 * b'\x33') - @skipIf(sys.version_info[0] == 2 or is_pypy) + @skipIf(is_pypy) def test_mmap_readonly(self): with open(self.tmpfname, 'wb') as fo: fo.write(994 * b'\x89' + b'Veedon') @@ -4263,8 +4288,7 @@ class DecodeTreeTests(unittest.TestCase, def test_create(self): dt = decodetree(alphabet_code) - self.assertIsType(dt, 'decodetree') - self.assertIsInstance(dt, decodetree) + self.assertEqual(type(dt), decodetree) self.assertRaises(TypeError, decodetree, None) self.assertRaises(TypeError, decodetree, 'foo') d = dict(alphabet_code) @@ -4302,7 +4326,7 @@ class DecodeTreeTests(unittest.TestCase, def test_complete(self): dt = decodetree({'.': bitarray('1')}) - self.assertIsInstance(dt.complete(), bool) + self.assertEqual(type(dt.complete()), bool) self.assertFalse(dt.complete()) dt = decodetree({'a': bitarray('0'), @@ -4321,17 +4345,16 @@ class DecodeTreeTests(unittest.TestCase, def test_todict(self): t = decodetree(alphabet_code) d = t.todict() - self.assertIsInstance(d, dict) + self.assertEqual(type(d), dict) self.assertEqual(d, alphabet_code) def test_decode(self): t = decodetree(alphabet_code) a = bitarray('1011 01110 0110 1001') - self.assertEqual(a.decode(t), ['i', 'l', 'a', 'n']) - self.assertEqual(''.join(a.iterdecode(t)), 'ilan') + self.assertEqual(list(a.decode(t)), ['i', 'l', 'a', 'n']) + self.assertEqual(''.join(a.decode(t)), 'ilan') a = bitarray() - self.assertEqual(a.decode(t), []) - self.assertEqual(''.join(a.iterdecode(t)), '') + self.assertEqual(list(a.decode(t)), []) self.check_obj(a) @skipIf(is_pypy) @@ -4341,6 +4364,7 @@ class DecodeTreeTests(unittest.TestCase, t = decodetree(d) self.assertEqual(t.todict(), d) self.assertEqual(t.nodes(), 2047) + self.assertTrue(t.complete()) self.assertTrue(sys.getsizeof(t) > 10000) # ------------------ variable length encoding and decoding ------------------ @@ -4383,9 +4407,7 @@ class PrefixCodeTests(unittest.TestCase, a.encode(d, 'is') self.assertEqual(a, bitarray('1011 1100')) self.assertRaises(ValueError, a.encode, d, 'ilAn') - msg = "symbol not defined in prefix code" - if is_py3k: - msg += ": None" + msg = "symbol not defined in prefix code: None" self.assertRaisesMessage(ValueError, msg, a.encode, d, [None, 2]) def test_encode_not_iterable(self): @@ -4406,58 +4428,52 @@ class PrefixCodeTests(unittest.TestCase, self.assertEqual(len(a), 0) def test_check_codedict_decode(self): - a = bitarray('101') + a = bitarray('1100101') self.assertRaises(TypeError, a.decode, 0) self.assertRaises(ValueError, a.decode, {}) self.assertRaises(TypeError, a.decode, {'a': 42}) + self.assertRaises(TypeError, a.decode, {'a': []}) self.assertRaises(ValueError, a.decode, {'a': bitarray()}) - self.assertEqual(a, bitarray('101')) - - def test_check_codedict_iterdecode(self): - a = bitarray('1100101') - self.assertRaises(TypeError, a.iterdecode, 0) - self.assertRaises(ValueError, a.iterdecode, {}) - self.assertRaises(TypeError, a.iterdecode, {'a': []}) - self.assertRaises(ValueError, a.iterdecode, {'a': bitarray()}) self.assertEqual(a, bitarray('1100101')) + def test_no_iterdecode(self): + a = bitarray() + # removed in bitarray 3.0 + self.assertRaises(AttributeError, a.__getattribute__, 'iterdecode') + def test_decode_simple(self): d = {'I': bitarray('1'), 'l': bitarray('01'), 'a': bitarray('001'), 'n': bitarray('000')} dcopy = dict(d) a = bitarray('101001000') res = list("Ilan") - self.assertEqual(a.decode(d), res) - self.assertEqual(list(a.iterdecode(d)), res) + self.assertEqual(list(a.decode(d)), res) self.assertEqual(d, dcopy) self.assertEqual(a, bitarray('101001000')) - def test_iterdecode_type(self): + def test_decode_type(self): a = bitarray('0110') - it = a.iterdecode(alphabet_code) + it = a.decode(alphabet_code) self.assertIsType(it, 'decodeiterator') self.assertEqual(list(it), ['a']) - def test_iterdecode_remove(self): + def test_decode_remove(self): d = {'I': bitarray('1'), 'l': bitarray('01'), 'a': bitarray('001'), 'n': bitarray('000')} t = decodetree(d) a = bitarray('101001000') - it = a.iterdecode(t) + it = a.decode(t) del t # remove tree self.assertEqual(''.join(it), "Ilan") - it = a.iterdecode(d) + it = a.decode(d) del a self.assertEqual(''.join(it), "Ilan") def test_decode_empty(self): d = {'a': bitarray('1')} a = bitarray() - self.assertEqual(a.decode(d), []) - self.assertEqual(d, {'a': bitarray('1')}) - # test decode iterator - self.assertEqual(list(a.iterdecode(d)), []) + self.assertEqual(list(a.decode(d)), []) self.assertEqual(d, {'a': bitarray('1')}) self.assertEqual(len(a), 0) @@ -4465,13 +4481,12 @@ class PrefixCodeTests(unittest.TestCase, d = {'a': bitarray('0'), 'b': bitarray('111')} a = bitarray('00011') msg = "incomplete prefix code at position 3" - self.assertRaisesMessage(ValueError, msg, a.decode, d) - it = a.iterdecode(d) + self.assertRaisesMessage(ValueError, msg, list, a.decode(d)) + it = a.decode(d) self.assertIsType(it, 'decodeiterator') self.assertRaisesMessage(ValueError, msg, list, it) t = decodetree(d) - self.assertRaisesMessage(ValueError, msg, a.decode, t) - self.assertRaisesMessage(ValueError, msg, list, a.iterdecode(t)) + self.assertRaisesMessage(ValueError, msg, list, a.decode(t)) self.assertEqual(a, bitarray('00011')) self.assertEqual(d, {'a': bitarray('0'), 'b': bitarray('111')}) @@ -4483,49 +4498,46 @@ class PrefixCodeTests(unittest.TestCase, x = len(a) a.extend('00') msg = "incomplete prefix code at position %d" % x - self.assertRaisesMessage(ValueError, msg, a.decode, alphabet_code) + self.assertRaisesMessage(ValueError, msg, + list, a.decode(alphabet_code)) + + def test_decode_no_term(self): + d = {'a': bitarray('0'), 'b': bitarray('111')} + a = bitarray('011') + it = a.decode(d) + self.assertEqual(next(it), 'a') + self.assertRaisesMessage(ValueError, + "incomplete prefix code at position 1", + next, it) + self.assertEqual(a, bitarray('011')) def test_decode_buggybitarray(self): d = dict(alphabet_code) # i s t a = bitarray('1011 1100 0100 011110111001101001') msg = "prefix code unrecognized in bitarray at position 12 .. 21" - self.assertRaisesMessage(ValueError, msg, a.decode, d) - self.assertRaisesMessage(ValueError, msg, list, a.iterdecode(d)) + self.assertRaisesMessage(ValueError, msg, list, a.decode(d)) t = decodetree(d) - self.assertRaisesMessage(ValueError, msg, a.decode, t) - self.assertRaisesMessage(ValueError, msg, list, a.iterdecode(d)) + self.assertRaisesMessage(ValueError, msg, list, a.decode(t)) self.check_obj(a) self.assertEqual(t.todict(), d) - def test_iterdecode_no_term(self): - d = {'a': bitarray('0'), 'b': bitarray('111')} - a = bitarray('011') - it = a.iterdecode(d) - self.assertEqual(next(it), 'a') - self.assertRaisesMessage(ValueError, - "incomplete prefix code at position 1", - next, it) - self.assertEqual(a, bitarray('011')) - - def test_iterdecode_buggybitarray(self): + def test_decode_buggybitarray2(self): d = {'a': bitarray('0')} a = bitarray('1') - it = a.iterdecode(d) + it = a.decode(d) self.assertRaises(ValueError, next, it) self.assertEqual(a, bitarray('1')) self.assertEqual(d, {'a': bitarray('0')}) - def test_decode_buggybitarray2(self): + def test_decode_buggybitarray3(self): d = {'a': bitarray('00'), 'b': bitarray('01')} a = bitarray('1') - self.assertRaises(ValueError, a.decode, d) - self.assertRaises(ValueError, next, a.iterdecode(d)) + self.assertRaises(ValueError, next, a.decode(d)) t = decodetree(d) - self.assertRaises(ValueError, a.decode, t) - self.assertRaises(ValueError, next, a.iterdecode(t)) + self.assertRaises(ValueError, next, a.decode(t)) self.assertEqual(a, bitarray('1')) self.assertEqual(d, {'a': bitarray('00'), 'b': bitarray('01')}) @@ -4559,7 +4571,6 @@ class PrefixCodeTests(unittest.TestCase, ]: a = bitarray() self.assertRaises(ValueError, a.decode, d) - self.assertRaises(ValueError, a.iterdecode, d) self.check_obj(a) def test_miscitems(self): @@ -4571,15 +4582,15 @@ class PrefixCodeTests(unittest.TestCase, a = bitarray() a.encode(d, [None, 0, 1, '', 2]) self.assertEqual(a, bitarray('00110111010011')) - self.assertEqual(a.decode(d), [None, 0, 1, '', 2]) + self.assertEqual(list(a.decode(d)), [None, 0, 1, '', 2]) # iterator - it = a.iterdecode(d) - self.assertEqual(next(it), None) + it = a.decode(d) + self.assertTrue(next(it) is None) self.assertEqual(next(it), 0) self.assertEqual(next(it), 1) self.assertEqual(next(it), '') self.assertEqual(next(it), 2) - self.assertStopIteration(it) + self.assertRaises(StopIteration, next, it) def test_quick_example(self): a = bitarray() @@ -4597,10 +4608,8 @@ class PrefixCodeTests(unittest.TestCase, # d o g . '01011 1000 101000 0101010')) self.assertEqual(''.join(a.decode(alphabet_code)), message) - self.assertEqual(''.join(a.iterdecode(alphabet_code)), message) t = decodetree(alphabet_code) self.assertEqual(''.join(a.decode(t)), message) - self.assertEqual(''.join(a.iterdecode(t)), message) self.check_obj(a) # --------------------------- Buffer Import --------------------------------- @@ -4640,7 +4649,7 @@ class BufferImportTests(unittest.TestCas self.assertRaises(BufferError, a.pop) a[8:16] = bitarray('10000010', endian='big') self.assertEqual(b, bytearray([255, 65] + 98 * [255])) - self.assertEqual(a.tobytes(), bytes(b)) + self.assertEqual(a.tobytes(), b) for n in 7, 9: self.assertRaises(BufferError, a.__setitem__, slice(8, 16), bitarray(n)) @@ -4649,8 +4658,7 @@ class BufferImportTests(unittest.TestCas self.assertEqual(a, 800 * bitarray('1')) self.check_obj(a) - # Python 2's array cannot be used as buffer - @skipIf(sys.version_info[0] == 2 or is_pypy) + @skipIf(is_pypy) def test_array(self): a = array.array('B', [0, 255, 64]) b = bitarray(None, 'little', a) @@ -4662,8 +4670,8 @@ class BufferImportTests(unittest.TestCas self.check_obj(b) def test_bitarray(self): - a = urandom(10000) - b = bitarray(buffer=a) + a = urandom_2(10000, 'little') + b = bitarray(endian='little', buffer=a) # a and b are two distinct bitarrays that share the same buffer now self.assertFalse(a is b) @@ -4673,9 +4681,8 @@ class BufferImportTests(unittest.TestCas b_info = buffer_info(b) self.assertTrue(b_info['imported']) self.assertEqual(b_info['exports'], 0) - # buffer address is the same! - self.assertEqual(a_info['address'], - b_info['address']) + # buffer address is the same + self.assertEqual(a_info['address'], b_info['address']) self.assertFalse(a is b) self.assertEqual(a, b) @@ -4689,12 +4696,10 @@ class BufferImportTests(unittest.TestCas self.assertEqual(a, b) self.assertRaisesMessage( - BufferError, - "cannot resize bitarray that is exporting buffers", + BufferError, "cannot resize bitarray that is exporting buffers", a.pop) self.assertRaisesMessage( - BufferError, - "cannot resize imported buffer", + BufferError, "cannot resize imported buffer", b.pop) self.check_obj(a) self.check_obj(b) @@ -4710,7 +4715,7 @@ class BufferImportTests(unittest.TestCas @skipIf(is_pypy) def test_bitarray_shared_sections(self): - a = urandom(0x2000) + a = urandom_2(0x2000, 'big') b = bitarray(buffer=memoryview(a)[0x100:0x300]) self.assertEqual(buffer_info(b, 'address'), buffer_info(a, 'address') + 0x100) @@ -4730,9 +4735,9 @@ class BufferImportTests(unittest.TestCas def test_bitarray_range(self): for n in range(100): - a = urandom(n, self.random_endian()) - b = bitarray(buffer=a, endian=a.endian()) - # an imported buffer will never have padbits + a = urandom_2(n) + b = bitarray(buffer=a, endian=a.endian) + # an imported buffer will never have any pad bits self.assertEqual(b.padbits, 0) self.assertEqual(len(b) % 8, 0) self.assertEQUAL(b[:n], a) @@ -4740,18 +4745,16 @@ class BufferImportTests(unittest.TestCas self.check_obj(b) def test_bitarray_chain(self): - a = urandom(64) - d = {0: a} + d = [urandom_2(64, 'big')] for n in range(1, 100): - d[n] = bitarray(buffer=d[n - 1]) + d.append(bitarray(endian='big', buffer=d[n - 1])) - self.assertEqual(d[99], a) - a.setall(0) - self.assertEqual(d[99], zeros(64)) - a[:] = 1 - self.assertTrue(d[99].all()) - for c in d.values(): - self.check_obj(c) + self.assertEqual(d[99], d[0]) + d[0].setall(1) + for a in d: + self.assertEqual(len(a), 64) + self.assertTrue(a.all()) + self.check_obj(a) def test_frozenbitarray(self): a = frozenbitarray('10011011 011') @@ -4840,7 +4843,7 @@ class BufferExportTests(unittest.TestCas self.assertFalse(v.readonly) self.assertEqual(buffer_info(a, 'exports'), 1) self.assertEqual(len(v), 3) - self.assertEqual(v[0], 65 if is_py3k else 'A') + self.assertEqual(v[0], 65) self.assertEqual(v.tobytes(), b'ABC') a[13] = 1 self.assertEqual(v.tobytes(), b'AFC') @@ -4871,8 +4874,7 @@ class BufferExportTests(unittest.TestCas self.check_obj(a) def test_read_random(self): - a = bitarray() - a.frombytes(os.urandom(100)) + a = bitarray(os.urandom(100)) v = memoryview(a) self.assertEqual(len(v), 100) b = a[34 * 8 : 67 * 8] @@ -4910,7 +4912,7 @@ class BufferExportTests(unittest.TestCas a = zeros(8000) v = memoryview(a) self.assertFalse(v.readonly) - v[500] = 255 if is_py3k else '\xff' + v[500] = 255 self.assertEqual(a[3999:4009], bitarray('0111111110')) a[4003] = 0 self.assertEqual(a[3999:4009], bitarray('0111011110')) @@ -4918,8 +4920,7 @@ class BufferExportTests(unittest.TestCas self.assertEqual(a[300 * 8 : 305 * 8].tobytes(), b'\x00ABC\x00') self.check_obj(a) - @skipIf(sys.version_info[0] == 2) - def test_write_py3(self): + def test_write_memoryview_slice(self): a = zeros(40) m = memoryview(a) v = m[1:4] @@ -4927,33 +4928,35 @@ class BufferExportTests(unittest.TestCas v[1] = 66 v[2] = 67 self.assertEqual(a.tobytes(), b'\x00ABC\x00') + m[1:4] = b'XYZ' + self.assertEqual(a.tobytes(), b'\x00XYZ\x00') self.check_obj(a) # --------------------------------------------------------------------------- -class TestsFrozenbitarray(unittest.TestCase, Util): +class FrozenbitarrayTests(unittest.TestCase, Util): def test_init(self): a = frozenbitarray('110') self.assertEqual(a, bitarray('110')) self.assertEqual(a.to01(), '110') self.assertIsInstance(a, bitarray) - self.assertIsType(a, 'frozenbitarray') + self.assertEqual(type(a), frozenbitarray) self.assertTrue(a.readonly) self.check_obj(a) a = frozenbitarray(bitarray()) self.assertEQUAL(a, frozenbitarray()) - self.assertIsType(a, 'frozenbitarray') + self.assertEqual(type(a), frozenbitarray) for endian in 'big', 'little': a = frozenbitarray(0, endian) - self.assertEqual(a.endian(), endian) - self.assertIsType(a, 'frozenbitarray') + self.assertEqual(a.endian, endian) + self.assertEqual(type(a), frozenbitarray) a = frozenbitarray(bitarray(0, endian)) - self.assertEqual(a.endian(), endian) - self.assertIsType(a, 'frozenbitarray') + self.assertEqual(a.endian, endian) + self.assertEqual(type(a), frozenbitarray) def test_methods(self): # test a few methods which do not raise the TypeError @@ -4964,7 +4967,7 @@ class TestsFrozenbitarray(unittest.TestC self.assertEqual(a.index(0), 2) b = a.copy() self.assertEqual(b, a) - self.assertIsType(b, 'frozenbitarray') + self.assertEqual(type(b), frozenbitarray) self.assertEqual(len(b), 7) self.assertFalse(b.all()) self.assertTrue(b.any()) @@ -4985,7 +4988,20 @@ class TestsFrozenbitarray(unittest.TestC tup = 0, 1, 0, 1, 1, False, True for obj in list(tup), tup, iter(tup), bitarray(tup): a = frozenbitarray(obj) + self.assertEqual(type(a), frozenbitarray) self.assertEqual(a, bitarray(tup)) + a = frozenbitarray(b'AB', "big") + self.assertEqual(a.to01(), "0100000101000010") + self.assertEqual(a.endian, "big") + + def test_init_bytes_bytearray(self): + for x in b'\x80ABCD', bytearray(b'\x80ABCD'): + a = frozenbitarray(x, 'little') + self.assertEqual(type(a), frozenbitarray) + self.assertEqual(len(a), 8 * len(x)) + self.assertEqual(a.tobytes(), x) + self.assertEqual(a[:8].to01(), '00000001') + self.check_obj(a) def test_repr(self): a = frozenbitarray() @@ -5028,7 +5044,7 @@ class TestsFrozenbitarray(unittest.TestC for b in [a, a.copy(), 3 * a, 5 * a, a & bitarray('110'), a >> 2, ~a, a + bitarray(8*'1'), a[:], a[::2], a[[0, 1]], a[bitarray('011')]]: - self.assertIsType(b, 'frozenbitarray') + self.assertEqual(type(b), frozenbitarray) self.assertTrue(b.readonly) self.check_obj(b) @@ -5043,10 +5059,10 @@ class TestsFrozenbitarray(unittest.TestC a = frozenbitarray('01000001 01000010', 'big') v = memoryview(a) self.assertEqual(v.tobytes(), b'AB') - self.assertRaises(TypeError, v.__setitem__, 0, 255) + self.assertRaises(TypeError, v.__setitem__, 0, 0x7c) def test_buffer_import_readonly(self): - b = bytes(bytearray([15, 95, 128])) + b = bytes([15, 95, 128]) a = frozenbitarray(buffer=b, endian='big') self.assertEQUAL(a, bitarray('00001111 01011111 10000000', 'big')) info = buffer_info(a) @@ -5061,7 +5077,7 @@ class TestsFrozenbitarray(unittest.TestC "cannot import writable buffer into frozenbitarray", frozenbitarray, buffer=c) - def test_set(self): + def test_as_set(self): a = frozenbitarray('1') b = frozenbitarray('11') c = frozenbitarray('01') @@ -5071,18 +5087,18 @@ class TestsFrozenbitarray(unittest.TestC self.assertTrue(d in s) self.assertFalse(frozenbitarray('0') in s) - def test_dictkey(self): + def test_as_dictkey(self): a = frozenbitarray('01') b = frozenbitarray('1001') d = {a: 123, b: 345} self.assertEqual(d[frozenbitarray('01')], 123) self.assertEqual(d[frozenbitarray(b)], 345) - def test_dictkey2(self): # taken slightly modified from issue #74 - a1 = frozenbitarray([True, False]) - a2 = frozenbitarray([False, False]) + def test_as_dictkey2(self): # taken slightly modified from issue #74 + a1 = frozenbitarray("10") + a2 = frozenbitarray("00") dct = {a1: "one", a2: "two"} - a3 = frozenbitarray([True, False]) + a3 = frozenbitarray("10") self.assertEqual(a3, a1) self.assertEqual(dct[a3], 'one') @@ -5105,9 +5121,9 @@ class TestsFrozenbitarray(unittest.TestC def test_hash_endianness_random(self): for a in self.randombitarrays(): a = frozenbitarray(a) - b = frozenbitarray(a, self.opposite_endian(a.endian())) + b = frozenbitarray(a, self.opposite_endian(a.endian)) self.assertEqual(a, b) - self.assertNotEqual(a.endian(), b.endian()) + self.assertNotEqual(a.endian, b.endian) self.assertEqual(hash(a), hash(b)) d = {a: 1, b: 2} self.assertEqual(len(d), 1) @@ -5118,7 +5134,7 @@ class TestsFrozenbitarray(unittest.TestC f.foo = 42 # unlike bitarray itself, we can have attributes g = pickle.loads(pickle.dumps(f)) self.assertEqual(f, g) - self.assertEqual(f.endian(), g.endian()) + self.assertEqual(f.endian, g.endian) self.assertTrue(str(g).startswith('frozenbitarray')) self.assertTrue(g.readonly) self.check_obj(a) @@ -5126,6 +5142,13 @@ class TestsFrozenbitarray(unittest.TestC self.check_obj(g) self.assertEqual(g.foo, 42) + def test_bytes_bytearray(self): + for a in self.randombitarrays(): + a = frozenbitarray(a) + self.assertEqual(bytes(a), a.tobytes()) + self.assertEqual(bytearray(a), a.tobytes()) + self.check_obj(a) + # --------------------------------------------------------------------------- def run(verbosity=1): diff -pruN 2.9.2-1/bitarray/test_util.py 3.6.1-1/bitarray/test_util.py --- 2.9.2-1/bitarray/test_util.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/test_util.py 2025-08-12 08:35:42.000000000 +0000 @@ -5,75 +5,66 @@ from __future__ import absolute_import import os import sys +import math +import array import base64 import binascii +import operator +import struct import shutil import tempfile import unittest -from array import array +from io import StringIO +from functools import reduce from string import hexdigits -from random import choice, getrandbits, randrange, randint, random +from random import (choice, getrandbits, randrange, randint, random, sample, + seed) from collections import Counter from bitarray import (bitarray, frozenbitarray, decodetree, bits2bytes, _set_default_endian) -from bitarray.test_bitarray import Util, skipIf, SYSINFO, DEBUG +from bitarray.test_bitarray import (Util, skipIf, is_pypy, urandom_2, + PTRSIZE, WHITESPACE) from bitarray.util import ( - zeros, ones, urandom, pprint, make_endian, rindex, strip, count_n, - parity, count_and, count_or, count_xor, any_and, subset, _correspond_all, - intervals, + zeros, ones, urandom, random_k, random_p, pprint, strip, count_n, + parity, sum_indices, xor_indices, + count_and, count_or, count_xor, any_and, subset, + correspond_all, byteswap, intervals, serialize, deserialize, ba2hex, hex2ba, ba2base, base2ba, ba2int, int2ba, sc_encode, sc_decode, vl_encode, vl_decode, - huffman_code, canonical_huffman, canonical_decode, + _huffman_tree, huffman_code, canonical_huffman, canonical_decode, ) -if DEBUG: - from bitarray._util import _sc_rts, _SEGSIZE # type: ignore - -if sys.version_info[0] == 3: - from io import StringIO -else: - from io import BytesIO as StringIO +from bitarray.util import _Random, _ssqi # type: ignore # --------------------------------------------------------------------------- -class TestsZerosOnes(unittest.TestCase): - - def test_range(self): - for n in range(100): - a = zeros(n) - self.assertEqual(len(a), n) - self.assertFalse(a.any()) - self.assertEqual(a.count(0), n) - self.assertEqual(a.count(1), 0) - self.assertIsInstance(a, bitarray) - b = ones(n) - self.assertEqual(len(b), n) - self.assertTrue(b.all()) - self.assertEqual(b.count(0), 0) - self.assertEqual(b.count(1), n) - self.assertIsInstance(b, bitarray) +class ZerosOnesTests(unittest.TestCase): - def test_endian(self): - for default_endian in 'big', 'little': + def test_basic(self): + for _ in range(50): + default_endian = choice(['little', 'big']) _set_default_endian(default_endian) + a = choice([zeros(0), zeros(0, None), zeros(0, endian=None), + ones(0), ones(0, None), ones(0, endian=None)]) + self.assertEqual(a, bitarray()) + self.assertEqual(a.endian, default_endian) + self.assertEqual(type(a), bitarray) - for a in (zeros(0), zeros(0, None), zeros(0, endian=None), - ones(0), ones(0, None), ones(0, endian=None)): - self.assertEqual(a, bitarray()) - self.assertEqual(a.endian(), default_endian) - - for endian in 'big', 'little': - for a in zeros(3, endian), zeros(3, endian=endian): - self.assertEqual(a, bitarray('000')) - self.assertEqual(a.endian(), endian) - for b in ones(3, endian), ones(3, endian=endian): - self.assertEqual(b, bitarray('111')) - self.assertEqual(b.endian(), endian) + endian = choice(['little', 'big', None]) + n = randrange(100) - def test_wrong_args(self): + a = choice([zeros(n, endian), zeros(n, endian=endian)]) + self.assertEqual(a.to01(), n * "0") + self.assertEqual(a.endian, endian or default_endian) + + b = choice([ones(n, endian), ones(n, endian=endian)]) + self.assertEqual(b.to01(), n * "1") + self.assertEqual(b.endian, endian or default_endian) + + def test_errors(self): for f in zeros, ones: self.assertRaises(TypeError, f) # no argument self.assertRaises(TypeError, f, '') @@ -90,47 +81,299 @@ class TestsZerosOnes(unittest.TestCase): # --------------------------------------------------------------------------- -class TestsURandom(unittest.TestCase): +class URandomTests(unittest.TestCase): def test_basic(self): - for default_endian in 'big', 'little': + for _ in range(20): + default_endian = choice(['little', 'big']) _set_default_endian(default_endian) + a = choice([urandom(0), urandom(0, endian=None)]) + self.assertEqual(a, bitarray()) + self.assertEqual(a.endian, default_endian) - for a in urandom(0), urandom(0, None), urandom(0, endian=None): - self.assertEqual(a, bitarray()) - self.assertEqual(a.endian(), default_endian) + endian = choice(['little', 'big', None]) + n = randrange(100) - for n in range(50): - a = urandom(n) - self.assertEqual(len(a), n) - self.assertEqual(a.endian(), default_endian) + a = choice([urandom(n, endian), urandom(n, endian=endian)]) + self.assertEqual(len(a), n) + self.assertEqual(a.endian, endian or default_endian) + self.assertEqual(type(a), bitarray) - for endian in 'big', 'little': - for a in urandom(11, endian), urandom(11, endian=endian): - self.assertEqual(len(a), 11) - self.assertEqual(a.endian(), endian) + def test_errors(self): + U = urandom + self.assertRaises(TypeError, U) + self.assertRaises(TypeError, U, '') + self.assertRaises(TypeError, U, bitarray()) + self.assertRaises(TypeError, U, []) + self.assertRaises(TypeError, U, 1.0) + self.assertRaises(ValueError, U, -1) + self.assertRaises(TypeError, U, 0, 1) + self.assertRaises(ValueError, U, 0, 'foo') def test_count(self): - a = urandom(1000) - b = urandom(1000) - self.assertNotEqual(a, b) - self.assertTrue(400 < a.count() < 600) - self.assertTrue(400 < b.count() < 600) + a = urandom(10_000_000) + # see if population is within expectation + self.assertTrue(abs(a.count() - 5_000_000) <= 15_811) - def test_wrong_args(self): - self.assertRaises(TypeError, urandom) - self.assertRaises(TypeError, urandom, '') - self.assertRaises(TypeError, urandom, bitarray()) - self.assertRaises(TypeError, urandom, []) - self.assertRaises(TypeError, urandom, 1.0) - self.assertRaises(ValueError, urandom, -1) +# ---------------------------- .random_k() ---------------------------------- + +HAVE_RANDBYTES = sys.version_info[:2] >= (3, 9) + +@skipIf(HAVE_RANDBYTES) +class Random_K_Not_Implemented(unittest.TestCase): + + def test_not_implemented(self): + self.assertRaises(NotImplementedError, random_k, 100, 60) + + +@skipIf(not HAVE_RANDBYTES) +class Random_K_Tests(unittest.TestCase): + + def test_basic(self): + for _ in range(250): + default_endian = choice(['little', 'big']) + _set_default_endian(default_endian) + endian = choice(['little', 'big', None]) + n = randrange(120) + k = randint(0, n) + a = random_k(n, k, endian) + self.assertTrue(type(a), bitarray) + self.assertEqual(len(a), n) + self.assertEqual(a.count(), k) + self.assertEqual(a.endian, endian or default_endian) + + def test_inputs_and_edge_cases(self): + R = random_k + self.assertRaises(TypeError, R) + self.assertRaises(TypeError, R, 4) + self.assertRaises(TypeError, R, 1, "0.5") + self.assertRaises(TypeError, R, 1, p=1) + self.assertRaises(TypeError, R, 11, 5.5) # see issue #239 + self.assertRaises(ValueError, R, -1, 0) + for k in -1, 11: # k is not 0 <= k <= n + self.assertRaises(ValueError, R, 10, k) + self.assertRaises(ValueError, R, 10, 7, 'foo') + self.assertRaises(ValueError, R, 10, 7, endian='foo') + for n in range(20): + self.assertEqual(R(n, k=0), zeros(n)) + self.assertEqual(R(n, k=n), ones(n)) + + def test_count(self): + for n in range(10): # test explicitly for small n + for k in range(n + 1): + a = random_k(n, k) + self.assertEqual(len(a), n) + self.assertEqual(a.count(), k) + + for _ in range(100): + n = randrange(10_000) + k = randint(0, n) + a = random_k(n, k) + self.assertEqual(len(a), n) + self.assertEqual(a.count(), k) + + def test_active_bits(self): + # test if all bits are active + n = 240 + cum = zeros(n) + for _ in range(1000): + k = randint(30, 40) + a = random_k(n, k) + self.assertEqual(a.count(), k) + cum |= a + if cum.all(): + break + else: + self.fail() + + def test_combinations(self): + # for entire range of 0 <= k <= n, validate that random_k() + # generates all possible combinations + n = 7 + total = 0 + for k in range(n + 1): + expected = math.comb(n, k) + combs = set() + for _ in range(10_000): + combs.add(frozenbitarray(random_k(n, k))) + if len(combs) == expected: + total += expected + break + else: + self.fail() + self.assertEqual(total, 2 ** n) + + def collect_code_branches(self): + # return list of bitarrays from all code branches of random_k() + res = [] + # test small k (no .combine_half()) + res.append(random_k(300, 10)) + # general cases + for k in 100, 500, 2_500, 4_000: + res.append(random_k(5_000, k)) + return res + + def test_seed(self): + # We ensure that after setting a seed value, random_k() will + # always return the same random bitarrays. However, we do not ensure + # that these results will not change in future versions of bitarray. + _set_default_endian("little") + a = [] + for val in 654321, 654322, 654321, 654322: + seed(val) + a.append(self.collect_code_branches()) + self.assertEqual(a[0], a[2]) + self.assertEqual(a[1], a[3]) + self.assertNotEqual(a[0], a[1]) + # initialize seed with current system time again + seed() + + # ---------------- tests for internal _Random methods ------------------- + + def test_op_seq(self): + r = _Random() + G = r.op_seq + K = r.K + M = r.M + + # special cases + self.assertRaises(ValueError, G, 0) + self.assertEqual(G(1), zeros(M - 1)) + self.assertEqual(G(K // 2), bitarray()) + self.assertEqual(G(K - 1), ones(M - 1)) + self.assertRaises(ValueError, G, K) + + # examples + for p, s in [ + (0.15625, '0100'), + (0.25, '0'), # 1/2 AND -> 1/4 + (0.375, '10'), # 1/2 OR -> 3/4 AND -> 3/8 + (0.5, ''), + (0.625, '01'), # 1/2 AND -> 1/4 OR -> 5/8 + (0.6875, '101'), + (0.75, '1'), # 1/2 OR -> 3/4 + ]: + seq = G(int(p * K)) + self.assertEqual(seq.to01(), s) + + for i in range(1, K): + seq = G(i) + self.assertTrue(0 <= len(s) < M) + q = 0.5 # a = random_half() + for k in seq: + # k=0: AND k=1: OR + if k: + q += 0.5 * (1.0 - q) # a |= random_half() + else: + q *= 0.5 # a &= random_half() + self.assertEqual(q, i / K) + + def test_combine_half(self): + r = _Random(1_000_000) + for seq, mean in [ + ([], 500_000), # .random_half() itself + ([0], 250_000), # AND + ([1], 750_000), # OR + ([1, 0], 375_000), # OR followed by AND + ]: + a = r.combine_half(seq) + self.assertTrue(abs(a.count() - mean) < 5_000) + +# ---------------------------- .random_p() ---------------------------------- + +HAVE_BINOMIALVARIATE = sys.version_info[:2] >= (3, 12) + +@skipIf(HAVE_BINOMIALVARIATE) +class Random_P_Not_Implemented(unittest.TestCase): - self.assertRaises(TypeError, urandom, 0, 1) - self.assertRaises(ValueError, urandom, 0, 'foo') + def test_not_implemented(self): + self.assertRaises(NotImplementedError, random_p, 100, 0.25) + + +@skipIf(not HAVE_BINOMIALVARIATE) +class Random_P_Tests(unittest.TestCase): + + def test_basic(self): + for _ in range(250): + default_endian = choice(['little', 'big']) + _set_default_endian(default_endian) + endian = choice(['little', 'big', None]) + n = randrange(120) + p = choice([0.0, 0.0001, 0.2, 0.5, 0.9, 1.0]) + a = random_p(n, p, endian) + self.assertTrue(type(a), bitarray) + self.assertEqual(len(a), n) + self.assertEqual(a.endian, endian or default_endian) + + def test_inputs_and_edge_cases(self): + R = random_p + self.assertRaises(TypeError, R) + self.assertRaises(TypeError, R, 0.25) + self.assertRaises(TypeError, R, 1, "0.5") + self.assertRaises(ValueError, R, -1) + self.assertRaises(ValueError, R, 1, -0.5) + self.assertRaises(ValueError, R, 1, p=1.5) + self.assertRaises(ValueError, R, 1, 0.15, 'foo') + self.assertRaises(ValueError, R, 10, 0.5, endian='foo') + self.assertEqual(R(0), bitarray()) + for n in range(20): + self.assertEqual(R(n, 0), zeros(n)) + self.assertEqual(len(R(n, 0.5)), n) + self.assertEqual(R(n, p=1), ones(n)) + + def test_default(self): + a = random_p(10_000_000) # p defaults to 0.5 + # see if population is within expectation + self.assertTrue(abs(a.count() - 5_000_000) <= 15_811) + + def test_count(self): + for _ in range(500): + n = choice([randrange(4, 120), randrange(100, 1000)]) + p = choice([0.0001, 0.001, 0.01, 0.1, 0.25, 0.5, 0.9]) + sigma = math.sqrt(n * p * (1.0 - p)) + a = random_p(n, p) + self.assertEqual(len(a), n) + self.assertTrue(abs(a.count() - n * p) < max(4, 10 * sigma)) + + def collect_code_branches(self): + # return list of bitarrays from all code branches of random_p() + res = [] + # for default p=0.5, random_p uses randbytes + res.append(random_p(32)) + # test small p + res.append(random_p(5_000, 0.002)) + # small n (note that p=0.4 will call the "literal definition" case) + res.append(random_p(15, 0.4)) + # general cases + for p in 0.1, 0.2, 0.375, 0.4999, 0.7: + res.append(random_p(150, p)) + return res + + def test_seed(self): + # We ensure that after setting a seed value, random_p() will always + # return the same random bitarrays. However, we do not ensure that + # these results will not change in future versions of bitarray. + _set_default_endian("little") + a = [] + for val in 123456, 123457, 123456, 123457: + seed(val) + a.append(self.collect_code_branches()) + self.assertEqual(a[0], a[2]) + self.assertEqual(a[1], a[3]) + self.assertNotEqual(a[0], a[1]) + # initialize seed with current system time again + seed() + + def test_small_p_limit(self): + # For understanding how the algorithm works, see ./doc/random_p.rst + # Also, see VerificationTests in devel/test_random.py + r = _Random() + limit = 1.0 / (r.K + 1) # lower limit for p + self.assertTrue(r.SMALL_P > limit) # --------------------------------------------------------------------------- -class TestsPPrint(unittest.TestCase): +class PPrintTests(unittest.TestCase): @staticmethod def get_code_string(a): @@ -201,156 +444,7 @@ class TestsPPrint(unittest.TestCase): # --------------------------------------------------------------------------- -class TestsMakeEndian(unittest.TestCase, Util): - - def test_simple(self): - a = bitarray('1110001', endian='big') - b = make_endian(a, 'big') - self.assertTrue(b is a) - c = make_endian(a, endian='little') - self.assertTrue(c == a) - self.assertEqual(c.endian(), 'little') - self.assertIsType(c, 'bitarray') - - # wrong arguments - self.assertRaises(TypeError, make_endian, '', 'big') - self.assertRaises(TypeError, make_endian, bitarray(), 1) - self.assertRaises(ValueError, make_endian, bitarray(), 'foo') - - def test_empty(self): - a = bitarray(endian='little') - b = make_endian(a, 'big') - self.assertTrue(b == a) - self.assertEqual(len(b), 0) - self.assertEqual(b.endian(), 'big') - - def test_from_frozen(self): - a = frozenbitarray('1101111', 'big') - b = make_endian(a, 'big') - self.assertTrue(b is a) - c = make_endian(a, 'little') - self.assertTrue(c == a) - self.assertEqual(c.endian(), 'little') - #self.assertIsType(c, 'frozenbitarray') - - def test_random(self): - for a in self.randombitarrays(): - aa = a.copy() - for endian in 'big', 'little': - b = make_endian(a, endian) - self.assertEqual(a, b) - self.assertEqual(b.endian(), endian) - if a.endian() == endian: - self.assertTrue(b is a) - self.assertEQUAL(a, aa) - -# --------------------------------------------------------------------------- - -class TestsRIndex(unittest.TestCase, Util): - - def test_simple(self): - self.assertRaises(TypeError, rindex) - self.assertRaises(TypeError, rindex, None) - self.assertRaises(ValueError, rindex, bitarray(), 1) - for endian in 'big', 'little': - a = bitarray('00010110 000', endian) - self.assertEqual(rindex(a), 6) - self.assertEqual(rindex(a, 1), 6) - self.assertEqual(rindex(a, 1, 3), 6) - self.assertEqual(rindex(a, 1, 3, 8), 6) - self.assertEqual(rindex(a, 1, -20, 20), 6) - self.assertEqual(rindex(a, 1, 0, 5), 3) - self.assertEqual(rindex(a, 1, 0, -6), 3) - self.assertEqual(rindex(a, 1, 0, -5), 5) - self.assertRaises(TypeError, rindex, a, 'A') - self.assertRaises(ValueError, rindex, a, 2) - self.assertRaises(ValueError, rindex, a, 1, 7) - self.assertRaises(ValueError, rindex, a, 1, 10, 3) - self.assertRaises(ValueError, rindex, a, 1, -1, 0) - self.assertRaises(TypeError, rindex, a, 1, 10, 3, 4) - - a = bitarray('00010110 111', endian) - self.assertEqual(rindex(a, 0), 7) - self.assertEqual(rindex(a, 0, 0, 4), 2) - self.assertEqual(rindex(a, False), 7) - - a = frozenbitarray('00010110 111', endian) - self.assertEqual(rindex(a, 0), 7) - self.assertRaises(TypeError, rindex, a, None) - self.assertRaises(ValueError, rindex, a, 7) - - for v in 0, 1: - self.assertRaises(ValueError, rindex, - bitarray(0, endian), v) - self.assertRaises(ValueError, rindex, - bitarray('000', endian), 1) - self.assertRaises(ValueError, rindex, - bitarray('11111', endian), 0) - - def test_range(self): - n = 100 - a = bitarray(n) - for m in range(n): - a.setall(0) - self.assertRaises(ValueError, rindex, a, 1) - a[m] = 1 - self.assertEqual(rindex(a, 1), m) - - a.setall(1) - self.assertRaises(ValueError, rindex, a, 0) - a[m] = 0 - self.assertEqual(rindex(a, 0), m) - - def test_random(self): - for a in self.randombitarrays(): - v = getrandbits(1) - try: - i = rindex(a, v) - except ValueError: - i = None - s = a.to01() - try: - j = s.rindex(str(v)) - except ValueError: - j = None - self.assertEqual(i, j) - - def test_random_start_stop(self): - for _ in range(10): - n = randrange(1, 1000) - a = zeros(n) - indices = [randrange(n) for _ in range(100)] - a[indices] = 1 - start = randint(0, n) - stop = randint(0, n) - filtered = [i for i in indices if i >= start and i < stop] - ref = max(filtered) if filtered else -1 - try: - res = rindex(a, 1, start, stop) - except ValueError: - res = -1 - self.assertEqual(res, ref) - - def test_many_set(self): - for _ in range(10): - n = randint(1, 10000) - v = getrandbits(1) - a = bitarray(n) - a.setall(not v) - lst = [randrange(n) for _ in range(100)] - a[lst] = v - self.assertEqual(rindex(a, v), max(lst)) - - def test_one_set(self): - for _ in range(10): - N = randint(1, 10000) - a = zeros(N) - a[randrange(N)] = 1 - self.assertEqual(rindex(a), a.index(1)) - -# --------------------------------------------------------------------------- - -class TestsStrip(unittest.TestCase, Util): +class StripTests(unittest.TestCase, Util): def test_simple(self): self.assertRaises(TypeError, strip, '0110') @@ -365,25 +459,26 @@ class TestsStrip(unittest.TestCase, Util b = frozenbitarray('00010110000') c = strip(b, 'both') self.assertEqual(c, bitarray('1011')) - self.assertIsType(c, 'frozenbitarray') + self.assertEqual(type(c), frozenbitarray) def test_zeros_ones(self): - for n in range(10): - for mode in 'left', 'right', 'both': - a = zeros(n) - c = strip(a, mode) - self.assertIsType(c, 'bitarray') - self.assertEqual(c, bitarray()) - self.assertEqual(a, zeros(n)) - - b = frozenbitarray(a) - c = strip(b, mode) - self.assertIsType(c, 'frozenbitarray') - self.assertEqual(c, bitarray()) + for _ in range(50): + n = randrange(10) + mode = choice(['left', 'right', 'both']) + a = zeros(n) + c = strip(a, mode) + self.assertEqual(type(c), bitarray) + self.assertEqual(len(c), 0) + self.assertEqual(a, zeros(n)) - a.setall(1) - c = strip(a, mode) - self.assertEqual(c, ones(n)) + b = frozenbitarray(a) + c = strip(b, mode) + self.assertEqual(type(c), frozenbitarray) + self.assertEqual(len(c), 0) + + a.setall(1) + c = strip(a, mode) + self.assertEqual(c, ones(n)) def test_random(self): for a in self.randombitarrays(): @@ -391,18 +486,18 @@ class TestsStrip(unittest.TestCase, Util f = frozenbitarray(a) s = a.to01() for mode, res in [ - ('left', bitarray(s.lstrip('0'), a.endian())), - ('right', bitarray(s.rstrip('0'), a.endian())), - ('both', bitarray(s.strip('0'), a.endian())), + ('left', bitarray(s.lstrip('0'), a.endian)), + ('right', bitarray(s.rstrip('0'), a.endian)), + ('both', bitarray(s.strip('0'), a.endian)), ]: c = strip(a, mode) self.assertEQUAL(c, res) - self.assertIsType(c, 'bitarray') + self.assertEqual(type(c), bitarray) self.assertEQUAL(a, b) c = strip(f, mode) self.assertEQUAL(c, res) - self.assertIsType(c, 'frozenbitarray') + self.assertEqual(type(c), frozenbitarray) self.assertEQUAL(f, b) def test_one_set(self): @@ -416,7 +511,7 @@ class TestsStrip(unittest.TestCase, Util # --------------------------------------------------------------------------- -class TestsCount_N(unittest.TestCase, Util): +class CountN_Tests(unittest.TestCase, Util): @staticmethod def count_n(a, n): @@ -444,7 +539,7 @@ class TestsCount_N(unittest.TestCase, Ut self.assertRaises(TypeError, count_n, a, 7.0) self.assertRaises(ValueError, count_n, a, 0, 2) self.assertRaisesMessage(ValueError, "n = 1 larger than bitarray " - "size (len(a) = 0)", count_n, a, 1) + "length 0", count_n, a, 1) def test_simple(self): a = bitarray('111110111110111110111110011110111110111110111000') @@ -453,21 +548,10 @@ class TestsCount_N(unittest.TestCase, Ut self.assertEqual(a.count(), 37) self.assertEqual(a.count(0), 11) + self.assertEqual(count_n(a, 0), 0) self.assertEqual(count_n(a, 0, 0), 0) self.assertEqual(count_n(a, 2, 0), 12) self.assertEqual(count_n(a, 10, 0), 47) - # n < 0 - self.assertRaisesMessage(ValueError, "non-negative integer expected", - count_n, a, -1, 0) - # n > len(a) - self.assertRaisesMessage(ValueError, "n = 49 larger than bitarray " - "size (len(a) = 48)", count_n, a, 49, 0) - # n > a.count(0) - self.assertRaisesMessage(ValueError, "n = 12 exceeds total count " - "(a.count(0) = 11)", - count_n, a, 12, 0) - - self.assertEqual(count_n(a, 0), 0) self.assertEqual(count_n(a, 20), 23) self.assertEqual(count_n(a, 20, 1), 23) self.assertEqual(count_n(a, 37), 45) @@ -476,10 +560,13 @@ class TestsCount_N(unittest.TestCase, Ut count_n, a, -1) # n > len(a) self.assertRaisesMessage(ValueError, "n = 49 larger than bitarray " - "size (len(a) = 48)", count_n, a, 49) + "length 48", count_n, a, 49) + # n > a.count(0) + self.assertRaisesMessage(ValueError, "n = 12 exceeds total count " + "(a.count(0) = 11)", count_n, a, 12, 0) # n > a.count(1) self.assertRaisesMessage(ValueError, "n = 38 exceeds total count " - "(a.count(1) = 37)", count_n, a, 38) + "(a.count(1) = 37)", count_n, a, 38, 1) for v in 0, 1: for n in range(a.count(v) + 1): @@ -489,7 +576,7 @@ class TestsCount_N(unittest.TestCase, Ut self.assertEqual(i, self.count_n(a if v else ~a, n)) self.assertEQUAL(a, b) - def test_frozen(self): + def test_frozenbitarray(self): a = frozenbitarray('001111101111101111101111100111100') self.assertEqual(len(a), 33) self.assertEqual(a.count(), 24) @@ -503,7 +590,7 @@ class TestsCount_N(unittest.TestCase, Ut self.check_result(a, n, count_n(a, n)) def test_ones(self): - n = randint(1, 100000) + n = randint(1, 100_000) a = ones(n) self.assertEqual(count_n(a, n), n) self.assertRaises(ValueError, count_n, a, 1, 0) @@ -513,7 +600,7 @@ class TestsCount_N(unittest.TestCase, Ut self.assertEqual(count_n(a, i), i) def test_one_set(self): - n = randint(1, 100000) + n = randint(1, 100_000) a = zeros(n) self.assertEqual(count_n(a, 0), 0) self.assertRaises(ValueError, count_n, a, 1) @@ -528,17 +615,16 @@ class TestsCount_N(unittest.TestCase, Ut for N in range(1, 1000): a = zeros(N) a[-1] = 1 - self.assertEqual(a.count(), 1) self.assertEqual(count_n(a, 1), N) if N == 1: - msg = "n = 2 larger than bitarray size (len(a) = 1)" + msg = "n = 2 larger than bitarray length 1" else: msg = "n = 2 exceeds total count (a.count(1) = 1)" self.assertRaisesMessage(ValueError, msg, count_n, a, 2) def test_large(self): for _ in range(100): - N = randint(100000, 250000) + N = randint(100_000, 250_000) a = bitarray(N) v = getrandbits(1) a.setall(not v) @@ -556,23 +642,13 @@ class TestsCount_N(unittest.TestCase, Ut i = count_n(a, n, v) self.check_result(a, n, i, v) - def test_random(self): - for a in self.randombitarrays(): - for v in 0, 1: - n = a.count(v) // 2 - i = count_n(a, n, v) - self.check_result(a, n, i, v) - # n = 0 -> count_n always 0 - self.assertEqual(count_n(a, 0, v), 0) - # --------------------------------------------------------------------------- -class TestsBitwiseCount(unittest.TestCase, Util): +class BitwiseCountTests(unittest.TestCase, Util): def test_count_byte(self): for i in range(256): - a = bitarray() - a.frombytes(bytes(bytearray([i]))) + a = bitarray(bytearray([i])) cnt = a.count() self.assertEqual(count_and(a, zeros(8)), 0) self.assertEqual(count_and(a, ones(8)), cnt) @@ -621,8 +697,8 @@ class TestsBitwiseCount(unittest.TestCas def test_random(self): for _ in range(100): n = randrange(1000) - a = urandom(n, self.random_endian()) - b = urandom(n, a.endian()) + a = urandom_2(n) + b = urandom(n, a.endian) self.assertEqual(count_and(a, b), (a & b).count()) self.assertEqual(count_or(a, b), (a | b).count()) self.assertEqual(count_xor(a, b), (a ^ b).count()) @@ -630,7 +706,7 @@ class TestsBitwiseCount(unittest.TestCas def test_misc(self): for a in self.randombitarrays(): n = len(a) - b = urandom(n, a.endian()) + b = urandom(n, a.endian) # any and self.assertEqual(any(a & b), count_and(a, b) > 0) self.assertEqual(any_and(a, b), any(a & b)) @@ -652,7 +728,7 @@ class TestsBitwiseCount(unittest.TestCas # --------------------------------------------------------------------------- -class TestsBitwiseAny(unittest.TestCase, Util): +class BitwiseAnyTests(unittest.TestCase, Util): def test_basic(self): a = frozenbitarray('0101') @@ -666,9 +742,41 @@ class TestsBitwiseAny(unittest.TestCase, bitarray('01', 'little'), bitarray('11', 'big')) + def test_overlap(self): + n = 100 + for _ in range(500): + i1 = randint(0, n) + j1 = randint(i1, n) + r1 = range(i1, j1) + + i2 = randint(0, n) + j2 = randint(i2, n) + r2 = range(i2, j2) + + # test if ranges r1 and r2 overlap + res1 = bool(r1) and bool(r2) and (i2 in r1 or i1 in r2) + res2 = bool(set(r1) & set(r2)) + self.assertEqual(res1, res2) + + a1, a2 = bitarray(n), bitarray(n) + a1[i1:j1] = a2[i2:j2] = 1 + self.assertEqual(any_and(a1, a2), res1) + + def test_common(self): + n = 100 + for _ in range(500): + s1 = self.random_slice(n) + s2 = self.random_slice(n) + r1 = range(n)[s1] + r2 = range(n)[s2] + # test if ranges r1 and r2 have common items + a1, a2 = bitarray(n), bitarray(n) + a1[s1] = a2[s2] = 1 + self.assertEqual(any_and(a1, a2), bool(set(r1) & set(r2))) + def check(self, a, b): r = any_and(a, b) - self.assertIsInstance(r, bool) + self.assertEqual(type(r), bool) self.assertEqual(r, any_and(b, a)) # symmetry self.assertEqual(r, any(a & b)) self.assertEqual(r, (a & b).any()) @@ -690,7 +798,7 @@ class TestsBitwiseAny(unittest.TestCase, def test_random(self): for a in self.randombitarrays(): n = len(a) - b = urandom(n, a.endian()) + b = urandom(n, a.endian) self.check(a, b) def test_one(self): @@ -703,7 +811,7 @@ class TestsBitwiseAny(unittest.TestCase, # --------------------------------------------------------------------------- -class TestsSubset(unittest.TestCase, Util): +class SubsetTests(unittest.TestCase, Util): def test_basic(self): a = frozenbitarray('0101') @@ -722,7 +830,7 @@ class TestsSubset(unittest.TestCase, Uti def check(self, a, b, res): r = subset(a, b) - self.assertIsInstance(r, bool) + self.assertEqual(type(r), bool) self.assertEqual(r, res) self.assertEqual(a | b == b, res) self.assertEqual(a & b == a, res) @@ -752,16 +860,16 @@ class TestsSubset(unittest.TestCase, Uti # --------------------------------------------------------------------------- -class TestsCorrespondAll(unittest.TestCase, Util): +class CorrespondAllTests(unittest.TestCase, Util): def test_basic(self): a = frozenbitarray('0101') b = bitarray('0111') - self.assertTrue(_correspond_all(a, b), (1, 1, 1, 1)) - self.assertRaises(TypeError, _correspond_all) + self.assertTrue(correspond_all(a, b), (1, 1, 1, 1)) + self.assertRaises(TypeError, correspond_all) b.append(1) - self.assertRaises(ValueError, _correspond_all, a, b) - self.assertRaises(ValueError, _correspond_all, + self.assertRaises(ValueError, correspond_all, a, b) + self.assertRaises(ValueError, correspond_all, bitarray('01', 'little'), bitarray('11', 'big')) @@ -771,13 +879,13 @@ class TestsCorrespondAll(unittest.TestCa ('0000011111', '0000100111', (4, 1, 2, 3)), ]: - self.assertEqual(_correspond_all(bitarray(a), bitarray(b)), res) + self.assertEqual(correspond_all(bitarray(a), bitarray(b)), res) def test_random(self): for a in self.randombitarrays(): n = len(a) - b = urandom(n, a.endian()) - res = _correspond_all(a, b) + b = urandom(n, a.endian) + res = correspond_all(a, b) self.assertEqual(res[0], count_and(~a, ~b)) self.assertEqual(res[1], count_and(~a, b)) self.assertEqual(res[2], count_and(a, ~b)) @@ -789,36 +897,132 @@ class TestsCorrespondAll(unittest.TestCa # --------------------------------------------------------------------------- -class TestsParity(unittest.TestCase, Util): +@skipIf(is_pypy) +class ByteswapTests(unittest.TestCase, Util): - def test_bitarray(self): - a = bitarray() - self.assertBitEqual(parity(a), 0) - par = False - for _ in range(1000): - self.assertEqual(parity(a), par) - a.append(1) - par = not par + def test_basic_bytearray(self): + a = bytearray(b"ABCD") + byteswap(a, 2) + self.assertEqual(a, bytearray(b"BADC")) + byteswap(a) + self.assertEqual(a, bytearray(b"CDAB")) + + a = bytearray(b"ABCDEF") + byteswap(a, 3) + self.assertEqual(a, bytearray(b"CBAFED")) + byteswap(a, 1) + self.assertEqual(a, bytearray(b"CBAFED")) + + def test_basic_bitarray(self): + a = bitarray("11110000 01010101") + byteswap(a) + self.assertEqual(a, bitarray("01010101 11110000")) - def test_pad_ignored(self): - a = ones(1) - self.assertTrue(parity(a)) + a = bitarray("01111000 1001") + b = a.copy() + a.tobytes() # clear padbits + byteswap(a) + self.assertEqual(a, bitarray("10010000 0111")) + byteswap(a) + self.assertEqual(a, b) + + def test_basic_array(self): + r = os.urandom(64) + for typecode in array.typecodes: + # type code 'u' is deprecated and will be removed in Python 3.16 + if typecode == 'u': + continue + a = array.array(typecode, r) + self.assertEqual(len(a) * a.itemsize, 64) + a.byteswap() + byteswap(a, a.itemsize) + self.assertEqual(a.tobytes(), r) - def test_frozenbitarray(self): - for s, p in [('', 0), ('0010011', 1), ('10100110', 0)]: - self.assertBitEqual(parity(frozenbitarray(s)), p) + def test_empty(self): + a = bytearray() + byteswap(a) + self.assertEqual(a, bytearray()) + for n in range(10): + byteswap(a, n) + self.assertEqual(a, bytearray()) - def test_wrong_args(self): - self.assertRaises(TypeError, parity, '') - self.assertRaises(TypeError, bitarray(), 1) + def test_one_byte(self): + a = bytearray(b'\xab') + byteswap(a) + self.assertEqual(a, bytearray(b'\xab')) + for n in range(2): + byteswap(a, n) + self.assertEqual(a, bytearray(b'\xab')) + + def test_errors(self): + # buffer not writable + for a in b"AB", frozenbitarray(16): + self.assertRaises(BufferError, byteswap, a) + + a = bytearray(b"ABCD") + b = bitarray(32) + for n in -1, 3, 5, 6: + # byte size not multiple of n + self.assertRaises(ValueError, byteswap, a, n) + self.assertRaises(ValueError, byteswap, b, n) - def test_byte(self): - for i in range(256): - a = bitarray() - a.frombytes(bytes(bytearray([i]))) - self.assertEqual(parity(a), a.count() % 2) + def test_range(self): + for n in range(20): + for m in range(20): + r = os.urandom(m * n) + a = bytearray(r) + byteswap(a, n) + lst = [] + for i in range(m): + x = r[i * n:i * n + n] + lst.extend(x[::-1]) + self.assertEqual(a, bytearray(lst)) + + def test_reverse_bytearray(self): + for n in range(100): + r = os.urandom(n) + a = bytearray(r) + byteswap(a) + self.assertEqual(a, bytearray(r[::-1])) + + def test_reverse_bitarray(self): + for n in range(100): + a = urandom(8 * n) + b = a.copy() + byteswap(a) + a.bytereverse() + self.assertEqual(a, b[::-1]) + +# --------------------------------------------------------------------------- + +class ParityTests(unittest.TestCase, Util): + + def test_explitcit(self): + for s, res in [('', 0), ('1', 1), ('0010011', 1), ('10100110', 0)]: + self.assertTrue(parity(bitarray(s)) is res) + self.assertTrue(parity(frozenbitarray(s)) is res) + + def test_zeros_ones(self): + for n in range(2000): + self.assertEqual(parity(zeros(n)), 0) + self.assertEqual(parity(ones(n)), n % 2) def test_random(self): + a = bitarray() + par = 0 + for _ in range(2000): + self.assertEqual(parity(a), par) + v = getrandbits(1) + a.append(v) + par ^= v + + def test_wrong_args(self): + self.assertRaises(TypeError, parity, '') + self.assertRaises(TypeError, parity, 1) + self.assertRaises(TypeError, parity) + self.assertRaises(TypeError, parity, bitarray("110"), 1) + + def test_random2(self): for a in self.randombitarrays(): b = a.copy() self.assertEqual(parity(a), a.count() % 2) @@ -826,67 +1030,241 @@ class TestsParity(unittest.TestCase, Uti # --------------------------------------------------------------------------- -class TestsIntervals(unittest.TestCase, Util): +class SumIndicesUtil(unittest.TestCase): + + def check_explicit(self, S): + for s, r1, r2 in [ + ("", 0, 0), ("0", 0, 0), ("1", 0, 0), ("11", 1, 1), + ("011", 3, 5), ("001", 2, 4), ("0001100", 7, 25), + ("00001111", 22, 126), ("01100111 1101", 49, 381), + ]: + for a in [bitarray(s, choice(['little', 'big'])), + frozenbitarray(s, choice(['little', 'big']))]: + self.assertEqual(S(a, 1), r1) + self.assertEqual(S(a, 2), r2) + self.assertEqual(a, bitarray(s)) + + def check_wrong_args(self, S): + self.assertRaises(TypeError, S, '') + self.assertRaises(TypeError, S, 1.0) + self.assertRaises(TypeError, S) + for mode in -1, 0, 3, 4: + self.assertRaises(ValueError, S, bitarray("110"), mode) + + def check_urandom(self, S, n): + a = urandom_2(n) + self.assertEqual(S(a, 1), sum(i for i, v in enumerate(a) if v)) + self.assertEqual(S(a, 2), sum(i * i for i, v in enumerate(a) if v)) + + def check_sparse(self, S, n, k, mode=1, freeze=False, inv=False): + a = zeros(n, choice(['little', 'big'])) + self.assertEqual(S(a, mode), 0) + self.assertFalse(a.any()) + + indices = sample(range(n), k) + a[indices] = 1 + res = sum(indices) if mode == 1 else sum(i * i for i in indices) + + if inv: + a.invert() + sum_ones = 3 if mode == 1 else 2 * n - 1 + sum_ones *= n * (n - 1) + sum_ones //= 6 + res = sum_ones - res + + if freeze: + a = frozenbitarray(a) + + c = a.copy() + self.assertEqual(a.count(), n - k if inv else k) + self.assertEqual(S(a, mode), res) + self.assertEqual(a, c) + + +class SSQI_Tests(SumIndicesUtil): + + # Additional tests for _ssqi() in: devel/test_sum_indices.py + + def test_explicit(self): + self.check_explicit(_ssqi) + + def test_wrong_args(self): + self.check_wrong_args(_ssqi) + + def test_small(self): + a = bitarray() + sm1 = sm2 = 0 + for i in range(100): + v = getrandbits(1) + a.append(v) + if v: + sm1 += i + sm2 += i * i + self.assertEqual(_ssqi(a, 1), sm1) + self.assertEqual(_ssqi(a, 2), sm2) + + def test_urandom(self): + self.check_urandom(_ssqi, 10_037) + + def test_sparse(self): + for _ in range(5): + mode = randint(1, 2) + freeze = getrandbits(1) + inv = getrandbits(1) + self.check_sparse(_ssqi, n=1_000_003, k=400, + mode=mode, freeze=freeze, inv=inv) + + +class SumIndicesTests(SumIndicesUtil): + + # Additional tests in: devel/test_sum_indices.py + + def test_explicit(self): + self.check_explicit(sum_indices) + + def test_wrong_args(self): + self.check_wrong_args(sum_indices) + + def test_ones(self): + for mode in 1, 2: + self.check_sparse(sum_indices, n=1_600_037, k=0, + mode=mode, freeze=True, inv=True) + + def test_sparse(self): + for _ in range(20): + n = choice([500_029, 600_011]) # below and above block size + k = randrange(1_000) + mode = randint(1, 2) + freeze = getrandbits(1) + inv = getrandbits(1) + self.check_sparse(sum_indices, n, k, mode, freeze, inv) + +# --------------------------------------------------------------------------- + +class XoredIndicesTests(unittest.TestCase, Util): + + def test_explicit(self): + for s, r in [("", 0), ("0", 0), ("1", 0), ("11", 1), + ("011", 3), ("001", 2), ("0001100", 7), + ("01100111 1101", 13)]: + for a in [bitarray(s, self.random_endian()), + frozenbitarray(s, self.random_endian())]: + self.assertEqual(xor_indices(a), r) + + def test_wrong_args(self): + X = xor_indices + self.assertRaises(TypeError, X, '') + self.assertRaises(TypeError, X, 1) + self.assertRaises(TypeError, X) + self.assertRaises(TypeError, X, bitarray("110"), 1) + + def test_ones(self): + # OEIS A003815 + lst = [0, 1, 3, 0, 4, 1, 7, 0, 8, 1, 11, 0, 12, 1, 15, 0, 16, 1, 19] + self.assertEqual([xor_indices(ones(i)) for i in range(1, 20)], lst) + a = bitarray() + x = 0 + for i in range(1000): + a.append(1) + x ^= i + self.assertEqual(xor_indices(a), x) + if i < 19: + self.assertEqual(lst[i], x) + + def test_large_random(self): + n = 10_037 + for a in [urandom_2(n), frozenbitarray(urandom_2(n))]: + res = reduce(operator.xor, (i for i, v in enumerate(a) if v)) + b = a.copy() + self.assertEqual(xor_indices(a), res) + self.assertEqual(a, b) + + def test_random(self): + for a in self.randombitarrays(): + c = 0 + for i, v in enumerate(a): + c ^= i * v + self.assertEqual(xor_indices(a), c) + + def test_flips(self): + a = bitarray(128) + c = 0 + for _ in range(1000): + self.assertEqual(xor_indices(a), c) + i = randrange(len(a)) + a.invert(i) + c ^= i + + def test_error_correct(self): + parity_bits = [1, 2, 4, 8, 16, 32, 64, 128] # parity bit positions + a = urandom(256) + a[parity_bits] = 0 + c = xor_indices(a) + # set parity bits such that block is well prepared + a[parity_bits] = int2ba(c, length=8, endian="little") + for i in range(0, 256): + self.assertEqual(xor_indices(a), 0) # ensure well prepared + a.invert(i) + self.assertEqual(xor_indices(a), i) # index of the flipped bit! + a.invert(i) + +# ------------------ intervals of uninterrupted runs -------------------- + +def runs(a): + "return number of uninterrupted intervals of 1s and 0s" + n = len(a) + if n < 2: + return n + return 1 + count_xor(a[:-1], a[1:]) + +class IntervalsTests(unittest.TestCase, Util): def test_explicit(self): for s, lst in [ ('', []), ('0', [(0, 0, 1)]), ('1', [(1, 0, 1)]), - ('00111100 00000111 00', - [(0, 0, 2), (1, 2, 6), (0, 6, 13), (1, 13, 16), (0, 16, 18)]), + ('00111100 0000011', + [(0, 0, 2), (1, 2, 6), (0, 6, 13), (1, 13, 15)]), ]: a = bitarray(s) self.assertEqual(list(intervals(a)), lst) + self.assertEqual(runs(a), len(lst)) - def test_count(self): - for s, res in [ - ('', 0), - ('0', 1), - ('1', 1), - ('00', 1), - ('01', 2), - ('10', 2), - ('11', 1), - ('0011110000000', 3), - ]: - a = bitarray(s) - self.assertEqual(res, len(list(intervals(a)))) - self.assertEqual(res, sum(1 for _ in intervals(a))) + def test_uniform(self): + for n in range(1, 100): + for v in 0, 1: + a = n * bitarray([v], self.random_endian()) + self.assertEqual(list(intervals(a)), [(v, 0, n)]) + self.assertEqual(runs(a), 1) def test_random(self): for a in self.randombitarrays(): - b = urandom(len(a)) - cnt = [0, 0] - v = a[0] if a else None + n = len(a) + b = urandom(n) for value, start, stop in intervals(a): self.assertFalse(isinstance(value, bool)) - self.assertEqual(value, v) - v = not v - self.assertTrue(0 <= start < stop <= len(a)) - cnt[value] += stop - start + self.assertTrue(0 <= start < stop <= n) b[start:stop] = value self.assertEqual(a, b) - for v in 0, 1: - self.assertEqual(cnt[v], a.count(v)) - def test_runs(self): + def test_list_runs(self): for a in self.randombitarrays(): - first = a[0] if a else None - # list runs of alternating bits - runs = [stop - start for _, start, stop in intervals(a)] + # list of length of runs of alternating bits + alt_runs = [stop - start for _, start, stop in intervals(a)] + self.assertEqual(len(alt_runs), runs(a)) b = bitarray() - v = first - for length in runs: + v = a[0] if a else None # value of first run + for length in alt_runs: + self.assertTrue(length > 0) b.extend(length * bitarray([v])) v = not v - self.assertEqual(a, b) # --------------------------------------------------------------------------- -class TestsHexlify(unittest.TestCase, Util): +class HexlifyTests(unittest.TestCase, Util): def test_ba2hex(self): self.assertEqual(ba2hex(bitarray(0, 'big')), '') @@ -902,50 +1280,62 @@ class TestsHexlify(unittest.TestCase, Ut self.assertRaises(TypeError, ba2hex, '101') c = ba2hex(bitarray('1101', 'big')) - self.assertIsInstance(c, str) + self.assertEqual(type(c), str) - for n in range(7): - a = bitarray(n * '1111', 'big') - b = a.copy() - self.assertEqual(ba2hex(a), n * 'f') - # ensure original object wasn't altered - self.assertEQUAL(a, b) + def test_ba2hex_group(self): + a = bitarray('1000 0000 0101 1111', 'little') + self.assertEqual(ba2hex(a), "10af") + self.assertEqual(ba2hex(a, 0), "10af") + self.assertEqual(ba2hex(a, 1, ""), "10af") + self.assertEqual(ba2hex(a, 1), "1 0 a f") + self.assertEqual(ba2hex(a, group=2), "10 af") + self.assertEqual(ba2hex(a, 2, "-"), "10-af") + self.assertEqual(ba2hex(a, group=3, sep="_"), "10a_f") + self.assertEqual(ba2hex(a, 3, sep=", "), "10a, f") + + def test_ba2hex_errors(self): + a = bitarray('1000 0000 0101 1111', 'little') + self.assertRaises(ValueError, ba2hex, a, -1) + self.assertRaises(ValueError, ba2hex, a, group=-1) + self.assertRaises(TypeError, ba2hex, a, 1, b" ") + # embedded null character in sep + self.assertRaises(ValueError, ba2hex, a, 2, "\0") def test_hex2ba(self): _set_default_endian('big') self.assertEqual(hex2ba(''), bitarray()) - for c in 'e', 'E', b'e', b'E', u'e', u'E': + for c in 'e', 'E', b'e', b'E', bytearray(b'e'), bytearray(b'E'): a = hex2ba(c) self.assertEqual(a.to01(), '1110') - self.assertEqual(a.endian(), 'big') - self.assertIsType(a, 'bitarray') + self.assertEqual(a.endian, 'big') + self.assertEqual(type(a), bitarray) self.assertEQUAL(hex2ba('01'), bitarray('0000 0001', 'big')) self.assertEQUAL(hex2ba('08', 'little'), bitarray('0000 0001', 'little')) self.assertEQUAL(hex2ba('aD'), bitarray('1010 1101', 'big')) - self.assertEQUAL(hex2ba(b'10aF'), + self.assertEQUAL(hex2ba('10aF'), bitarray('0001 0000 1010 1111', 'big')) - self.assertEQUAL(hex2ba(b'10aF', 'little'), + self.assertEQUAL(hex2ba(b'10 aF', 'little'), bitarray('1000 0000 0101 1111', 'little')) + def test_hex2ba_whitespace(self): + _set_default_endian('big') + self.assertEqual(hex2ba("F1 F2 %s f3 c0" % WHITESPACE), + bitarray("11110001 11110010 11110011 11000000")) + self.assertEQUAL(hex2ba(b' a F ', 'big'), + bitarray('1010 1111', 'big')) + self.assertEQUAL(hex2ba(860 * " " + '0 1D' + 590 * " ", 'little'), + bitarray('0000 1000 1011', 'little')) + def test_hex2ba_errors(self): self.assertRaises(TypeError, hex2ba, 0) - for endian in 'little', 'big': - _set_default_endian(endian) - self.assertRaises(ValueError, hex2ba, '01a7g89') - self.assertRaises(ValueError, hex2ba, u'0\u20ac') - - for s in 'g', 'ag', 'aag' 'aaaga', 'ag': - msg = "non-hexadecimal digit found, got 'g' (0x67)" - self.assertRaisesMessage(ValueError, msg, hex2ba, s, endian) - - # check for NUL bytes - for b in b'\0', b'\0f', b'f\0', b'\0ff', b'f\0f', b'ff\0': - msg = "non-hexadecimal digit found, got '\0' (0x00)" - if sys.version_info[0] == 2: - msg = msg.replace("0x00", "0x0") - self.assertRaisesMessage(ValueError, msg, hex2ba, b, endian) + for s in '01a7g89', '0\u20ac', '0 \0', b'\x00': + self.assertRaises(ValueError, hex2ba, s) + + for s in 'g', 'ag', 'aag' 'aaaga', 'ag': + msg = "invalid digit found for base16, got 'g' (0x67)" + self.assertRaisesMessage(ValueError, msg, hex2ba, s, 'big') def test_explicit(self): data = [ # little big @@ -964,43 +1354,86 @@ class TestsHexlify(unittest.TestCase, Ut self.assertEqual(ba2hex(a_be), hex_be) self.assertEqual(ba2hex(a_le), hex_le) + def test_random(self): + for _ in range(100): + a = urandom_2(4 * randrange(100)) + s = ba2hex(a, group=randrange(10), sep=randrange(5) * " ") + b = hex2ba(s, endian=a.endian) + self.assertEQUAL(a, b) + self.check_obj(b) + def test_hexdigits(self): for default_endian in 'big', 'little': _set_default_endian(default_endian) a = hex2ba(hexdigits) self.assertEqual(len(a) % 4, 0) - self.assertEqual(a.endian(), default_endian) - self.assertIsType(a, 'bitarray') + self.assertEqual(a.endian, default_endian) + self.assertEqual(type(a), bitarray) self.check_obj(a) t = ba2hex(a) self.assertEqual(t, hexdigits.lower()) - self.assertIsInstance(t, str) + self.assertEqual(type(t), str) self.assertEQUAL(a, hex2ba(t, default_endian)) def test_binascii(self): a = urandom(80, 'big') s = binascii.hexlify(a.tobytes()).decode() self.assertEqual(ba2hex(a), s) - b = bitarray(endian='big') - b.frombytes(binascii.unhexlify(s)) + b = bitarray(binascii.unhexlify(s), endian='big') self.assertEQUAL(hex2ba(s, 'big'), b) # --------------------------------------------------------------------------- -class TestsBase(unittest.TestCase, Util): +class BaseTests(unittest.TestCase, Util): def test_ba2base(self): - c = ba2base(16, bitarray('1101', 'big')) - self.assertIsInstance(c, str) + s = ba2base(16, bitarray('1101 0100', 'big')) + self.assertEqual(type(s), str) + self.assertEqual(s, 'd4') def test_base2ba(self): _set_default_endian('big') - for c in 'e', 'E', b'e', b'E', u'e', u'E': + for c in 'e', 'E', b'e', b'E': a = base2ba(16, c) self.assertEqual(a.to01(), '1110') - self.assertEqual(a.endian(), 'big') - self.assertIsType(a, 'bitarray') + self.assertEqual(a.endian, 'big') + self.assertEqual(type(a), bitarray) + + def test_base2ba_whitespace(self): + self.assertEqual(base2ba(8, bytearray(b"17 0"), "little"), + bitarray("100 111 000")) + self.assertEqual(base2ba(32, "7 A"), bitarray("11111 00000")) + self.assertEqual(base2ba(64, b"A /"), bitarray("000000 111111")) + for n in 2, 4, 8, 16, 32, 64: + a = base2ba(n, WHITESPACE) + self.assertEqual(a, bitarray()) + a = urandom(60) + c = list(ba2base(n, a)) + for _ in range(randrange(80)): + c.insert(randint(0, len(c)), choice(WHITESPACE)) + s = ''.join(c) + self.assertEqual(base2ba(n, s), a) + + def test_ba2base_group(self): + a = bitarray("001 011 100 111", "little") + self.assertEqual(ba2base(8, a, 3), "461 7") + self.assertEqual(ba2base(8, a, group=2), "46 17") + self.assertEqual(ba2base(8, a, sep="_", group=2), "46_17") + self.assertEqual(ba2base(8, a, 2, sep="."), "46.17") + for n, s, group, sep, res in [ + (2, '10100', 2, '-', '10-10-0'), + (4, '10 11 00 01', 1, "_", "2_3_0_1"), + (8, "101 100 011 101 001 010", 3, " ", "543 512"), + (8, "101 100 011 101 001 010", 3, "", "543512"), + (16, '1011 0001 1101 1010 1111', 4, "+", "b1da+f"), + (32, "10110 00111 01101 01111", 2, ", ", "WH, NP"), + (64, "101100 011101 101011 111110 101110", 2, ".", "sd.r+.u"), + ]: + a = bitarray(s, "big") + s = ba2base(n, a, group, sep) + self.assertEqual(type(s), str) + self.assertEqual(s, res) def test_explicit(self): data = [ # n little big @@ -1026,26 +1459,26 @@ class TestsBase(unittest.TestCase, Util) self.assertEqual(a, bitarray()) self.assertEqual(ba2base(n, a), '') - def test_upper(self): - self.assertEqual(base2ba(16, 'F'), bitarray('1111')) - def test_invalid_characters(self): for n, s in ((2, '2'), (4, '4'), (8, '8'), (16, 'g'), (32, '8'), (32, '1'), (32, 'a'), (64, '-'), (64, '_')): - if n == 16: - msg = "non-hexadecimal digit found, got 'g' (0x67)" - else: - msg = ("invalid digit found for base %d, " - "got '%s' (0x%02x)" % (n, s, ord(s))) + msg = ("invalid digit found for base%d, " + "got '%s' (0x%02x)" % (n, s, ord(s))) self.assertRaisesMessage(ValueError, msg, base2ba, n, s) + for n in 2, 4, 8, 16, 32, 64: + for s in '_', '@', '[', '\u20ac', '\0', b'\0', b'\x80', b'\xff': + self.assertRaises(ValueError, base2ba, n, s) + msg = "invalid digit found for base%d, got '{' (0x7b)" % n + self.assertRaisesMessage(ValueError, msg, base2ba, n, '{') + def test_invalid_args(self): a = bitarray() self.assertRaises(TypeError, ba2base, None, a) self.assertRaises(TypeError, base2ba, None, '') self.assertRaises(TypeError, ba2base, 16.0, a) self.assertRaises(TypeError, base2ba, 16.0, '') - for i in range(-10, 260): + for i in range(-10, 100): if i in (2, 4, 8, 16, 32, 64): continue self.assertRaises(ValueError, ba2base, i, a) @@ -1054,10 +1487,6 @@ class TestsBase(unittest.TestCase, Util) self.assertRaises(TypeError, ba2base, 32, None) self.assertRaises(TypeError, base2ba, 32, None) - for i in 2, 4, 8, 16, 32, 64: - self.assertRaises(ValueError, base2ba, i, 60 * u'\u20ac') - self.assertRaises(ValueError, base2ba, i, 60 * b'\0') - def test_binary(self): a = base2ba(2, '1011') self.assertEqual(a, bitarray('1011')) @@ -1066,7 +1495,7 @@ class TestsBase(unittest.TestCase, Util) for a in self.randombitarrays(): s = ba2base(2, a) self.assertEqual(s, a.to01()) - self.assertEQUAL(base2ba(2, s, a.endian()), a) + self.assertEQUAL(base2ba(2, s, a.endian), a) def test_quaternary(self): a = base2ba(4, '0123', 'big') @@ -1085,10 +1514,10 @@ class TestsBase(unittest.TestCase, Util) for n in range(50): s = ''.join(choice(hexdigits) for _ in range(n)) - for endian in 'big', 'little': - a = base2ba(16, s, endian) - self.assertEQUAL(a, hex2ba(s, endian)) - self.assertEqual(ba2base(16, a), ba2hex(a)) + endian = self.random_endian() + a = base2ba(16, s, endian) + self.assertEQUAL(a, hex2ba(s, endian)) + self.assertEqual(ba2base(16, a), ba2hex(a)) def test_base32(self): a = base2ba(32, '7SH', 'big') @@ -1112,47 +1541,48 @@ class TestsBase(unittest.TestCase, Util) self.assertEqual(a.tobytes(), msg) self.assertEqual(ba2base(64, a), s) + alphabets = [ + # m n alphabet + (1, 2, '01'), + (2, 4, '0123'), + (3, 8, '01234567'), + (4, 16, '0123456789abcdef'), + (5, 32, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'), + (6, 64, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef' + 'ghijklmnopqrstuvwxyz0123456789+/'), + ] + def test_alphabets(self): - for m, n, alpabet in [ - (1, 2, '01'), - (2, 4, '0123'), - (3, 8, '01234567'), - (4, 16, '0123456789abcdef'), - (5, 32, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'), - (6, 64, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - 'abcdefghijklmnopqrstuvwxyz0123456789+/'), - ]: + for m, n, alphabet in self.alphabets: self.assertEqual(1 << m, n) - self.assertEqual(len(alpabet), n) - for i, c in enumerate(alpabet): - for endian in 'big', 'little': - self.assertEqual(ba2int(base2ba(n, c, endian)), i) - self.assertEqual(ba2base(n, int2ba(i, m, endian)), c) + self.assertEqual(len(alphabet), n) + for i, c in enumerate(alphabet): + endian = self.random_endian() + self.assertEqual(ba2int(base2ba(n, c, endian)), i) + self.assertEqual(ba2base(n, int2ba(i, m, endian)), c) + + def test_not_alphabets(self): + for m, n, alphabet in self.alphabets: + for i in range(256): + c = chr(i) + if c in alphabet or c.isspace(): + continue + if n == 16 and c in "ABCDEF": + continue + self.assertRaises(ValueError, base2ba, n, c) def test_random(self): - for a in self.randombitarrays(): - for m in range(1, 7): - n = 1 << m - if len(a) % m == 0: - s = ba2base(n, a) - b = base2ba(n, s, a.endian()) - self.assertEQUAL(a, b) - self.check_obj(b) - else: - self.assertRaises(ValueError, ba2base, n, a) - - def test_random2(self): - for m in range(1, 7): + for _ in range(100): + m = randint(1, 6) + a = urandom_2(m * randrange(100)) + self.assertEqual(len(a) % m, 0) n = 1 << m - for length in range(0, 100, m): - a = urandom(length, 'little') - s = ba2base(n, a) - self.assertIsInstance(s, str) - self.assertEQUAL(base2ba(n, s, 'little'), a) - b = bitarray(a, 'big') - self.assertEQUAL(base2ba(n, ba2base(n, b), 'big'), b) + s = ba2base(n, a, group=randrange(10), sep=randrange(5) * " ") + b = base2ba(n, s, a.endian) + self.assertEQUAL(a, b) + self.check_obj(b) -# --------------------------------------------------------------------------- +# --------------------------- sparse compression ---------------------------- class SC_Tests(unittest.TestCase, Util): @@ -1160,15 +1590,43 @@ class SC_Tests(unittest.TestCase, Util): for b, bits, endian in [ (b'\x00\0', '', 'little'), (b'\x01\x03\x01\x03\0', '110', 'little'), + (b'\x01\x07\x01\x40\0', '0000001', 'little'), (b'\x11\x07\x01\x02\0', '0000001', 'big'), (b'\x01\x10\x02\xf0\x0f\0', '00001111 11110000', 'little'), (b'\x11\x10\xa1\x0c\0', '00000000 00001000', 'big'), (b'\x11\x09\xa1\x08\0', '00000000 1', 'big'), - (b'\x01E\xa3ABD\0', 65 * '0' + '1101', 'little'), + (b'\x01g\xa4abde\0', 97 * '0' + '110110', 'little'), ]: a = bitarray(bits, endian) self.assertEqual(sc_encode(a), b) - self.assertEqual(sc_decode(b), a) + self.assertEQUAL(sc_decode(b), a) + + def test_encode_types(self): + for a in bitarray('1', 'big'), frozenbitarray('1', 'big'): + b = sc_encode(a) + self.assertEqual(type(b), bytes) + self.assertEqual(b, b'\x11\x01\x01\x80\0') + + for a in None, [], 0, 123, b'', b'\x00', 3.14: + self.assertRaises(TypeError, sc_encode, a) + + def test_decode_types(self): + blob = b'\x11\x03\x01\x20\0' + for b in blob, bytearray(blob), list(blob), array.array('B', blob): + a = sc_decode(b) + self.assertEqual(type(a), bitarray) + self.assertEqual(a.endian, 'big') + self.assertEqual(a.to01(), '001') + + a = [17, 3, 1, 32, 0] + self.assertEqual(sc_decode(a), bitarray("001")) + for x in 256, -1: + a[-1] = x + self.assertRaises(ValueError, sc_decode, a) + + self.assertRaises(TypeError, sc_decode, [0x02, None]) + for x in None, 3, 3.2, Ellipsis, 'foo': + self.assertRaises(TypeError, sc_decode, x) def test_decode_header_nbits(self): for b, n in [ @@ -1184,7 +1642,6 @@ class SC_Tests(unittest.TestCase, Util): self.assertEqual(len(a), n) self.assertFalse(a.any()) - @skipIf(sys.version_info[0] == 2) def test_decode_untouch(self): stream = iter(b'\x01\x03\x01\x03\0XYZ') self.assertEqual(sc_decode(stream), bitarray('110')) @@ -1195,34 +1652,30 @@ class SC_Tests(unittest.TestCase, Util): self.assertTrue(next(stream) is None) self.assertEqual(next(stream), 'foo') - @skipIf(sys.version_info[0] == 2) def test_decode_header_errors(self): # invalid header for c in 0x20, 0x21, 0x40, 0x80, 0xc0, 0xf0, 0xff: self.assertRaisesMessage(ValueError, "invalid header: 0x%02x" % c, - sc_decode, - bytearray([c])) + sc_decode, [c]) # invalid block head for c in 0xc0, 0xc1, 0xc5, 0xff: self.assertRaisesMessage(ValueError, "invalid block head: 0x%02x" % c, - sc_decode, - bytearray([0x01, 0x10, c])) + sc_decode, [0x01, 0x10, c]) def test_decode_header_overflow(self): - nbytes = SYSINFO[1] self.assertRaisesMessage( OverflowError, - "sizeof(Py_ssize_t) = %d: cannot read 9 bytes" % nbytes, + "sizeof(Py_ssize_t) = %d: cannot read 9 bytes" % PTRSIZE, sc_decode, b'\x09' + 9 * b'\x00') self.assertRaisesMessage( ValueError, - "read %d bytes got negative value: -1" % nbytes, - sc_decode, bytes(bytearray([nbytes] + nbytes * [0xff]))) + "read %d bytes got negative value: -1" % PTRSIZE, + sc_decode, [PTRSIZE] + PTRSIZE * [0xff]) - if nbytes == 4: + if PTRSIZE == 4: self.assertRaisesMessage( OverflowError, "sizeof(Py_ssize_t) = 4: cannot read 5 bytes", @@ -1241,6 +1694,7 @@ class SC_Tests(unittest.TestCase, Util): self.assertRaisesMessage( ValueError, "decode error (raw): 32 + 3 > 34", sc_decode, b"\x02\x0f\x01\xa0\x03\xff\xff\xff\0") + # sparse index too high self.assertRaisesMessage( ValueError, "decode error (n=1): 128 >= 128", @@ -1252,79 +1706,105 @@ class SC_Tests(unittest.TestCase, Util): ValueError, "decode error (n=3): 32768 >= 32768", sc_decode, b"\x02\x00\x80\xc3\x01\x00\x80\x00\0") - if SYSINFO[1] == 4: - msg = "read 4 bytes got negative value: -2147483648" - else: - msg = "decode error (n=4): 2147483648 >= 16" + msg = {4: "read 4 bytes got negative value: -2147483648", + 8: "decode error (n=4): 2147483648 >= 16"} self.assertRaisesMessage( - ValueError, msg, + ValueError, msg[PTRSIZE], sc_decode, b"\x01\x10\xc4\x01\x00\x00\x00\x80\0") - if SYSINFO[1] == 4: - msg = "read 4 bytes got negative value: -1" - else: - msg = "decode error (n=4): 4294967295 >= 16" + msg = {4: "read 4 bytes got negative value: -1", + 8: "decode error (n=4): 4294967295 >= 16"} self.assertRaisesMessage( - ValueError, msg, + ValueError, msg[PTRSIZE], sc_decode, b"\x01\x10\xc4\x01\xff\xff\xff\xff\0") def test_decode_end_of_stream(self): for stream in [b'', b'\x00', b'\x01', b'\x02\x77', b'\x01\x04\x01', b'\x01\x04\xa1', b'\x01\x04\xa0']: - self.assertRaisesMessage(ValueError, "unexpected end of stream", - sc_decode, stream) - - @skipIf(sys.version_info[0] == 2) - def test_decode_types(self): - blob = b'\x11\x03\x01\x20\0' - for b in blob, bytearray(blob), list(blob), array('B', blob): - a = sc_decode(b) - self.assertIsType(a, 'bitarray') - self.assertEqual(a.endian(), 'big') - self.assertEqual(a.to01(), '001') - - self.assertRaises(TypeError, sc_decode, [0x02, None]) - for x in None, 3, 3.2, Ellipsis: - self.assertRaises(TypeError, sc_decode, x) - for _ in range(10): - self.assertRaises(TypeError, sc_decode, [0x00, None]) + self.assertRaises(StopIteration, sc_decode, stream) def test_decode_ambiguity(self): for b in [ # raw: b'\x11\x03\x01\x20\0', # this is what sc_encode gives us - b'\x11\x03\x01\x2f\0', # some pad bits are 1 + b'\x11\x03\x01\x3f\0', # but we can set the pad bits to 1 # sparse: - b'\x11\x03\xa1\x02\0', # using block type 1 - b'\x11\x03\xc2\x01\x02\x00\0', # using block type 2 - b'\x11\x03\xc3\x01\x02\x00\x00\0', # using block type 3 - b'\x11\x03\xc4\x01\x02\x00\x00\x00\0', # using block type 4 + b'\x11\x03\xa1\x02\0', # block type 1 + b'\x11\x03\xc2\x01\x02\x00\0', # block type 2 + b'\x11\x03\xc3\x01\x02\x00\x00\0', # block type 3 + b'\x11\x03\xc4\x01\x02\x00\x00\x00\0', # block type 4 ]: a = sc_decode(b) self.assertEqual(a.to01(), '001') - @skipIf(sys.version_info[0] == 2) - def test_sparse_block_type1(self): + def test_block_type0(self): + for k in range(0x01, 0xa0): + nbytes = k if k <= 32 else 32 * (k - 31) + nbits = 8 * nbytes + a = ones(nbits, "little") + b = bytearray([0x01, nbits] if nbits < 256 else + [0x02, nbits % 256, nbits // 256]) + b.append(k) + b.extend(a.tobytes()) + b.append(0) # stop byte + + self.assertEqual(sc_decode(b), a) + self.assertEqual(sc_encode(a), b) + + def test_block_type1(self): a = bitarray(256, 'little') for n in range(1, 32): - positions = os.urandom(n) - b = bytearray([0x02, 0x00, 0x01, 0xa0 + n]) - b.extend(positions) + a[getrandbits(8)] = 1 + + b = bytearray([0x02, 0x00, 0x01, 0xa0 + a.count()]) + b.extend(list(a.search(1))) # sorted indices with no duplicates b.append(0) # stop byte - a.setall(0) - a[positions] = 1 self.assertEqual(sc_decode(b), a) + self.assertEqual(sc_encode(a), b) - # in order to recreate the block sc_encode generates, we need - # a sorted list of the positions with no duplicates - lst = sorted(set(positions)) - b = bytearray([0x02, 0x00, 0x01, 0xa0 + len(lst)]) - b.extend(lst) - b.append(0) # stop - + def test_block_type2(self): + a = bitarray(65536, 'little') + for n in range(1, 256): + a[getrandbits(16)] = 1 + + b = bytearray([0x03, 0x00, 0x00, 0x01, 0xc2, a.count()]) + for i in a.search(1): + b.extend(struct.pack("= 2 * 4) + indices = sorted(set(randrange(len(a)) for _ in range(5))) + a[indices] = 1 + b = bytearray(b'\x04\x00\x00\x00\x04\xc4') + b.append(len(indices)) + for i in indices: + b.extend(struct.pack(" 512) # second block head and second index byte - m += bool(n > 65536) # third index byte - self.assertEqual(m, len(sc_encode(a))) - self.round_trip(a) - - a = zeros(1 << 25, 'big') - a[0] = 1 - self.assertEqual( - sc_encode(a), - b'\x14\x00\x00\x00\x02\xc4\x01\x00\x00\x00\x00\x00') + m += 2 * bool(i > 9) # count byte and second index byte + m += bool(i > 16) # third index byte + m += bool(i > 24) # fourth index byte + self.check_blob_length(a, m) def test_encode_ones(self): - for _ in range(50): - nbits = randrange(100000) + for _ in range(10): + nbits = randrange(100_000) a = ones(nbits) m = 2 # head byte and stop byte m += bits2bytes(nbits.bit_length()) # size bytes nbytes = bits2bytes(nbits) - m += (nbytes // 32 + 127) // 128 # number of blocks (head bytes) - m += bool(nbytes % 32) # block type 0 range(1, 32) m += nbytes # actual raw bytes - self.assertEqual(m, len(sc_encode(a))) - self.round_trip(a) + # number of head bytes, all of block type 0: + m += bool(nbytes % 32) # number in 0x01 .. 0x1f + m += (nbytes // 32 + 127) // 128 # number in 0x20 .. 0xbf + self.check_blob_length(a, m) + + def round_trip(self, a): + c = a.copy() + i = iter(sc_encode(a)) + b = sc_decode(i) + self.assertTrue(a == b == c) + self.assertTrue(a.endian == b.endian == c.endian) + self.assertEqual(list(i), []) def test_random(self): for _ in range(10): - n = randrange(100000) + n = randrange(100_000) endian = self.random_endian() a = ones(n, endian) - for _ in range(16): + while a.count(): a &= urandom(n, endian) self.round_trip(a) - @skipIf(not DEBUG) - def test_rts_empty(self): - rts = _sc_rts(bitarray()) - self.assertEqual(len(rts), 1) - self.assertEqual(rts, [0]) - - @skipIf(not DEBUG or _SEGSIZE != 32) - def test_rts_example(self): - # see example before sc_calc_rts() in _util.c - a = zeros(987) - a[:5] = a[512:515] = a[768:772] = 1 - self.assertEqual(a.count(), 12) - rts = _sc_rts(a) - self.assertEqual(len(rts), 5) - self.assertEqual(rts, [0, 5, 5, 8, 12]) - - @skipIf(not DEBUG) - def test_rts_ones(self): - for _ in range(20): - n = randrange(10000) - a = ones(n) - rts = _sc_rts(a) - self.assertEqual(rts[0], 0) - self.assertEqual(rts[-1], n) - for i, v in enumerate(rts): - self.assertEqual(v, min(8 * _SEGSIZE * i, n)) - - @skipIf(not DEBUG) - def test_rts_random(self): - segbits = 8 * _SEGSIZE - for _ in range(20): - n = randrange(10000) - a = urandom(n) - rts = _sc_rts(a) - self.assertEqual(len(rts), (n + segbits - 1) // segbits + 1) - self.assertEqual(rts[0], 0) - self.assertEqual(rts[-1], a.count()) - for i in range(len(rts) - 1): - seg_pop = a.count(1, segbits * i, segbits * (i + 1)) - self.assertEqual(rts[i + 1] - rts[i], seg_pop) - # --------------------------------------------------------------------------- class VLFTests(unittest.TestCase, Util): def test_explicit(self): - for s, bits in [ + for blob, s in [ (b'\x40', ''), (b'\x30', '0'), (b'\x38', '1'), (b'\x00', '0000'), (b'\x01', '0001'), + (b'\xd3\x20', '001101'), (b'\xe0\x40', '0000 1'), (b'\x90\x02', '0000 000001'), (b'\xb5\xa7\x18', '0101 0100111 0011'), + (b'\x95\xb7\x1c', '0101 0110111 001110'), ]: - a = bitarray(bits) - self.assertEqual(vl_encode(a), s) - self.assertEqual(vl_decode(s), a) + default_endian = self.random_endian() + _set_default_endian(default_endian) - def test_encode(self): - for endian in 'big', 'little': - s = vl_encode(bitarray('001101', endian)) - self.assertIsInstance(s, bytes) - self.assertEqual(s, b'\xd3\x20') + a = bitarray(s) + self.assertEqual(vl_encode(a), blob) + c = vl_decode(blob) + self.assertEqual(c, a) + self.assertEqual(c.endian, default_endian) + + for endian in 'big', 'little', None: + a = bitarray(s, endian) + c = vl_encode(a) + self.assertEqual(type(c), bytes) + self.assertEqual(c, blob) + + c = vl_decode(blob, endian) + self.assertEqual(c, a) + self.assertEqual(c.endian, endian or default_endian) - def test_decode_args(self): - if sys.version_info[0] == 3: - self.assertRaises(TypeError, vl_decode, 'foo') - # item not integer - self.assertRaises(TypeError, vl_decode, iter([b'\x40'])) + def test_encode_types(self): + s = "0011 01" + for a in bitarray(s), frozenbitarray(s): + b = vl_encode(a) + self.assertEqual(type(b), bytes) + self.assertEqual(b, b'\xd3\x20') + + for a in None, [], 0, 123, b'', b'\x00', 3.14: + self.assertRaises(TypeError, vl_encode, a) + + def test_decode_types(self): + blob = b'\xd3\x20' + for s in (blob, iter(blob), memoryview(blob), iter([0xd3, 0x20]), + bytearray(blob)): + a = vl_decode(s, endian=self.random_endian()) + self.assertEqual(type(a), bitarray) + self.assertEqual(a, bitarray('0011 01')) - self.assertRaises(TypeError, vl_decode, b'\x40', 'big', 3) - self.assertRaises(ValueError, vl_decode, b'\x40', 'foo') # these objects are not iterable for arg in None, 0, 1, 0.0: self.assertRaises(TypeError, vl_decode, arg) # these items cannot be interpreted as ints - for item in None, 2.34, Ellipsis: + for item in None, 2.34, Ellipsis, 'foo': self.assertRaises(TypeError, vl_decode, iter([0x95, item])) - b = b'\xd3\x20' - lst = [b, iter(b), memoryview(b)] - if sys.version_info[0] == 3: - lst.append(iter([0xd3, 0x20])) - lst.append(bytearray(b)) - for s in lst: - a = vl_decode(s, endian=self.random_endian()) - self.assertIsType(a, 'bitarray') - self.assertEqual(a, bitarray('0011 01')) - - def test_decode_endian(self): - blob = b'\xd3\x20' - res = bitarray('0011 01') - - for default_endian in 'little', 'big': - _set_default_endian(default_endian) + def test_decode_args(self): + # item not integer + self.assertRaises(TypeError, vl_decode, iter([b'\x40'])) - for endian in 'little', 'big', None: - a = vl_decode(blob, endian) - self.assertEqual(a, res) - self.assertEqual(a.endian(), - endian if endian else default_endian) - - a = vl_decode(blob) - self.assertEqual(a, res) - self.assertEqual(a.endian(), default_endian) + self.assertRaises(TypeError, vl_decode, b'\x40', 'big', 3) + self.assertRaises(ValueError, vl_decode, b'\x40', 'foo') def test_decode_trailing(self): for s, bits in [(b'\x40ABC', ''), (b'\xe0\x40A', '00001')]: stream = iter(s) self.assertEqual(vl_decode(stream), bitarray(bits)) - self.assertEqual(next(stream), - b'A' if sys.version_info[0] == 2 else 65) + self.assertEqual(next(stream), 65) def test_decode_ambiguity(self): for s in b'\x40', b'\x4f', b'\x45': - self.assertEqual(vl_decode(iter(s)), bitarray()) + self.assertEqual(vl_decode(s), bitarray()) for s in b'\x1e', b'\x1f': - self.assertEqual(vl_decode(iter(s)), bitarray('111')) + self.assertEqual(vl_decode(s), bitarray('111')) def test_decode_stream(self): stream = iter(b'\x40\x30\x38\x40\x2c\xe0\x40\xd3\x20') @@ -1535,17 +1960,25 @@ class VLFTests(unittest.TestCase, Util): self.assertEqual(vl_decode(stream), a) def test_decode_errors(self): - # decode empty bits - self.assertRaises(ValueError, vl_decode, b'') - # invalid number of padding bits - for s in b'\x50', b'\x60', b'\x70': - self.assertRaises(ValueError, vl_decode, s) - self.assertRaises(ValueError, vl_decode, b'\xf0') + # decode empty bytes + self.assertRaises(StopIteration, vl_decode, b'') + # invalid head byte + for s in [ + b'\x70', b'\xf0', # padding = 7 + b'\x50', b'\x60', b'\x70', # no second byte, but padding > 4 + ]: + self.assertRaisesMessage(ValueError, + "invalid head byte: 0x%02x" % s[0], + vl_decode, s) # high bit set, but no terminating byte for s in b'\x80', b'\x80\x80': - self.assertRaises(ValueError, vl_decode, s) + self.assertRaises(StopIteration, vl_decode, s) + # decode list with out of range items + for i in -1, 256: + self.assertRaises(ValueError, vl_decode, [i]) + # wrong type + self.assertRaises(TypeError, vl_decode, [None]) - @skipIf(sys.version_info[0] == 2) def test_decode_invalid_stream(self): N = 100 s = iter(N * (3 * [0x80] + ['XX']) + ['end.']) @@ -1569,21 +2002,19 @@ class VLFTests(unittest.TestCase, Util): c = a.copy() s = vl_encode(a) b = vl_decode(s) + self.check_obj(b) self.assertTrue(a == b == c) LEN_PAD_BITS = 3 self.assertEqual(len(s), (len(a) + LEN_PAD_BITS + 6) // 7) - head = ord(s[0]) if sys.version_info[0] == 2 else s[0] + head = s[0] padding = (head & 0x70) >> 4 self.assertEqual(len(a) + padding, 7 * len(s) - LEN_PAD_BITS) - def test_range(self): - for n in range(500): - self.round_trip(urandom(n)) - def test_large(self): - a = urandom(randint(50000, 100000)) - self.round_trip(a) + for _ in range(10): + a = urandom(randrange(100_000)) + self.round_trip(a) def test_random(self): for a in self.randombitarrays(): @@ -1591,7 +2022,7 @@ class VLFTests(unittest.TestCase, Util): # --------------------------------------------------------------------------- -class TestsIntegerization(unittest.TestCase, Util): +class IntegerizationTests(unittest.TestCase, Util): def test_ba2int(self): self.assertEqual(ba2int(bitarray('0')), 0) @@ -1622,10 +2053,10 @@ class TestsIntegerization(unittest.TestC def test_ba2int_bytes(self): for n in range(1, 50): - a = urandom(8 * n, self.random_endian()) + a = urandom_2(8 * n) c = bytearray(a.tobytes()) i = 0 - for x in (c if a.endian() == 'big' else reversed(c)): + for x in (c if a.endian == 'big' else reversed(c)): i <<= 8 i |= x self.assertEqual(ba2int(a), i) @@ -1669,13 +2100,6 @@ class TestsIntegerization(unittest.TestC ('11111111 0', 255), ('00000000 1', -256), ('11111111 1', -1), - ('00000000 00000000 000000', 0), - ('10010000 11000000 100010', 9 + 3 * 256 + 17 * 2 ** 16), - ('11111111 11111111 111110', 2 ** 21 - 1), - ('00000000 00000000 000001', -2 ** 21), - ('10010000 11000000 100011', -2 ** 21 - + (9 + 3 * 256 + 17 * 2 ** 16)), - ('11111111 11111111 111111', -1), ]: self.assertEqual(ba2int(bitarray(s, 'little'), signed=1), i) self.assertEqual(ba2int(bitarray(s[::-1], 'big'), signed=1), i) @@ -1686,6 +2110,25 @@ class TestsIntegerization(unittest.TestC self.assertEQUAL(int2ba(i, len_s, 'big', signed=1), bitarray(s[::-1], 'big')) + def test_zero(self): + for endian in "little", "big": + a = int2ba(0, endian=endian) + self.assertEQUAL(a, bitarray('0', endian=endian)) + for n in range(1, 100): + a = int2ba(0, length=n, endian=endian, signed=True) + b = bitarray(n * '0', endian) + self.assertEQUAL(a, b) + for signed in 0, 1: + self.assertEqual(ba2int(b, signed=signed), 0) + + def test_negative_one(self): + for endian in "little", "big": + for n in range(1, 100): + a = int2ba(-1, length=n, endian=endian, signed=True) + b = bitarray(n * '1', endian) + self.assertEQUAL(a, b) + self.assertEqual(ba2int(b, signed=True), -1) + def test_int2ba_overflow(self): self.assertRaises(OverflowError, int2ba, -1) self.assertRaises(OverflowError, int2ba, -1, 4) @@ -1695,10 +2138,10 @@ class TestsIntegerization(unittest.TestC self.assertRaises(OverflowError, int2ba, -65, 7, signed=1) for n in range(1, 20): - self.assertRaises(OverflowError, int2ba, 2 ** n, n) - self.assertRaises(OverflowError, int2ba, 2 ** (n - 1), n, + self.assertRaises(OverflowError, int2ba, 1 << n, n) + self.assertRaises(OverflowError, int2ba, 1 << (n - 1), n, signed=1) - self.assertRaises(OverflowError, int2ba, -2 ** (n - 1) - 1, n, + self.assertRaises(OverflowError, int2ba, -(1 << (n - 1)) - 1, n, signed=1) def test_int2ba_length(self): @@ -1709,8 +2152,8 @@ class TestsIntegerization(unittest.TestC for n in range(1, 100): ab = int2ba(1, n, 'big') al = int2ba(1, n, 'little') - self.assertEqual(ab.endian(), 'big') - self.assertEqual(al.endian(), 'little') + self.assertEqual(ab.endian, 'big') + self.assertEqual(al.endian, 'little') self.assertEqual(len(ab), n), self.assertEqual(len(al), n) self.assertEqual(ab, bitarray((n - 1) * '0') + bitarray('1')) @@ -1726,9 +2169,6 @@ class TestsIntegerization(unittest.TestC self.assertEqual(int2ba(2 ** n - 1), bitarray(n * '1')) self.assertEqual(int2ba(2 ** n - 1, endian='little'), bitarray(n * '1')) - for endian in 'big', 'little': - self.assertEqual(int2ba(-1, n, endian, signed=True), - bitarray(n * '1')) def test_explicit(self): _set_default_endian('big') @@ -1747,7 +2187,7 @@ class TestsIntegerization(unittest.TestC for endian in 'big', 'little': a = int2ba(i, endian=endian) self.check_obj(a) - self.assertEqual(a.endian(), endian) + self.assertEqual(a.endian, endian) self.assertTrue(len(a) > 0) # ensure we have no leading zeros if a.endian == 'big': @@ -1760,12 +2200,11 @@ class TestsIntegerization(unittest.TestC a = zeros(randrange(4), endian) + a else: a = a + zeros(randrange(4), endian) - self.assertEqual(a.endian(), endian) + self.assertEqual(a.endian, endian) self.assertEqual(ba2int(a), i) def test_many(self): - for i in range(20): - self.check_round_trip(i) + for _ in range(20): self.check_round_trip(randrange(10 ** randint(3, 300))) @staticmethod @@ -1777,7 +2216,7 @@ class TestsIntegerization(unittest.TestC def test_random_signed(self): for a in self.randombitarrays(start=1): i = ba2int(a, signed=True) - b = int2ba(i, len(a), a.endian(), signed=True) + b = int2ba(i, len(a), a.endian, signed=True) self.assertEQUAL(a, b) j = ba2int(a, signed=False) # unsigned @@ -1791,7 +2230,8 @@ class TestsIntegerization(unittest.TestC class MixedTests(unittest.TestCase, Util): def test_bin(self): - for i in range(100): + for _ in range(20): + i = randrange(1000) s = bin(i) self.assertEqual(s[:2], '0b') a = bitarray(s[2:], 'big') @@ -1800,9 +2240,9 @@ class MixedTests(unittest.TestCase, Util self.assertEqual(t, s) self.assertEqual(eval(t), i) - @skipIf(sys.version_info[0] == 2) def test_oct(self): - for i in range(1000): + for _ in range(20): + i = randrange(1000) s = oct(i) self.assertEqual(s[:2], '0o') a = base2ba(8, s[2:], 'big') @@ -1812,7 +2252,8 @@ class MixedTests(unittest.TestCase, Util self.assertEqual(eval(t), i) def test_hex(self): - for i in range(1000): + for _ in range(20): + i = randrange(1000) s = hex(i) self.assertEqual(s[:2], '0x') a = hex2ba(s[2:], 'big') @@ -1823,7 +2264,7 @@ class MixedTests(unittest.TestCase, Util def test_bitwise(self): for a in self.randombitarrays(start=1): - b = urandom(len(a), a.endian()) + b = urandom(len(a), a.endian) aa = a.copy() bb = b.copy() i = ba2int(a) @@ -1833,7 +2274,7 @@ class MixedTests(unittest.TestCase, Util self.assertEqual(ba2int(a ^ b), i ^ j) n = randint(0, len(a)) - if a.endian() == 'big': + if a.endian == 'big': self.assertEqual(ba2int(a >> n), i >> n) c = zeros(len(a), 'big') + a self.assertEqual(ba2int(c << n), i << n) @@ -1843,7 +2284,7 @@ class MixedTests(unittest.TestCase, Util def test_bitwise_inplace(self): for a in self.randombitarrays(start=1): - b = urandom(len(a), a.endian()) + b = urandom(len(a), a.endian) bb = b.copy() i = ba2int(a) j = ba2int(b) @@ -1859,7 +2300,7 @@ class MixedTests(unittest.TestCase, Util self.assertEQUAL(b, bb) n = randint(0, len(a)) - if a.endian() == 'big': + if a.endian == 'big': c = a.copy() c >>= n self.assertEqual(ba2int(c), i >> n) @@ -1867,25 +2308,9 @@ class MixedTests(unittest.TestCase, Util c <<= n self.assertEqual(ba2int(c), i << n) - def test_primes(self): # Sieve of Eratosthenes - sieve = ones(10000) - sieve[:2] = 0 # zero and one are not prime - for i in range(2, 100): - if sieve[i]: - sieve[i * i::i] = 0 - # the first 15 primes - self.assertEqual(sieve.search(1, 15), [2, 3, 5, 7, 11, 13, 17, 19, - 23, 29, 31, 37, 41, 43, 47]) - # there are 1229 primes between 1 and 10000 - self.assertEqual(sieve.count(1), 1229) - # there are 119 primes between 4000 and 5000 - self.assertEqual(sieve.count(1, 4000, 5000), 119) - # the 1000th prime is 7919 - self.assertEqual(count_n(sieve, 1000) - 1, 7919) - # --------------------------------------------------------------------------- -class TestsSerialization(unittest.TestCase, Util): +class SerializationTests(unittest.TestCase, Util): def test_explicit(self): for blob, endian, bits in [ @@ -1900,12 +2325,12 @@ class TestsSerialization(unittest.TestCa a = bitarray(bits, endian) s = serialize(a) self.assertEqual(blob, s) - self.assertIsInstance(s, bytes) + self.assertEqual(type(s), bytes) b = deserialize(blob) self.assertEqual(b, a) - self.assertEqual(b.endian(), endian) - self.assertIsType(b, 'bitarray') + self.assertEqual(b.endian, endian) + self.assertEqual(type(b), bitarray) def test_serialize_args(self): for x in '0', 0, 1, b'\x00', 0.0, [0, 1], bytearray([0]): @@ -1919,7 +2344,7 @@ class TestsSerialization(unittest.TestCa self.assertEqual(serialize(a), b'\x14\x70') def test_deserialize_args(self): - for x in 0, 1, False, True, None, u'', u'01', 0.0, [0, 1]: + for x in 0, 1, False, True, None, '', '01', 0.0, [0, 1]: self.assertRaises(TypeError, deserialize, x) # no arguments self.assertRaises(TypeError, deserialize) @@ -1927,40 +2352,35 @@ class TestsSerialization(unittest.TestCa self.assertRaises(TypeError, deserialize, b'\x00', 1) blob = b'\x03\x06' - x = bitarray() # we can deserialize a bitarray as it has a buffer - x.frombytes(blob) + x = bitarray(blob) for s in blob, bytearray(blob), memoryview(blob), x: a = deserialize(s) self.assertEqual(a.to01(), '01100') - self.assertEqual(a.endian(), 'little') + self.assertEqual(a.endian, 'little') def test_invalid_bytes(self): self.assertRaises(ValueError, deserialize, b'') def check_msg(b): - # Python 2: PyErr_Format() seems to handle "0x%02x" - # incorrectly. E.g. instead of "0x01", I get "0x1" - if sys.version_info[0] == 3: - msg = "invalid header byte: 0x%02x" % b[0] - self.assertRaisesMessage(ValueError, msg, deserialize, b) + msg = "invalid header byte: 0x%02x" % b[0] + self.assertRaisesMessage(ValueError, msg, deserialize, b) for i in range(256): - b = bytes(bytearray([i])) + b = bytearray([i]) if i == 0 or i == 16: self.assertEqual(deserialize(b), bitarray()) else: self.assertRaises(ValueError, deserialize, b) check_msg(b) - b += b'\0' + b.append(0) if i < 32 and i % 16 < 8: self.assertEqual(deserialize(b), zeros(8 - i % 8)) else: self.assertRaises(ValueError, deserialize, b) check_msg(b) - def test_bits_ignored(self): - # the unused padding bits (with the last bytes) are ignored + def test_padbits_ignored(self): for blob, endian in [ (b'\x07\x01', 'little'), (b'\x07\x03', 'little'), @@ -1971,19 +2391,43 @@ class TestsSerialization(unittest.TestCa ]: a = deserialize(blob) self.assertEqual(a.to01(), '1') - self.assertEqual(a.endian(), endian) + self.assertEqual(a.endian, endian) def test_random(self): for a in self.randombitarrays(): b = serialize(a) c = deserialize(b) self.assertEqual(a, c) - self.assertEqual(a.endian(), c.endian()) + self.assertEqual(a.endian, c.endian) self.check_obj(c) # --------------------------------------------------------------------------- -class TestsHuffman(unittest.TestCase): +class HuffmanTreeTests(unittest.TestCase): # tests for _huffman_tree() + + def test_empty(self): + freq = {} + self.assertRaises(IndexError, _huffman_tree, freq) + + def test_one_symbol(self): + freq = {"A": 1} + tree = _huffman_tree(freq) + self.assertEqual(tree.symbol, "A") + self.assertEqual(tree.freq, 1) + self.assertRaises(AttributeError, getattr, tree, 'child') + + def test_two_symbols(self): + freq = {"A": 1, "B": 1} + tree = _huffman_tree(freq) + self.assertRaises(AttributeError, getattr, tree, 'symbol') + self.assertEqual(tree.freq, 2) + self.assertEqual(tree.child[0].symbol, "A") + self.assertEqual(tree.child[0].freq, 1) + self.assertEqual(tree.child[1].symbol, "B") + self.assertEqual(tree.child[1].freq, 1) + + +class HuffmanTests(unittest.TestCase): def test_simple(self): freq = {0: 10, 'as': 2, None: 1.6} @@ -1999,7 +2443,7 @@ class TestsHuffman(unittest.TestCase): code = huffman_code(freq, endian) self.assertEqual(len(code), 3) for v in code.values(): - self.assertEqual(v.endian(), endian) + self.assertEqual(v.endian, endian) def test_wrong_arg(self): self.assertRaises(TypeError, huffman_code, [('a', 1)]) @@ -2019,10 +2463,9 @@ class TestsHuffman(unittest.TestCase): a = bitarray() a.encode(code, msg) self.assertEqual(a.to01(), n * '0') - self.assertEqual(a.decode(code), msg) + self.assertEqual(list(a.decode(code)), msg) a.append(1) - self.assertRaises(ValueError, a.decode, code) - self.assertRaises(ValueError, list, a.iterdecode(code)) + self.assertRaises(ValueError, list, a.decode(code)) def check_tree(self, code): n = len(code) @@ -2036,22 +2479,22 @@ class TestsHuffman(unittest.TestCase): def test_balanced(self): n = 6 freq = {} - for i in range(2 ** n): + for i in range(1 << n): freq[i] = 1 code = huffman_code(freq) - self.assertEqual(len(code), 2 ** n) + self.assertEqual(len(code), 1 << n) self.assertTrue(all(len(v) == n for v in code.values())) self.check_tree(code) def test_unbalanced(self): - N = 27 + n = 27 freq = {} - for i in range(N): - freq[i] = 2 ** i + for i in range(n): + freq[i] = 1 << i code = huffman_code(freq) - self.assertEqual(len(code), N) - for i in range(N): - self.assertEqual(len(code[i]), N - (1 if i <= 1 else i)) + self.assertEqual(len(code), n) + for i in range(n): + self.assertEqual(len(code[i]), n - max(1, i)) self.check_tree(code) def test_counter(self): @@ -2067,30 +2510,45 @@ class TestsHuffman(unittest.TestCase): code = huffman_code(Counter(plain)) a = bitarray() a.encode(code, plain) - self.assertEqual(a.decode(code), plain) + self.assertEqual(list(a.decode(code)), plain) self.check_tree(code) def test_random_freq(self): - for n in 2, 3, 5, randint(50, 200): + for n in 2, 3, 4, randint(5, 200): # create Huffman code for n symbols code = huffman_code({i: random() for i in range(n)}) self.check_tree(code) # --------------------------------------------------------------------------- -class TestsCanonicalHuffman(unittest.TestCase, Util): +class CanonicalHuffmanTests(unittest.TestCase, Util): def test_basic(self): plain = bytearray(b'the quick brown fox jumps over the lazy dog.') chc, count, symbol = canonical_huffman(Counter(plain)) - self.assertIsInstance(chc, dict) - self.assertIsInstance(count, list) - self.assertIsInstance(symbol, list) + self.assertEqual(type(chc), dict) + self.assertEqual(type(count), list) + self.assertEqual(type(symbol), list) a = bitarray() a.encode(chc, plain) - self.assertEqual(bytearray(a.iterdecode(chc)), plain) + self.assertEqual(bytearray(a.decode(chc)), plain) self.assertEqual(bytearray(canonical_decode(a, count, symbol)), plain) + def test_example(self): + cnt = {'a': 5, 'b': 3, 'c': 1, 'd': 1, 'r': 2} + codedict, count, symbol = canonical_huffman(cnt) + self.assertEqual(codedict, {'a': bitarray('0'), + 'b': bitarray('10'), + 'c': bitarray('1110'), + 'd': bitarray('1111'), + 'r': bitarray('110')}) + self.assertEqual(count, [0, 1, 1, 1, 2]) + self.assertEqual(symbol, ['a', 'b', 'r', 'c', 'd']) + a = bitarray('01011001110011110101100') + msg = "abracadabra" + self.assertEqual(''.join(a.decode(codedict)), msg) + self.assertEqual(''.join(canonical_decode(a, count, symbol)), msg) + def test_canonical_huffman_errors(self): self.assertRaises(TypeError, canonical_huffman, []) # frequency map cannot be empty @@ -2126,10 +2584,6 @@ class TestsCanonicalHuffman(unittest.Tes self.assertRaises(TypeError, canonical_decode, a, [0, 1.0], s) # count element overflow self.assertRaises(OverflowError, canonical_decode, a, [0, 1 << 65], s) - # negative count - self.assertRaises(ValueError, canonical_decode, a, [0, -1], s) - # count list too long - self.assertRaises(ValueError, canonical_decode, a, 32 * [0], s) # symbol not sequence self.assertRaises(TypeError, canonical_decode, a, [0, 1], 43) @@ -2138,10 +2592,38 @@ class TestsCanonicalHuffman(unittest.Tes self.assertRaisesMessage(ValueError, "sum(count) = 3, but len(symbol) = 4", canonical_decode, a, [0, 1, 2], symbol) - # count[i] > 1 << i + # count list too long self.assertRaisesMessage(ValueError, - "count[2] cannot be negative or larger than 4, got 5", - canonical_decode, a, [0, 2, 5], symbol) + "len(count) cannot be larger than 32", + canonical_decode, a, 33 * [0], symbol) + + def test_canonical_decode_count_range(self): + a = bitarray() + for i in range(1, 32): + count = 32 * [0] + # negative count + count[i] = -1 + self.assertRaisesMessage(ValueError, + "count[%d] not in [0..%d], got -1" % (i, 1 << i), + canonical_decode, a, count, []) + + maxbits = 1 << i + count[i] = maxbits + if i == 31 and PTRSIZE == 4: + self.assertRaises(OverflowError, + canonical_decode, a, count, []) + continue + self.assertRaisesMessage(ValueError, + "sum(count) = %d, but len(symbol) = 0" % maxbits, + canonical_decode, a, count, []) + + count[i] = maxbits + 1 + self.assertRaisesMessage(ValueError, + "count[%d] not in [0..%d], got %d" % (i, maxbits, count[i]), + canonical_decode, a, count, []) + + iter = canonical_decode(a, 32 * [0], []) + self.assertEqual(list(iter), []) def test_canonical_decode_simple(self): # symbols can be anything, they do not even have to be hashable here @@ -2159,15 +2641,14 @@ class TestsCanonicalHuffman(unittest.Tes # the element count[0] is unused self.assertEqual(list(canonical_decode(a, [-47, 0, 4], s)), s) # in fact it can be anything, as it is entirely ignored - self.assertEqual(list(canonical_decode(a, [s, 0, 4], s)), s) + self.assertEqual(list(canonical_decode(a, [None, 0, 4], s)), s) # the symbol argument can be any sequence object s = [65, 66, 67, 98] self.assertEqual(list(canonical_decode(a, cnt, s)), s) self.assertEqual(list(canonical_decode(a, cnt, bytearray(s))), s) self.assertEqual(list(canonical_decode(a, cnt, tuple(s))), s) - if sys.version_info[0] == 3: - self.assertEqual(list(canonical_decode(a, cnt, bytes(s))), s) + self.assertEqual(list(canonical_decode(a, cnt, bytes(s))), s) # Implementation Note: # The symbol can even be an iterable. This was done because we # want to use PySequence_Fast in order to convert sequence @@ -2240,31 +2721,32 @@ class TestsCanonicalHuffman(unittest.Tes self.assertTrue(ba2int(a) < ba2int(b)) def ensure_consecutive(self, chc, count, symbol): - first = 0 + start = 0 for nbits, cnt in enumerate(count): - for i in range(first, first + cnt - 1): + for i in range(start, start + cnt - 1): # ensure two consecutive codes (with same bit length) have # consecutive integer values a = chc[symbol[i]] b = chc[symbol[i + 1]] self.assertTrue(len(a) == len(b) == nbits) self.assertEqual(ba2int(a) + 1, ba2int(b)) - first += cnt + start += cnt def ensure_count(self, chc, count): # ensure count list corresponds to length counts from codedict - maxbits = max(len(a) for a in chc.values()) + maxbits = len(count) - 1 + self.assertEqual(maxbits, max(len(a) for a in chc.values())) my_count = (maxbits + 1) * [0] for a in chc.values(): - self.assertEqual(a.endian(), 'big') + self.assertEqual(a.endian, 'big') my_count[len(a)] += 1 - self.assertEqual(my_count, list(count)) + self.assertEqual(my_count, count) def ensure_complete(self, count): # ensure code is complete and not oversubscribed - maxbits = len(count) - x = sum(count[i] << (maxbits - i) for i in range(1, maxbits)) - self.assertEqual(x, 1 << maxbits) + len_c = len(count) + x = sum(count[i] << (len_c - i) for i in range(1, len_c)) + self.assertEqual(x, 1 << len_c) def ensure_complete_2(self, chc): # ensure code is complete @@ -2301,35 +2783,35 @@ class TestsCanonicalHuffman(unittest.Tes def test_simple_counter(self): plain = bytearray(b'the quick brown fox jumps over the lazy dog.') cnt = Counter(plain) - code, count, symbol = canonical_huffman(cnt) - self.check_code(code, count, symbol) - self.check_code(code, tuple(count), tuple(symbol)) - self.check_code(code, bytearray(count), symbol) - self.check_code(code, count, bytearray(symbol)) + self.check_code(*canonical_huffman(cnt)) + + def test_no_comp(self): + freq = {None: 1, "A": 1} # None and "A" are not comparable + self.check_code(*canonical_huffman(freq)) def test_balanced(self): n = 7 freq = {} - for i in range(2 ** n): + for i in range(1 << n): freq[i] = 1 code, count, sym = canonical_huffman(freq) - self.assertEqual(len(code), 2 ** n) + self.assertEqual(len(code), 1 << n) self.assertTrue(all(len(v) == n for v in code.values())) self.check_code(code, count, sym) def test_unbalanced(self): - n = 29 + n = 32 freq = {} for i in range(n): - freq[i] = 2 ** i + freq[i] = 1 << i code = canonical_huffman(freq)[0] self.assertEqual(len(code), n) for i in range(n): - self.assertEqual(len(code[i]), n - (1 if i <= 1 else i)) + self.assertEqual(len(code[i]), n - max(1, i)) self.check_code(*canonical_huffman(freq)) def test_random_freq(self): - for n in 2, 3, 5, randint(50, 200): + for n in 2, 3, 4, randint(5, 200): freq = {i: random() for i in range(n)} self.check_code(*canonical_huffman(freq)) diff -pruN 2.9.2-1/bitarray/util.py 3.6.1-1/bitarray/util.py --- 2.9.2-1/bitarray/util.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/util.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright (c) 2019 - 2024, Ilan Schnell; All Rights Reserved +# Copyright (c) 2019 - 2025, Ilan Schnell; All Rights Reserved # bitarray is published under the PSF license. # # Author: Ilan Schnell @@ -9,13 +9,15 @@ from __future__ import absolute_import import os import sys +import math +import random from bitarray import bitarray, bits2bytes from bitarray._util import ( - zeros, ones, count_n, parity, + zeros, ones, count_n, parity, _ssqi, xor_indices, count_and, count_or, count_xor, any_and, subset, - _correspond_all, + correspond_all, byteswap, serialize, deserialize, ba2hex, hex2ba, ba2base, base2ba, @@ -25,10 +27,11 @@ from bitarray._util import ( ) __all__ = [ - 'zeros', 'ones', 'urandom', - 'pprint', 'make_endian', 'rindex', 'strip', 'count_n', - 'parity', 'count_and', 'count_or', 'count_xor', 'any_and', 'subset', - 'intervals', + 'zeros', 'ones', 'urandom', 'random_k', 'random_p', + 'pprint', 'strip', 'count_n', + 'parity', 'sum_indices', 'xor_indices', + 'count_and', 'count_or', 'count_xor', 'any_and', 'subset', + 'correspond_all', 'byteswap', 'intervals', 'ba2hex', 'hex2ba', 'ba2base', 'base2ba', 'ba2int', 'int2ba', @@ -39,38 +42,256 @@ __all__ = [ ] -_is_py2 = bool(sys.version_info[0] == 2) - - def urandom(__length, endian=None): - """urandom(length, /, endian=None) -> bitarray + """urandom(n, /, endian=None) -> bitarray -Return a bitarray of `length` random bits (uses `os.urandom`). +Return random bitarray of length `n` (uses `os.urandom()`). """ - a = bitarray(0, endian) - a.frombytes(os.urandom(bits2bytes(__length))) + a = bitarray(os.urandom(bits2bytes(__length)), endian) del a[__length:] return a -def rindex(__a, __sub_bitarray=1, __start=0, __stop=sys.maxsize): - """rindex(bitarray, sub_bitarray=1, start=0, stop=, /) -> int +def random_k(__n, k, endian=None): + """random_k(n, /, k, endian=None) -> bitarray -Return rightmost (highest) index where sub_bitarray (or item - defaults -to 1) is found in bitarray (`a`), such that sub_bitarray is contained -within `a[start:stop]`. -Raises `ValueError` when the sub_bitarray is not present. -""" - from warnings import warn - - warn("rindex() is deprecated and will be removed in bitarray 3.0 - " - "use .index(..., right=True) method instead.", - DeprecationWarning, stacklevel=1) +Return (pseudo-) random bitarray of length `n` with `k` elements +set to one. Mathematically equivalent to setting (in a bitarray of +length `n`) all bits at indices `random.sample(range(n), k)` to one. +The random bitarrays are reproducible when giving Python's `random.seed()` +with a specific seed value. + +This function requires Python 3.9 or higher, as it depends on the standard +library function `random.randbytes()`. Raises `NotImplementedError` +when Python version is too low. +""" + if sys.version_info[:2] < (3, 9): + raise NotImplementedError("bitarray.util.random_k() requires " + "Python 3.9 or higher") + r = _Random(__n, endian) + if not isinstance(k, int): + raise TypeError("int expected, got '%s'" % type(k).__name__) + + return r.random_k(k) + + +def random_p(__n, p=0.5, endian=None): + """random_p(n, /, p=0.5, endian=None) -> bitarray + +Return (pseudo-) random bitarray of length `n`, where each bit has +probability `p` of being one (independent of any other bits). Mathematically +equivalent to `bitarray((random() < p for _ in range(n)), endian)`, but much +faster for large `n`. The random bitarrays are reproducible when giving +Python's `random.seed()` with a specific seed value. + +This function requires Python 3.12 or higher, as it depends on the standard +library function `random.binomialvariate()`. Raises `NotImplementedError` +when Python version is too low. +""" + if sys.version_info[:2] < (3, 12): + raise NotImplementedError("bitarray.util.random_p() requires " + "Python 3.12 or higher") + r = _Random(__n, endian) + return r.random_p(p) + +class _Random: + + # The main reason for this class it to enable testing functionality + # individually in the test class Random_P_Tests in 'test_util.py'. + # The test class also contains many comments and explanations. + # To better understand how the algorithm works, see ./doc/random_p.rst + # See also, VerificationTests in devel/test_random.py + + # maximal number of calls to .random_half() in .combine() + M = 8 + + # number of resulting probability intervals + K = 1 << M + + # limit for setting individual bits randomly + SMALL_P = 0.01 + + def __init__(self, n=0, endian=None): + self.n = n + self.nbytes = bits2bytes(n) + self.endian = endian - if not isinstance(__a, bitarray): - raise TypeError("bitarray expected, got '%s'" % type(__a).__name__) + def random_half(self): + """ + Return bitarray with each bit having probability p = 1/2 of being 1. + """ + # use randbytes() for reproducibility (not urandom()) + a = bitarray(random.randbytes(self.nbytes), self.endian) + del a[self.n:] + return a - return __a.index(__sub_bitarray, __start, __stop, right=True) + def op_seq(self, i): + """ + Return bitarray containing operator sequence. + Each item represents a bitwise operation: 0: AND 1: OR + After applying the sequence (see .combine_half()), we + obtain a bitarray with probability q = i / K + """ + if not 0 < i < self.K: + raise ValueError("0 < i < %d, got i = %d" % (self.K, i)) + + # sequence of &, | operations - least significant operations first + a = bitarray(i.to_bytes(2, byteorder="little"), "little") + return a[a.index(1) + 1 : self.M] + + def combine_half(self, seq): + """ + Combine random bitarrays with probability 1/2 + according to given operator sequence. + """ + a = self.random_half() + for k in seq: + if k: + a |= self.random_half() + else: + a &= self.random_half() + return a + + def random_k(self, k): + n = self.n + # error check inputs and handle edge cases + if k <= 0 or k >= n: + if k == 0: + return zeros(n, self.endian) + if k == n: + return ones(n, self.endian) + raise ValueError("k must be in range 0 <= k <= n, got %s" % k) + + # exploit symmetry to establish: k <= n // 2 + if k > n // 2: + a = self.random_k(n - k) + a.invert() # use in-place to avoid copying + return a + + # decide on sequence, see VerificationTests devel/test_random.py + if k < 16 or k * self.K < 3 * n: + i = 0 + else: + p = k / n # p <= 0.5 + p -= (0.2 - 0.4 * p) / math.sqrt(n) + i = int(p * (self.K + 1)) + + # combine random bitarrays using bitwise AND and OR operations + if i < 3: + a = zeros(n, self.endian) + diff = -k + else: + a = self.combine_half(self.op_seq(i)) + diff = a.count() - k + + randrange = random.randrange + if diff < 0: # not enough bits 1 - increase count + for _ in range(-diff): + i = randrange(n) + while a[i]: + i = randrange(n) + a[i] = 1 + elif diff > 0: # too many bits 1 - decrease count + for _ in range(diff): + i = randrange(n) + while not a[i]: + i = randrange(n) + a[i] = 0 + + return a + + def random_p(self, p): + # error check inputs and handle edge cases + if p <= 0.0 or p == 0.5 or p >= 1.0: + if p == 0.0: + return zeros(self.n, self.endian) + if p == 0.5: + return self.random_half() + if p == 1.0: + return ones(self.n, self.endian) + raise ValueError("p must be in range 0.0 <= p <= 1.0, got %s" % p) + + # for small n, use literal definition + if self.n < 16: + return bitarray((random.random() < p for _ in range(self.n)), + self.endian) + + # exploit symmetry to establish: p < 0.5 + if p > 0.5: + a = self.random_p(1.0 - p) + a.invert() # use in-place to avoid copying + return a + + # for small p, set randomly individual bits + if p < self.SMALL_P: + return self.random_k(random.binomialvariate(self.n, p)) + + # calculate operator sequence + i = int(p * self.K) + if p * (self.K + 1) > i + 1: # see devel/test_random.py + i += 1 + seq = self.op_seq(i) + q = i / self.K + + # when n is small compared to number of operations, also use literal + if self.n < 100 and self.nbytes <= len(seq) + 3 * bool(q != p): + return bitarray((random.random() < p for _ in range(self.n)), + self.endian) + + # combine random bitarrays using bitwise AND and OR operations + a = self.combine_half(seq) + if q < p: + x = (p - q) / (1.0 - q) + a |= self.random_p(x) + elif q > p: + x = p / q + a &= self.random_p(x) + + return a + + +def sum_indices(__a, __mode=1): + """sum_indices(a, /) -> int + +Return sum of indices of all active bits in bitarray `a`. +Equivalent to `sum(i for i, v in enumerate(a) if v)`. +""" + if __mode not in (1, 2): + raise ValueError("unexpected mode %r" % __mode) + + # For details see: devel/test_sum_indices.py + n = 1 << 19 # block size 512 Kbits + if len(__a) <= n: # shortcut for single block + return _ssqi(__a, __mode) + + # Constants + m = n // 8 # block size in bytes + o1 = n * (n - 1) // 2 + o2 = o1 * (2 * n - 1) // 3 + + nblocks = (len(__a) + n - 1) // n + padbits = __a.padbits + sm = 0 + for i in range(nblocks): + # use memoryview to avoid copying memory + v = memoryview(__a)[i * m : (i + 1) * m] + block = bitarray(None, __a.endian, buffer=v) + if padbits and i == nblocks - 1: + if block.readonly: + block = bitarray(block) + block[-padbits:] = 0 + + k = block.count() + if k: + y = n * i + z1 = o1 if k == n else _ssqi(block) + if __mode == 1: + sm += k * y + z1 + else: + z2 = o2 if k == n else _ssqi(block, 2) + sm += (k * y + 2 * z1) * y + z2 + + return sm def pprint(__a, stream=None, group=8, indent=4, width=80): @@ -129,30 +350,6 @@ function `pprint.pprint()`. stream.flush() -def make_endian(__a, endian): - """make_endian(bitarray, /, endian) -> bitarray - -When the endianness of the given bitarray is different from `endian`, -return a new bitarray, with endianness `endian` and the same elements -as the original bitarray. -Otherwise (endianness is already `endian`) the original bitarray is returned -unchanged. -""" - from warnings import warn - - warn("make_endian() is deprecated and will be removed in bitarray 3.0 - " - "use bitarray(..., endian=...) instead", - DeprecationWarning, stacklevel=1) - - if not isinstance(__a, bitarray): - raise TypeError("bitarray expected, got '%s'" % type(__a).__name__) - - if __a.endian() == endian: - return __a - - return bitarray(__a, endian) - - def strip(__a, mode='right'): """strip(bitarray, /, mode='right') -> bitarray @@ -160,7 +357,8 @@ Return a new bitarray with zeros strippe Allowed values for mode are the strings: `left`, `right`, `both` """ if not isinstance(mode, str): - raise TypeError("str expected for mode, got '%s'" % type(__a).__name__) + raise TypeError("str expected for mode, got '%s'" % + type(__a).__name__) if mode not in ('left', 'right', 'both'): raise ValueError("mode must be 'left', 'right' or 'both', got %r" % mode) @@ -210,20 +408,13 @@ The bit-endianness of the bitarray is re if length == 0: raise ValueError("non-empty bitarray expected") - le = bool(__a.endian() == 'little') if __a.padbits: - pad = zeros(__a.padbits, __a.endian()) - __a = __a + pad if le else pad + __a + pad = zeros(__a.padbits, __a.endian) + __a = __a + pad if __a.endian == "little" else pad + __a - if _is_py2: - a = bitarray(__a, 'big') - if le: - a.reverse() - res = int(ba2hex(a), 16) - else: # py3 - res = int.from_bytes(__a.tobytes(), byteorder=__a.endian()) + res = int.from_bytes(__a.tobytes(), byteorder=__a.endian) - if signed and res >= 1 << (length - 1): + if signed and res >> length - 1: res -= 1 << length return res @@ -231,63 +422,49 @@ The bit-endianness of the bitarray is re def int2ba(__i, length=None, endian=None, signed=False): """int2ba(int, /, length=None, endian=None, signed=False) -> bitarray -Convert the given integer to a bitarray (with given endianness, +Convert the given integer to a bitarray (with given bit-endianness, and no leading (big-endian) / trailing (little-endian) zeros), unless the `length` of the bitarray is provided. An `OverflowError` is raised if the integer is not representable with the given number of bits. `signed` determines whether two's complement is used to represent the integer, and requires `length` to be provided. """ - if not isinstance(__i, (int, long) if _is_py2 else int): + if not isinstance(__i, int): raise TypeError("int expected, got '%s'" % type(__i).__name__) if length is not None: if not isinstance(length, int): - raise TypeError("int expected for length") + raise TypeError("int expected for argument 'length'") if length <= 0: raise ValueError("length must be > 0") - if signed and length is None: - raise TypeError("signed requires length") - - if __i == 0: - # there are special cases for 0 which we'd rather not deal with below - return zeros(length or 1, endian) if signed: - m = 1 << (length - 1) + if length is None: + raise TypeError("signed requires argument 'length'") + m = 1 << length - 1 if not (-m <= __i < m): raise OverflowError("signed integer not in range(%d, %d), " "got %d" % (-m, m, __i)) if __i < 0: __i += 1 << length else: # unsigned - if __i < 0: - raise OverflowError("unsigned integer not positive, got %d" % __i) - if length and __i >= (1 << length): + if length and __i >> length: raise OverflowError("unsigned integer not in range(0, %d), " "got %d" % (1 << length, __i)) a = bitarray(0, endian) - le = bool(a.endian() == 'little') - if _is_py2: - s = hex(__i)[2:].rstrip('L') - a.extend(hex2ba(s, 'big')) - if le: - a.reverse() - else: # py3 - b = __i.to_bytes(bits2bytes(__i.bit_length()), byteorder=a.endian()) - a.frombytes(b) - + b = __i.to_bytes(bits2bytes(__i.bit_length()), byteorder=a.endian) + a.frombytes(b) + le = a.endian == 'little' if length is None: - return strip(a, 'right' if le else 'left') + return strip(a, 'right' if le else 'left') if a else a + '0' - la = len(a) - if la > length: - a = a[:length] if le else a[-length:] - if la < length: - pad = zeros(length - la, a.endian()) - a = a + pad if le else pad + a - assert len(a) == length - return a + if len(a) > length: + return a[:length] if le else a[-length:] + if len(a) == length: + return a + # len(a) < length, we need padding + pad = zeros(length - len(a), a.endian) + return a + pad if le else pad + a # ------------------------------ Huffman coding ----------------------------- @@ -301,9 +478,9 @@ and return its root node. class Node(object): """ - A Node instance will either have a 'symbol' (leaf node) or - a 'child' (a tuple with both children) attribute. - The 'freq' attribute will always be present. + There are to tyes of Node instances (both have 'freq' attribute): + * leaf node: has 'symbol' attribute + * parent node: has 'child' attribute (tuple with both children) """ def __lt__(self, other): # heapq needs to be able to compare the nodes @@ -320,7 +497,7 @@ and return its root node. # repeat the process until only one node remains while len(minheap) > 1: # take the two nodes with lowest frequencies from the queue - # to construct a new node and push it onto the queue + # to construct a new parent node and push it onto the queue parent = Node() parent.child = heappop(minheap), heappop(minheap) parent.freq = parent.child[0].freq + parent.child[1].freq @@ -335,15 +512,12 @@ def huffman_code(__freq_map, endian=None Given a frequency map, a dictionary mapping symbols to their frequency, calculate the Huffman code, i.e. a dict mapping those symbols to -bitarrays (with given endianness). Note that the symbols are not limited -to being strings. Symbols may may be any hashable object (such as `None`). +bitarrays (with given bit-endianness). Note that the symbols are not limited +to being strings. Symbols may be any hashable object. """ if not isinstance(__freq_map, dict): raise TypeError("dict expected, got '%s'" % type(__freq_map).__name__) - b0 = bitarray('0', endian) - b1 = bitarray('1', endian) - if len(__freq_map) < 2: if len(__freq_map) == 0: raise ValueError("cannot create Huffman code with no symbols") @@ -353,16 +527,17 @@ to being strings. Symbols may may be an # So we represent the symbol by a single code of length one, in # particular one 0 bit. This is an incomplete code, since if a 1 bit # is received, it has no meaning and will result in an error. - return {list(__freq_map)[0]: b0} + sym = list(__freq_map)[0] + return {sym: bitarray('0', endian)} result = {} def traverse(nd, prefix=bitarray(0, endian)): try: # leaf result[nd.symbol] = prefix - except AttributeError: # parent, so traverse each of the children - traverse(nd.child[0], prefix + b0) - traverse(nd.child[1], prefix + b1) + except AttributeError: # parent, so traverse each child + traverse(nd.child[0], prefix + '0') + traverse(nd.child[1], prefix + '1') traverse(_huffman_tree(__freq_map)) return result @@ -397,18 +572,18 @@ Note: the two lists may be used as input # now just simply record the length for reaching each symbol try: # leaf code_length[nd.symbol] = length - except AttributeError: # parent, so traverse each of the children + except AttributeError: # parent, so traverse each child traverse(nd.child[0], length + 1) traverse(nd.child[1], length + 1) traverse(_huffman_tree(__freq_map)) - # we now have a mapping of symbols to their code length, - # which is all we need - - table = sorted(code_length.items(), key=lambda item: (item[1], item[0])) + # We now have a mapping of symbols to their code length, which is all we + # need to construct a list of tuples (symbol, code length) sorted by + # code length: + table = sorted(code_length.items(), key=lambda item: item[1]) - maxbits = max(item[1] for item in table) + maxbits = table[-1][1] codedict = {} count = (maxbits + 1) * [0] diff -pruN 2.9.2-1/bitarray/util.pyi 3.6.1-1/bitarray/util.pyi --- 2.9.2-1/bitarray/util.pyi 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/bitarray/util.pyi 2025-08-12 08:35:42.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright (c) 2021 - 2024, Ilan Schnell; All Rights Reserved +# Copyright (c) 2021 - 2025, Ilan Schnell; All Rights Reserved from collections import Counter from collections.abc import Iterable, Iterator, Sequence @@ -14,17 +14,18 @@ def zeros(length: int, endian: Optional[ def ones(length: int, endian: Optional[str] = ...) -> bitarray: ... def urandom(length: int, endian: Optional[str] = ...) -> bitarray: ... +def random_p(n: int, + p = ..., + endian: Optional[str] = ...) -> bitarray: ... +def random_k(n: int, + k: int, + endian: Optional[str] = ...) -> bitarray: ... + def pprint(a: Any, stream: BinaryIO = ..., group: int = ..., indent: int = ..., width: int = ...) -> None: ... -def make_endian(a: bitarray, endian: str) -> bitarray: ... -def rindex(a: bitarray, - sub_bitarray: Union[bitarray, int] = ..., - start: int = ..., - stop: int = ...) -> int: ... - def strip(a: bitarray, mode: str = ...) -> bitarray: ... def count_n(a: bitarray, @@ -32,18 +33,27 @@ def count_n(a: bitarray, value: int = ...) -> int: ... def parity(a: bitarray) -> int: ... +def sum_indices(a: bitarray) -> int: ... +def xor_indices(a: bitarray) -> int: ... def count_and(a: bitarray, b: bitarray) -> int: ... def count_or(a: bitarray, b: bitarray) -> int: ... def count_xor(a: bitarray, b: bitarray) -> int: ... def any_and(a: bitarray, b: bitarray) -> bool: ... def subset(a: bitarray, b: bitarray) -> bool: ... -def _correspond_all(a: bitarray, b: bitarray) -> tuple: ... +def correspond_all(a: bitarray, b: bitarray) -> tuple: ... +def byteswap(a: BytesLike, n: int) -> None: ... def intervals(a: bitarray) -> Iterator: ... -def ba2hex(a: bitarray) -> str: ... -def hex2ba(s: AnyStr, endian: Optional[str] = ...) -> bitarray: ... -def ba2base(n: int, a: bitarray) -> str: ... +def ba2hex(a: bitarray, + group: int = ..., + sep: str = ...) -> str: ... +def hex2ba(s: AnyStr, + endian: Optional[str] = ...) -> bitarray: ... +def ba2base(n: int, + a: bitarray, + group: int = ..., + sep: str = ...) -> str: ... def base2ba(n: int, s: AnyStr, endian: Optional[str] = ...) -> bitarray: ... @@ -57,9 +67,9 @@ def int2ba(i: int, def serialize(a: bitarray) -> bytes: ... def deserialize(b: BytesLike) -> bitarray: ... def sc_encode(a: bitarray) -> bytes: ... -def sc_decode(stream: BytesLike) -> bitarray: ... +def sc_decode(stream: Iterable[int]) -> bitarray: ... def vl_encode(a: bitarray) -> bytes: ... -def vl_decode(stream: BytesLike, +def vl_decode(stream: Iterable[int], endian: Optional[str] = ...) -> bitarray: ... def _huffman_tree(freq_map: FreqMap) -> Any: ... diff -pruN 2.9.2-1/debian/.gitignore 3.6.1-1/debian/.gitignore --- 2.9.2-1/debian/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/debian/.gitignore 2025-08-17 00:02:06.000000000 +0000 @@ -0,0 +1 @@ +/files diff -pruN 2.9.2-1/debian/changelog 3.6.1-1/debian/changelog --- 2.9.2-1/debian/changelog 2024-01-05 20:01:39.000000000 +0000 +++ 3.6.1-1/debian/changelog 2025-08-17 00:02:06.000000000 +0000 @@ -1,3 +1,14 @@ +python-bitarray (3.6.1-1) unstable; urgency=medium + + * Team upload. + * New upstream release. + * Use dh-sequence-python3. + * Simplify some debhelper and pybuild overrides. + * Use pybuild-plugin-pyproject. + * Standards-Version: 4.7.2. + + -- Colin Watson Sun, 17 Aug 2025 01:02:06 +0100 + python-bitarray (2.9.2-1) unstable; urgency=medium * New upstream release diff -pruN 2.9.2-1/debian/control 3.6.1-1/debian/control --- 2.9.2-1/debian/control 2024-01-05 20:01:35.000000000 +0000 +++ 3.6.1-1/debian/control 2025-08-17 00:02:06.000000000 +0000 @@ -5,12 +5,13 @@ Maintainer: Debian Python Team , Scott Kitterman Build-Depends: debhelper-compat (= 13), - dh-python, + dh-sequence-python3, pandoc, + pybuild-plugin-pyproject, python3-all, python3-all-dev, python3-setuptools -Standards-Version: 4.6.2 +Standards-Version: 4.7.2 Homepage: https://github.com/ilanschnell/bitarray Vcs-Git: https://salsa.debian.org/python-team/packages/python-bitarray.git Vcs-Browser: https://salsa.debian.org/python-team/packages/python-bitarray @@ -19,6 +20,7 @@ Rules-Requires-Root: no Package: python3-bitarray Architecture: any Depends: ${misc:Depends}, ${python3:Depends}, ${shlibs:Depends} +Breaks: python3-bitstring (<< 4.3.1) Description: Python3 module for efficient boolean array handling The bitarry module provides an object type which efficiently represents an array of booleans. Bitarrays are sequence types and behave very much like diff -pruN 2.9.2-1/debian/rules 3.6.1-1/debian/rules --- 2.9.2-1/debian/rules 2023-08-12 17:54:10.000000000 +0000 +++ 3.6.1-1/debian/rules 2025-08-17 00:02:06.000000000 +0000 @@ -9,6 +9,9 @@ include /usr/share/python3/python.mk DOCTEMPDIR=$(CURDIR)/debian/buildhtml +export PYBUILD_TEST_CUSTOM := 1 +export PYBUILD_TEST_ARGS := {interpreter} -c 'import bitarray; bitarray.test()' + # Move README.html creation here: # we need to have the build-directory, so that we can import `bitarray`, # so let's have pybuild do the heavy lifting of locating it @@ -20,20 +23,14 @@ PYTHONPATH={build_dir} {interpreter} upd pandoc -r gfm -w html5 -o README.html -s --toc --metadata title="BitArray Documentation" README.rst; \ mv $(DOCTEMPDIR)/README.html $(CURDIR) -override_dh_auto_clean: +execute_before_dh_auto_clean: rm -rf $(DOCTEMPDIR) README.html build - dh_auto_clean override_dh_installchangelogs: dh_installchangelogs CHANGE_LOG -override_dh_auto_test: - PYBUILD_SYSTEM=custom \ - PYBUILD_TEST_ARGS="cd {build_dir}; {interpreter} -c 'import bitarray; bitarray.test()'" dh_auto_test - -override_dh_installexamples: - dh_installexamples +execute_after_dh_installexamples: find $(CURDIR)/debian/python*-bitarray -name '.gitignore' -delete %: - dh $@ --with python3 --buildsystem=pybuild + dh $@ --buildsystem=pybuild diff -pruN 2.9.2-1/devel/README 3.6.1-1/devel/README --- 2.9.2-1/devel/README 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/README 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,42 @@ +This directory contains files that are relevant for development +of bitarray, as well as additional tests and verifications. + + +copy_n.py + Illustrate how copy_n() in _bitarray.c works. This is essentially + a Python implementation of copy_n() with output of the different stages + of the bitarray we copy into. + + +random/ + development files for statistical tests + + +resize/ + Things to study the bitarray resize function, including the growth + pattern it creates and tests for the current implementation. + + +shift_r8.c + C program is to illustrate and document shift_r8() + + +test_debug.py + Tests for internal C code which is exposed in debug builds. + These tests will only work when bitarray is compiled in debug mode. + + +test_sum_indices.py + Additional tests for util.sum_indices() for very large n, as well as + some verifications and test for the internal function _ssqi(). + + +test_random.py + * statistical tests for random functions in bitarray.util + * verification of some statistical equations + * verification of code used in random_k() and random_p() + + +tricks.py + Some little tricks and verifications for some code which is used + mostly in the C implementation of bitarray. diff -pruN 2.9.2-1/devel/architecture.txt 3.6.1-1/devel/architecture.txt --- 2.9.2-1/devel/architecture.txt 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/architecture.txt 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,26 @@ +Dependency of files +------------------- + +A depends on B A --------> B + + +A imports B in a function A - - - - > B + + + + +------------+ +------------+ + | util.py |------->| _util.c | + +------------+ +------------+ + | ^ + | | + | +--------------------------------------------------+ + | | | + V v | + +-------------+ - - - > +------------------+ - - - > +--------------+ + | __init__.py | | test_bitarray.py | | test_util.py | + +-------------+ <------ +------------------+ <------ +--------------+ + | + V + +-------------+ + | _bitarray.c | + +-------------+ diff -pruN 2.9.2-1/devel/copy_n.py 3.6.1-1/devel/copy_n.py --- 2.9.2-1/devel/copy_n.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/copy_n.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,235 @@ +""" +The purpose of this script is to illustrate how copy_n() in _bitarray.c works. +This is a Python implementation of copy_n() with output of the different +stages of the bitarray we copy into. + +Sample output: +a = 21 +b = 6 +n = 31 +p1 = 2 +p2 = 6 +p3 = 0 +sa = 5 +sb = -6 + -> p3 = 1 + -> sb = 2 +other +bitarray('00101110 11111001 01011101 11001011 10110000 01011110 011') +b..b+n ^^ ^^^^^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^^ + ======== ======== ======== ======== + 33 +self +bitarray('01011101 11100101 01110101 01011001 01110100 10001010 01111011') +a..a+n ^^^ ^^^^^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^ + 11111 + 2222 +memmove 4 + ======== ======== ======== ======== +bitarray('01011101 11100101 11111001 01011101 11001011 10110000 01111011') +rshift 7 + >>>>>>>> >>>>>>>> >>>>>>>> >>>>>>>> >>>>>>>> +bitarray('01011101 11100101 00000001 11110010 10111011 10010111 01100000') + = ======== ======== ======== ======== + 11111 + 2222 + 33 +bitarray('01011101 11100101 01110101 11110010 10111011 10010111 01101011') +""" +from io import StringIO + +from bitarray import bitarray, bits2bytes +from bitarray.util import pprint + + +verbose = False + +def mark_range_n(i, n, c, text=''): + a = bitarray(i * '0' + n * '1') + f = StringIO() + pprint(a, stream=f) + s = f.getvalue() + print("%-10s" % text + ''.join(c if e == '1' else ' ' for e in s[10:])) + + +def mark_range(i, j, c, text=''): + mark_range_n(i, j - i, c, text) + + +def shift_r8(self, a, b, k): + """ + shift bits in byte-range(a, b) by k bits to right (in-place) + """ + assert 0 <= k < 8 + assert 0 <= a <= self.nbytes + assert 0 <= b <= self.nbytes + if k == 0 or a >= b: + return + self[8 * a : 8 * b] >>= k + +def is_be(self): + return self.endian == 'big' + +bitmask_table = [ + [0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80], # little endian + [0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01], # big endian +] + +ones_table = [ + [0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f], # little endian + [0x00, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe], # big endian +] + +def copy_n(self, a, other, b, n): + """ + copy n bits from other (starting at b) onto self (starting at a) + """ + p1 = a // 8 # first byte to be copied to + p2 = (a + n - 1) // 8 # last byte to be copied to + p3 = b // 8 # first byte to be memmoved from + sa = a % 8 + sb = -(b % 8) + t3 = 0 + + if verbose: + print('a =', a) + print('b =', b) + print('n =', n) + print('p1 =', p1) + print('p2 =', p2) + print('p3 =', p3) + print('sa =', sa) + print('sb =', sb) + + assert 0 <= n <= min(len(self), len(other)) + assert 0 <= a <= len(self) - n + assert 0 <= b <= len(other) - n + if n == 0 or (self is other and a == b): + return + + if sa + sb < 0: + # In order to keep total right shift (sa + sb) positive, we + # increase the first byte to be copied from (p3) by one byte, + # such that memmove() will move all bytes one extra to the left. + + # As other may be self, we need to store this byte as its memory + # location may be overwritten or changed by memmove or rshift. + t3 = memoryview(other)[p3] + p3 += 1 + sb += 8 + if verbose: + print(' -> p3 =', p3) + print(' -> sb =', sb) + + assert a - sa == 8 * p1 + assert b + sb == 8 * p3 + assert p1 <= p2 + assert 8 * p2 < a + n <= 8 * (p2 + 1) + + if verbose: + print('other') + pprint(other) + mark_range_n(b, n, '^', 'b..b+n') + if n > sb: + mark_range_n(8 * p3, 8 * bits2bytes(n - sb), '=') + mark_range_n(b, sb, '3') + print('self') + pprint(self) + mark_range_n(a, n, '^', 'a..a+n') + if n > sb: + mark_range(8 * p1, a, '1') + mark_range(a + n, 8 * p2 + 8, '2') + + if n > sb: + m = bits2bytes(n - sb) # number of bytes memmoved + table = ones_table[is_be(self)] + m1 = table[sa] + m2 = table[(a + n) % 8] + t1 = memoryview(self)[p1] + t2 = memoryview(self)[p2] + + assert p1 + m in [p2, p2 + 1] + assert p1 + m <= self.nbytes and p3 + m <= other.nbytes + + # aligned copy -- copy first sb bits (if any) later + memoryview(self)[p1:p1 + m] = memoryview(other)[p3:p3 + m] + if self.endian != other.endian: + self.bytereverse(p1, p1 + m) + + if verbose: + print('memmove', m) + mark_range_n(8 * p1, 8 * m, '=') + pprint(self) + print('rshift', sa + sb) + mark_range(8 * p1, 8 * (p2 + 1), '>') + + shift_r8(self, p1, p2 + 1, sa + sb) # right shift + if verbose: + pprint(self) + mark_range(8 * p1 + sa + sb, 8 * (p2 + 1), '=') + + if m1: # restore bits at p1 + if verbose: + mark_range(8 * p1, a, '1') + memoryview(self)[p1] = (memoryview(self)[p1] & ~m1) | (t1 & m1) + + if m2: # restore bits at p2 + if verbose: + mark_range(a + n, 8 * p2 + 8, '2') + memoryview(self)[p2] = (memoryview(self)[p2] & m2) | (t2 & ~m2) + + if verbose: + mark_range_n(a, sb, '3') + for i in range(min(sb, n)): # copy first sb bits + self[i + a] = bool(t3 & bitmask_table[is_be(other)][(i + b) % 8]) + + if verbose: + pprint(self) + + +def test_copy_n(): + from random import choice, randrange, randint + from bitarray.util import urandom + + def random_endian(): + return choice(['little', 'big']) + + max_size = 56 + + for _ in range(10_000): + N = randrange(max_size) + M = randrange(max_size) + n = randint(0, min(N, M)) + a = randint(0, N - n) + b = randint(0, M - n) + x = urandom(N, random_endian()) + y = urandom(M, random_endian()) + z = x.copy() + copy_n(x, a, y, b, n) + z[a:a + n] = y[b:b + n] + assert x == z + + for _ in range(10_000): + N = randrange(max_size) + n = randint(0, N) + a = randint(0, N - n) + b = randint(0, N - n) + x = urandom(N, random_endian()) + z = x.copy() + copy_n(x, a, x, b, n) + z[a:a + n] = z[b:b + n] + assert x == z + + +if __name__ == '__main__': + test_copy_n() + verbose = True + other = bitarray( + '00101110 11111001 01011101 11001011 10110000 01011110 011') + self = bitarray( + '01011101 11100101 01110101 01011001 01110100 10001010 01111011') + copy_n(self, 21, other, 6, 31) + assert self == bitarray( + '01011101 11100101 01110101 11110010 10111011 10010111 01101011') + #copy_n(self, 2, other, 12, 1) + #copy_n(self, 9, other, 17, 23) diff -pruN 2.9.2-1/devel/random/binomial.py 3.6.1-1/devel/random/binomial.py --- 2.9.2-1/devel/random/binomial.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/random/binomial.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,134 @@ +import sys +from math import sqrt + +from scipy.special import betainc + + +class BinomialDist: + + # This class describes the binomial distribution with parameters n and p. + # That is, the (discrete) probability distribution of the number of + # successes in a sequence of n independent Bernoulli trails, with each + # trail having a probability p of success. + + def __init__(self, n, p): + assert n > 0 + assert 0.0 <= p <= 1.0 + self.n = n + self.p = p + self.q = 1.0 - p + self.mu = n * p + self.sigma = sqrt(n * p * self.q) + + def print(self): + print("n = %d p = %f" % (self.n, self.p)) + print("mu = %f" % self.mu) + print("sigma = %f" % self.sigma) + + def pmf(self, k): + assert 0 <= k <= self.n, k + # The reason we use .cdf() to calculate the PMF is because + # comb(n, k) * p ** k * (1.0 - p) ** (n - k) will fail for large + # n, whereas .cdf() uses the regularized incomplete beta function. + return self.cdf(k) - self.cdf(k - 1) + + def cdf(self, k): + return betainc(self.n - k, k + 1, self.q) + + def range_k(self, k1, k2): + "probability for k being in k1 <= k <= k2" + assert 0 <= k1 <= k2 <= self.n + return self.cdf(k2) - self.cdf(k1 - 1) + +# --------------------------------------------------------------------------- + +import unittest +from math import comb + +class BinomialDistTests(unittest.TestCase): + + def test_pmf_simple(self): + b = BinomialDist(1, 0.7) + self.assertAlmostEqual(b.pmf(0), 0.3) + self.assertAlmostEqual(b.pmf(1), 0.7) + + b = BinomialDist(2, 0.5) + self.assertAlmostEqual(b.pmf(0), 0.25) + self.assertAlmostEqual(b.pmf(1), 0.50) + self.assertAlmostEqual(b.pmf(2), 0.25) + + def test_pmf_sum(self): + for n in 10, 100, 1_000, 10_000: + b = BinomialDist(n, 0.5) + tot = 0 + for k in range(n + 1): + tot += b.pmf(k) + self.assertAlmostEqual(tot, 1.0) + + def test_pmf(self): + for n in 10, 50, 100, 250: + for p in 0.1, 0.2, 0.5, 0.7: + b = BinomialDist(n, p) + for k in range(n + 1): + res = comb(n, k) * p ** k * (1.0 - p) ** (n - k) + self.assertAlmostEqual(b.pmf(k), res, delta=1e-14) + + def test_cdf(self): + for n in 5, 50, 500: + b = BinomialDist(n, 0.3) + self.assertAlmostEqual(b.cdf(-1), 0.0) + self.assertAlmostEqual(b.cdf(n), 1.0) + sm = 0.0 + for k in range(n + 1): + sm += b.pmf(k) + self.assertAlmostEqual(b.cdf(k), sm) + + def test_range_k(self): + n = 10_000 + for p in 0.1, 0.2, 0.5, 0.7: + b = BinomialDist(n, p) + self.assertAlmostEqual(b.range_k(0, n), 1.0) + for k in range(n + 1): + self.assertAlmostEqual(b.range_k(k, k), b.pmf(k)) + + def test_range_half(self): + n = 1_000_001 + b = BinomialDist(n, 0.5) + self.assertAlmostEqual(b.range_k(0, 500_000), 0.5) + self.assertAlmostEqual(b.range_k(500_001, n), 0.5) + + +if __name__ == '__main__': + if len(sys.argv) == 1: + unittest.main() + + # This code was used to create some of the tests for util.random_p() + # in ../test_random.py + # + # python binomial.py 250_000 5 0.3 + # python binomial.py 100_000 100 .375 37..48 + # python binomial.py 25_000 100_000 .5 48_000..50_000 50_000..50_200 + + m, n = [int(i) for i in sys.argv[1:3]] + p = float(sys.argv[3]) + bd = BinomialDist(n, p) + bd.print() + if n <= 100_000: + p_tot = 0.0 + for k in range(n + 1): + p = bd.pmf(k) + p_tot += p + if p < 0.01: + continue + bb = BinomialDist(m, p) + fmt = "self.assertTrue(abs(C[{:d}] - {:6_d}) <= {:6_d}) # p = {:f}" + print(fmt.format(k, round(bb.mu), round(10 * bb.sigma), p)) + assert abs(p_tot - 1.0) < 1e-15, p_tot + + for s in sys.argv[4:]: + k1, k2 = [int(t) for t in s.split("..")] + p = bd.range_k(k1, k2) + bb = BinomialDist(m, p) + print("x = sum(C[k] for k in %s)" % range(k1, k2 + 1)) + fmt = "self.assertTrue(abs(x - {:6_d}) <= {:6_d}) # p = {:f}" + print(fmt.format(round(bb.mu), round(10 * bb.sigma), p)) diff -pruN 2.9.2-1/devel/random/plot.py 3.6.1-1/devel/random/plot.py --- 2.9.2-1/devel/random/plot.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/random/plot.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,47 @@ +from collections import Counter +from statistics import fmean, pstdev + +from matplotlib import pyplot as plt +from bitarray.util import random_k, sum_indices + +from sample import SampleMeanDist + +# The code will also work, if you change these parameters here: +SMD = SampleMeanDist(n=1_000, k=600) +M = 100_000 # number of trails +DX = 2.5 # width used for counting trail outcomes + +def plot_pdf(plt, xmin, xmax): + X = [] + Y = [] + n = 2_000 + for i in range(n + 1): + x = xmin + i * (xmax - xmin) / n + X.append(x) + Y.append(SMD.pdf(x) * DX) + plt.plot(X, Y) + +def plot_count(plt, C): + X = [] + Y = [] + for i in range(min(C), max(C) + 1): + x = i * DX + X.append(x) + Y.append(C[i] / M) + plt.scatter(X, Y, color='red') + +if __name__ == '__main__': + SMD.print() + C = Counter() + values = [] + for _ in range(M): + x = sum_indices(random_k(SMD.n, SMD.k)) / SMD.k + C[round(x / DX)] += 1 + values.append(x) + assert C.total() == M + print("mean", fmean(values)) + print("stdev", pstdev(values, SMD.mu)) + + plot_count(plt, C) + plot_pdf(plt, min(C) * DX, max(C) * DX) + plt.show() diff -pruN 2.9.2-1/devel/random/sample.py 3.6.1-1/devel/random/sample.py --- 2.9.2-1/devel/random/sample.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/random/sample.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,136 @@ +import sys +from math import sqrt, erf, exp, pi + + +class SampleMeanDist: + + # This class describes the mean of a discrete uniform distribution + # without replacement of k integers in range(n). + # That is, from the integers in range(n) (population), we take a random + # sample of k (without replacement) and calculate their mean value x. + # The distribution of x is what we are interested it. + + def __init__(self, n, k): + assert n >= 1 + assert 1 <= k < n + self.n = n + self.k = k + + # the sample mean is the population mean + self.mu = 0.5 * (n - 1) # sum(range(n)) / n = (n - 1) / 2 + + # The standard deviation of a sample, is also called standard error. + # The standard error of the mean is the standard deviation of the + # population, divided by the square root of the sample size k. + # The variance of the population is: + # + # (n**2 - 1) / 12 (see below) + # + # So for the standard deviation, we get: + self.sigma = sqrt((n * n - 1) / (12 * k)) + + # Finite population correction (FPC) + # ---------------------------------- + # + # Let us consider two cases: + # + # (a) For very small sample sizes k (compared to the population + # size n), the FPC is close to one. That is, it has no effect + # on the standard error. Our distribution is basically the same + # as if we had replacement. + # + # (b) For sample sizes k close to the population size n, the FPC (and + # hence the standard error) becomes zero. That is, when we + # sample all elements, we always get the same sample mean (the + # population mean) with no standard error. + # + fpc = sqrt((n - k) / (n - 1)) + self.sigma *= fpc + + def print(self): + print("n = %d k = %d" % (self.n, self.k)) + print("mu = %f" % self.mu) + print("sigma = %f" % self.sigma) + + def pdf(self, x): + return exp(-0.5 * ((x - self.mu) / self.sigma) ** 2) / ( + sqrt(2.0 * pi) * self.sigma) + + def cdf(self, x): + return 0.5 * (1.0 + erf((x - self.mu) / (self.sigma * sqrt(2.0)))) + + def range_p(self, x1, x2): + "probability for x (the mean of the sample) being in x1 < x < x2" + assert 0 <= x1 <= x2 <= self.n + return self.cdf(x2) - self.cdf(x1) + +# --------------------------------------------------------------------------- + +import unittest + +class SampleMeanDistTests(unittest.TestCase): + + def test_pdf(self): + n, k = 100, 30 + smd = SampleMeanDist(n, k) + N = 1000 + dx = n / N + p = 0.0 + for i in range(N + 1): + x = i * dx + p += smd.pdf(x) * dx + self.assertAlmostEqual(p, 1.0) + + def test_cdf(self): + smd = SampleMeanDist(100, 30) + self.assertAlmostEqual(smd.cdf( 0.0), 0.0) + self.assertAlmostEqual(smd.cdf( 49.5), 0.5) + self.assertAlmostEqual(smd.cdf(100.0), 1.0) + + def test_range_half(self): + for k in 10, 20, 50, 100, 150: + smd = SampleMeanDist(200, k) + self.assertAlmostEqual(smd.range_p(0.0, 99.5), 0.5) + self.assertAlmostEqual(smd.range_p(99.5, 200), 0.5) + + def test_verify_pop_mean(self): + for n in range(1, 100): + self.assertEqual((n - 1) /2, sum(range(n)) / n) + + def test_verify_pop_variance(self): + for n in range(1, 100): + mean = (n - 1) / 2 + sigma2 = sum((j - mean)**2 for j in range(n)) / n + self.assertEqual((n * n - 1) / 12, sigma2) + + +if __name__ == '__main__': + if len(sys.argv) == 1: + unittest.main() + + # This code was used to create some of the tests for util.random_k() + # in ../test_random.py + # + # python sample.py 100_000 1000 500 0 500 510 520 1000 + # python sample.py 100_000 500 400 200 249.5 251 255 260 300 + + from itertools import pairwise + + m, n, k = [int(i) for i in sys.argv[1:4]] + smd = SampleMeanDist(n, k) + smd.print() + ranges = [float(x) for x in sys.argv[4:]] + print("ranges = %r" % ranges) + p_tot = 0.0 + for i, (x1, x2) in enumerate(pairwise(ranges)): + p = smd.range_p(x1, x2) + p_tot += p + mu = m * p + sigma = sqrt(m * p * (1.0 - p)) + if mu < 10 * sigma: + continue + fmt = "self.assertTrue(abs(C[{:d}] - {:6_d}) <= {:6_d}) # p = {:f}" + print(fmt.format(i, round(mu), round(10 * sigma), p)) + + if abs(p_tot - 1.0) > 1e-15: + print(p_tot) diff -pruN 2.9.2-1/devel/random/test_sample.py 3.6.1-1/devel/random/test_sample.py --- 2.9.2-1/devel/random/test_sample.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/random/test_sample.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,30 @@ +import unittest +from itertools import pairwise +from collections import Counter +from statistics import fmean +from random import sample + + +class RandomSampleTests(unittest.TestCase): + + def test_mean(self): + # python sample.py 100_000 100 20 0 45.5 49.5 100 + M = 100_000 # number of trails + N = 100 # population size + K = 20 # sample size + C = Counter() + ranges = [0.0, 45.5, 49.5, 100.0] + for _ in range(M): + x = fmean(sample(range(N), K)) + for i, (x1, x2) in enumerate(pairwise(ranges)): + if x1 <= x < x2: + C[i] += 1 + + self.assertEqual(C.total(), M) + self.assertTrue(abs(C[0] - 24_529) <= 1_361) # p = 0.245291 + self.assertTrue(abs(C[1] - 25_471) <= 1_378) # p = 0.254709 + self.assertTrue(abs(C[2] - 50_000) <= 1_581) # p = 0.500000 + + +if __name__ == '__main__': + unittest.main() diff -pruN 2.9.2-1/devel/resize/.gitignore 3.6.1-1/devel/resize/.gitignore --- 2.9.2-1/devel/resize/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/resize/.gitignore 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,2 @@ +resize +pattern-* diff -pruN 2.9.2-1/devel/resize/Makefile 3.6.1-1/devel/resize/Makefile --- 2.9.2-1/devel/resize/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/resize/Makefile 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,13 @@ +resize: resize.c + gcc -Wall resize.c -o resize + +test: resize + ./resize >pattern-c.txt + python resize.py >pattern-py.txt + diff pattern-c.txt pattern-py.txt + python test_resize.py + +clean: + rm -f resize + rm -f pattern-* + rm -rf __pycache__ diff -pruN 2.9.2-1/devel/resize/README.md 3.6.1-1/devel/resize/README.md --- 2.9.2-1/devel/resize/README.md 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/resize/README.md 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,10 @@ +The bitarray resize function and growth pattern +=============================================== + +Running `python resize.py` will display the bitarray growth pattern. +This is done by appending one bit to a bitarray in a loop, and displaying +the allocated size of the bitarray object each time it changes. + +The program `resize.c` contains a distilled version of the `resize()` +function which contains the implementation of this growth pattern. +Running this C program gives exactly the same output. diff -pruN 2.9.2-1/devel/resize/resize.c 3.6.1-1/devel/resize/resize.c --- 2.9.2-1/devel/resize/resize.c 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/resize/resize.c 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,113 @@ +#include +#include + + +typedef struct { + int size; + int nbits; + int allocated; +} bitarrayobject; + + +/* number of bytes necessary to store given bits */ +#define BYTES(bits) (((bits) + 7) >> 3) + +uint64_t s = 290797; + +int bbs(void) +{ + s *= s; + s %= 50515093; + return s % 8000; +} + +void resize(bitarrayobject *self, int nbits) +{ + int size = self->size, allocated = self->allocated; + int newsize = BYTES(nbits), new_allocated; + + if (newsize == size) { + self->nbits = nbits; + return; + } + + if (newsize == 0) { + /* free(self->ob_item) */ + self->size = 0; + self->allocated = 0; + self->nbits = 0; + return; + } + + if (allocated >= newsize) { + if (newsize >= allocated / 2) { + self->size = newsize; + self->nbits = nbits; + return; + } + new_allocated = newsize; + } + else { + new_allocated = newsize; + if (size != 0 && newsize / 2 <= allocated) { + new_allocated += (newsize >> 4) + (newsize < 8 ? 3 : 7); + new_allocated &= ~(int) 3; + } + } + + /* realloc(self->ob_item) */ + self->size = newsize; + self->allocated = new_allocated; + self->nbits = nbits; +} + + +int main() +{ + int i, nbits, prev_alloc = -1; + bitarrayobject x; + +#define SHOW printf("%d %d\n", x.size, x.allocated) + + x.size = 0; + x.allocated = 0; + for (nbits = 0; nbits < 1000; nbits++) { + if (prev_alloc != x.allocated) + SHOW; + prev_alloc = x.allocated; + resize(&x, nbits); + } + + resize(&x, 800000); SHOW; + resize(&x, 400000); SHOW; + resize(&x, 399992); SHOW; + resize(&x, 500000); SHOW; + resize(&x, 0); SHOW; + resize(&x, 0); SHOW; + resize(&x, 10000); SHOW; + resize(&x, 400); SHOW; + resize(&x, 600); SHOW; + resize(&x, 2000); SHOW; + + for (nbits = 2000; nbits >= 0; nbits--) { + if (prev_alloc != x.allocated) + SHOW; + prev_alloc = x.allocated; + resize(&x, nbits); + } + SHOW; + + for (nbits = 0; nbits < 100; nbits += 8) { + x.size = 0; + x.allocated = 0; + resize(&x, nbits); + SHOW; + } + + for (i = 0; i < 100000; i++) { + resize(&x, bbs()); + SHOW; + } + + return 0; +} diff -pruN 2.9.2-1/devel/resize/resize.py 3.6.1-1/devel/resize/resize.py --- 2.9.2-1/devel/resize/resize.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/resize/resize.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,42 @@ +from bitarray import bitarray + +from test_resize import get_alloc, resize, show + + +s = 290797 +def bbs(): + global s + s = pow(s, 2, 50515093) + return s % 8000 + +a = bitarray() +prev = -1 +while len(a) < 1_000: + alloc = get_alloc(a) + if prev != alloc: + show(a) + prev = alloc + a.append(1) + +for i in 800_000, 400_000, 399_992, 500_000, 0, 0, 10_000, 400, 600, 2_000: + resize(a, i) + assert len(a) == i + show(a) + +while len(a): + alloc = get_alloc(a) + if prev != alloc: + show(a) + prev = alloc + a.pop() + +show(a) + +for nbits in range(0, 100, 8): + a = bitarray() + a.extend(bitarray(nbits)) + show(a) + +for _ in range(100_000): + resize(a, bbs()) + show(a) diff -pruN 2.9.2-1/devel/resize/test_resize.py 3.6.1-1/devel/resize/test_resize.py --- 2.9.2-1/devel/resize/test_resize.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/resize/test_resize.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,81 @@ +import unittest + +from bitarray import bitarray + +PATTERN = [0, 1, 4, 8, 16, 24, 32, 40, 48, 56, 64, 76, 88, 100, 112, 124, 136] + +def get_alloc(a): + return a.buffer_info()[4] + +def resize(a, n): + increase = n - len(a) + if increase > 0: + a.extend(bitarray(increase)) + elif increase < 0: + del a[n:] + +def show(a): + info = a.buffer_info() + size = info[1] + alloc = info[4] + print('%d %d' % (size, alloc)) + + +class ResizeTests(unittest.TestCase): + + def test_pattern(self): + pat = [] + a = bitarray() + prev = -1 + while len(a) < 1000: + alloc = get_alloc(a) + if prev != alloc: + pat.append(alloc) + prev = alloc + a.append(0) + self.assertEqual(pat, PATTERN) + + def test_increase(self): + # make sure sequence of appends will always increase allocated size + a = bitarray() + prev = -1 + while len(a) < 100_000: + alloc = get_alloc(a) + self.assertTrue(prev <= alloc) + prev = alloc + a.append(1) + + def test_decrease(self): + # ensure that when we start from a large array and delete part, we + # always get a decreasing allocation + a = bitarray(10_000_000) + prev = get_alloc(a) + while a: + del a[-100_000:] + alloc = a.buffer_info()[4] + self.assertTrue(alloc <= prev) + prev = alloc + + def test_no_overalloc(self): + # initalizing a bitarray from a list or bitarray does not overallocate + for n in range(1000): + a = bitarray(8 * n * [1]) + self.assertEqual(get_alloc(a), n) + b = bitarray(a) + self.assertEqual(get_alloc(b), n) + c = bitarray(8 * n) + self.assertEqual(get_alloc(c), n) + + def test_no_overalloc_large(self): + # starting from a large bitarray, make we sure we don't realloc each + # time we extend + a = bitarray(1_000_000) # no overallocation + self.assertEqual(get_alloc(a), 125_000) + a.extend(bitarray(8)) # overallocation happens here + alloc = get_alloc(a) + for _ in range(1000): + a.extend(bitarray(8)) + self.assertEqual(get_alloc(a), alloc) + +if __name__ == '__main__': + unittest.main() diff -pruN 2.9.2-1/devel/sc/compress.py 3.6.1-1/devel/sc/compress.py --- 2.9.2-1/devel/sc/compress.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/sc/compress.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,58 @@ +import bz2 +import gzip +import random +from time import perf_counter + +from bitarray.util import ( + ones, random_p, + serialize, deserialize, + sc_encode, sc_decode, + vl_encode, vl_decode, +) + +from sc_stat import sc_stat + + +def p_range(): + n = 1 << 28 + p = 1.0 + a = ones(n) + print(" p ratio raw" + " type 1 type 2 type 3 type 4") + print(" " + 73 *'-') + while p > 1e-8: + b = sc_encode(a) + blocks = sc_stat(b)['blocks'] + print(' %11.8f %11.8f %8d %8d %8d %8d %8d' % + tuple([p, len(b) / (n / 8)] + blocks)) + assert a == sc_decode(b) + a &= random_p(n) + p /= 2 + +def compare(): + n = 1 << 26 + a = random_p(n, 1.0 / 1024) + + raw = a.tobytes() + print(20 * ' ' + "compress (ms) decompress (ms) ratio") + print(70 * '-') + for name, f_e, f_d in [ + ('serialize', serialize, deserialize), + ('vl', vl_encode, vl_decode), + ('sc' , sc_encode, sc_decode), + ('gzip', gzip.compress, gzip.decompress), + ('bz2', bz2.compress, bz2.decompress)]: + x = a if name in ('serialize', 'vl', 'sc') else raw + t0 = perf_counter() + b = f_e(x) # compression + t1 = perf_counter() + c = f_d(b) # decompression + t2 = perf_counter() + print(" %-11s %16.3f %16.3f %16.4f" % + (name, 1000 * (t1 - t0), 1000 * (t2 - t1), len(b) / len(raw))) + assert c == x + +if __name__ == '__main__': + random.seed(123) + compare() + p_range() diff -pruN 2.9.2-1/devel/sc/sc_stat.py 3.6.1-1/devel/sc/sc_stat.py --- 2.9.2-1/devel/sc/sc_stat.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/sc/sc_stat.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,162 @@ +from itertools import islice +from random import randrange + + +bitcount_table = [bin(i)[2:].count('1') for i in range(256)] + + +def read_n(n, stream): + i = 0 + for j in range(n): + i |= next(stream) << 8 * j + return i + +def sc_decode_header(stream): + head = next(stream) + if head & 0xe0: + raise ValueError("invalid header: 0x%02x" % head) + endian = 'big' if head & 0x10 else 'little' + length = head & 0x0f + nbits = read_n(length, stream) + return endian, nbits + +def sc_decode_block(stream, stats): + head = next(stream) + + if head < 0xa0: # type 0 - 0x00 -- 0x9f + if head == 0: # stop byte + return False + n = 0 + k = head if head <= 32 else 32 * (head - 31) + elif head < 0xc0: # type 1 - 0xa0 .. 0xbf + n = 1 + k = head - 0xa0 + elif 0xc2 <= head <= 0xc4: # type 2 .. 4 - 0xc2 .. 0xc4 + n = head - 0xc0 + k = next(stream) # index count byte + else: + raise ValueError("Invalid block head: 0x%02x" % head) + + stats['blocks'][n] += 1 + + # consume block data + nconsume = max(1, n) * k # size of block data to consume below + if stats.get('count'): + if n == 0: + stats['count'][0] += sum(bitcount_table[next(stream)] + for _ in range(k)) + nconsume = 0 + else: + stats['count'][n] += k + + next(islice(stream, nconsume, nconsume), None) + + return True + +def sc_stat(stream, count=False): + """sc_stat(stream) -> dict + +Decode a compressed byte stream (generated by `sc_encode()` and return +useful statistics. In particular, a list of length 5 with the count for +each block type. +""" + stream = iter(stream) + endian, nbits = sc_decode_header(stream) + + stats = {'endian': endian, + 'nbits': nbits, + 'blocks': 5 * [0]} + if count: + stats['count'] = 5 * [0] + + while sc_decode_block(stream, stats): + pass + + return stats + +# --------------------------------------------------------------------------- + +import unittest + +from bitarray import bitarray +from bitarray.util import sc_encode, sc_decode + + +class Tests(unittest.TestCase): + + def test_empty(self): + blob = b"\x01\x00\0" + self.assertEqual(sc_stat(blob), + {'endian': 'little', + 'nbits': 0, + 'blocks': [0, 0, 0, 0, 0]}) + self.assertEqual(sc_decode(blob), bitarray()) + + def test_zeros_explitcit(self): + for blob, blocks in [ + (b"\x11\x08\0", [0, 0, 0, 0, 0]), + (b"\x11\x08\x01\x00\0", [1, 0, 0, 0, 0]), + (b"\x11\x08\xa0\0", [0, 1, 0, 0, 0]), + (b"\x11\x08\xc2\x00\0", [0, 0, 1, 0, 0]), + (b"\x11\x08\xc3\x00\0", [0, 0, 0, 1, 0]), + (b"\x11\x08\xc4\x00\0", [0, 0, 0, 0, 1]), + ]: + stat = sc_stat(blob, count=True) + self.assertEqual(stat['blocks'], blocks) + self.assertEqual(stat['count'], 5 * [0]) + self.assertEqual(sc_decode(blob), bitarray(8)) + + def test_untouch(self): + stream = iter(b"\x01\x07\x01\x73\0XYZ") + self.assertEqual(sc_decode(stream), bitarray("1100111")) + self.assertEqual(next(stream), ord('X')) + + def test_random(self): + n = 20_000_000 + a = bitarray(n) + for c in range(0, 21, 2): + lst = [randrange(n) for _ in range(1 << c)] + a[lst] = 1 + stat = sc_stat(sc_encode(a), count=True) + # print(c, len(a), a.count(), stat['blocks']) + self.assertEqual(sum(stat['count']), a.count()) + + def test_end_of_stream(self): + for blob in [b'', b'\x00', b'\x01', b'\x02\x77', + b'\x01\x04\x01', b'\x01\x04\xa1', b'\x01\x04\xa0']: + self.assertRaises(StopIteration, sc_stat, blob) + self.assertRaises(StopIteration, sc_decode, blob) + + def test_values(self): + b = [0x11, 3, 1, 32, 0] + self.assertEqual(sc_decode(b), bitarray("001")) + self.assertEqual(sc_stat(b), {'endian': 'big', + 'nbits': 3, + 'blocks': [1, 0, 0, 0, 0]}) + for x in -1, 256: + b[-1] = x + self.assertRaises(ValueError, sc_stat, b) + for x in None, "F", Ellipsis, []: + b[-1] = x + self.assertRaises(TypeError, sc_stat, b) + + def test_example(self): + n = 1 << 26 + a = bitarray(n, 'little') + a[:1 << 16] = 1 + for i in range(2, 1 << 16): + a[n // i] = 1 + b = sc_encode(a) + stat = sc_stat(b, True) + self.assertEqual(stat['blocks'], [2, 147, 3, 1, 1]) + self.assertEqual(stat['count'], [1 << 16, 374, 427, 220, 2]) + self.assertEqual(a, sc_decode(b)) + + a.reverse() + b = sc_encode(a) + self.assertEqual(sc_stat(b)['blocks'], [2, 256, 254, 3, 0]) + self.assertEqual(a, sc_decode(b)) + + +if __name__ == '__main__': + unittest.main() diff -pruN 2.9.2-1/devel/set_range_opt.py 3.6.1-1/devel/set_range_opt.py --- 2.9.2-1/devel/set_range_opt.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/set_range_opt.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,84 @@ +import unittest +from random import getrandbits, randint, randrange +from time import perf_counter + +from bitarray import bitarray +from bitarray.util import urandom + + +def nxir(x, start, step): + assert x >= start and step > 0 + # in Python we can use a simpler expression than in C + return (start - x) % step + + +def set_range_opt(self, start, stop, step, value): + ca = (start + 7) // 8 + cb = stop // 8 + m = (cb - ca) * 8 + + assert m >= 0 + assert 0 <= 8 * ca - start < 8 + assert 0 <= stop - 8 * cb < 8 + + mask = bitarray(step, self.endian) + mask.setall(not value) + mask[nxir(8 * ca, start, step)] = value + mask *= (m - 1) // step + 1 + del mask[m:] # in the C version we wouldn't bother + assert len(mask) % 8 == 0 + + self[start : 8 * ca : step] = value + if value: + self[8 * ca : 8 * cb] |= mask + else: + self[8 * ca : 8 * cb] &= mask + self[8 * cb + nxir(8 * cb, start, step) : stop : step] = value + + +class Tests(unittest.TestCase): + + def test_nxir(self): + for _ in range(1000): + start = randrange(100) + step = randrange(1, 20) + x = randrange(start, start + 100) + nx = nxir(x, start, step) + self.assertTrue(0 <= nx < step) + self.assertEqual((x + nx) % step, start % step) + + def test_setslice_bool_step(self): + # this test exercises set_range() when stop is much larger than start + for _ in range(5000): + n = randrange(3000, 4000) + a = urandom(n) + aa = a.tolist() + s = slice(randrange(1000), randrange(1000, n), randint(1, 100)) + self.assertTrue(s.stop - s.start >= 0) + slicelength = len(range(n)[s]) + self.assertTrue(slicelength > 0) + v = getrandbits(1) + set_range_opt(a, s.start, s.stop, s.step, v) + aa[s] = slicelength * [v] + self.assertEqual(a.tolist(), aa) + + +def speed_cmp(): + n = 1_000_000 + print("n=%d\ntimes in micro seconds\n" % n) + print('%8s %12s %12s' % ("step", "this-code", "native")) + for step in range(1, 20): + a = bitarray(n) + b = bitarray(n) + t0 = perf_counter() + set_range_opt(a, 0, n, step, 1) + t1 = perf_counter() + b[::step] = 1 + t2 = perf_counter() + print('%8d %12.3f %12.3f' % (step, 1E6 * (t1 - t0), 1E6 * (t2 - t1))) + assert a == b + + +if __name__ == '__main__': + speed_cmp() + unittest.main() diff -pruN 2.9.2-1/devel/shift_r8.c 3.6.1-1/devel/shift_r8.c --- 2.9.2-1/devel/shift_r8.c 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/shift_r8.c 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,148 @@ +/* + The purpose of this C program is to illustrate the functions shift_r8le() + and shift_r8be(), which are called from shift_r8(). + These functions are symmetrical with the following replacements: + + PY_LITTLE_ENDIAN <-> PY_BIG_ENDIAN + <<= >>= + >> << + + Creating a macro from which both functions can be created is not possible, + unless one replaces the existing preprocessor introductions with ordinary + if statements. For the sake of simplicity we do not want to do this here, + even though it would avoid the spammish repetition. +*/ +#include +#include +#include + +#define Py_ssize_t ssize_t + +static inline uint64_t +builtin_bswap64(uint64_t word) +{ +#if (defined(__clang__) || \ + (defined(__GNUC__) \ + && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 3)))) + /* __builtin_bswap64() is available since GCC 4.3. */ +# define HAVE_BUILTIN_BSWAP64 1 + return __builtin_bswap64(word); +#elif defined(_MSC_VER) +# define HAVE_BUILTIN_BSWAP64 1 + return _byteswap_uint64(word); +#else +# define HAVE_BUILTIN_BSWAP64 0 + abort() +#endif +} + +/* machine byte-order */ +#define PY_LITTLE_ENDIAN 1 +#define PY_BIG_ENDIAN 0 + +/* bit-endianness */ +#define ENDIAN_LITTLE 0 +#define ENDIAN_BIG 1 + +#define BITMASK(endian, i) (((char) 1) << (endian == ENDIAN_LITTLE ? \ + ((i) % 8) : (7 - (i) % 8))) + +/* The following two functions operate on first n bytes in buffer. + Within this region, they shift all bits by k positions to right, + i.e. towards higher addresses. + They operate on little-endian and bit-endian bitarrays respectively. + As we shift right, we need to start with the highest address and loop + downwards such that lower bytes are still unaltered. +*/ +static void +shift_r8le(unsigned char *buff, Py_ssize_t n, int k) +{ + Py_ssize_t w = 0; + +#if HAVE_BUILTIN_BSWAP64 || PY_LITTLE_ENDIAN /* use shift word */ + w = n / 8; /* number of words used for shifting */ + n %= 8; /* number of remaining bytes */ +#endif + while (n--) { /* shift in byte-range(8 * w, n) */ + Py_ssize_t i = n + 8 * w; + buff[i] <<= k; /* shift byte */ + if (n || w) /* add shifted next lower byte */ + buff[i] |= buff[i - 1] >> (8 - k); + } + while (w--) { /* shift in word-range(0, w) */ + uint64_t *p = ((uint64_t *) buff) + w; +#if HAVE_BUILTIN_BSWAP64 && PY_BIG_ENDIAN + *p = builtin_bswap64(*p); + *p <<= k; + *p = builtin_bswap64(*p); +#else + *p <<= k; +#endif + if (w) /* add shifted byte from next lower word */ + buff[8 * w] |= buff[8 * w - 1] >> (8 - k); + } +} + +static void +shift_r8be(unsigned char *buff, Py_ssize_t n, int k) +{ + Py_ssize_t w = 0; + +#if HAVE_BUILTIN_BSWAP64 || PY_BIG_ENDIAN /* use shift word */ + w = n / 8; /* number of words used for shifting */ + n %= 8; /* number of remaining bytes */ +#endif + while (n--) { /* shift in byte-range(8 * w, n) */ + Py_ssize_t i = n + 8 * w; + buff[i] >>= k; /* shift byte */ + if (n || w) /* add shifted next lower byte */ + buff[i] |= buff[i - 1] << (8 - k); + } + while (w--) { /* shift in word-range(0, w) */ + uint64_t *p = ((uint64_t *) buff) + w; +#if HAVE_BUILTIN_BSWAP64 && PY_LITTLE_ENDIAN + *p = builtin_bswap64(*p); + *p >>= k; + *p = builtin_bswap64(*p); +#else + *p >>= k; +#endif + if (w) /* add shifted byte from next lower word */ + buff[8 * w] |= buff[8 * w - 1] << (8 - k); + } +} + +/* display first nbits bytes of buffer given assumed bit-endianness + to one line in stdout */ +void display(unsigned char *buffer, Py_ssize_t nbits, int endian) +{ + Py_ssize_t i; + + for (i = 0; i < nbits; i++) + printf("%d", (buffer[i / 8] & BITMASK(endian, i)) ? 1 : 0); + + printf("\n"); +} + +int main() +{ +#define NBYTES 10 + unsigned char array[NBYTES] = {1, 15, 0, 131, 0, 255, 0, 7, 0, 1}; + ssize_t i; + + if ((PY_LITTLE_ENDIAN != (*((uint64_t *) "\xff\0\0\0\0\0\0\0") == 0xff)) + || + (PY_BIG_ENDIAN != (*((uint64_t *) "\0\0\0\0\0\0\0\xff") == 0xff))) { + printf("Error: machine byte-order\n"); + return 1; + } + for (i = 0; i < 15; i++) { + display(array, 77, ENDIAN_LITTLE); + shift_r8le(array, NBYTES, 1); + } + for (i = 0; i < 15; i++) { + display(array, 77, ENDIAN_BIG); + shift_r8be(array, NBYTES, 1); + } + return 0; +} diff -pruN 2.9.2-1/devel/test_debug.py 3.6.1-1/devel/test_debug.py --- 2.9.2-1/devel/test_debug.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/test_debug.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,457 @@ +import os +import sys +import unittest +from random import randint, randrange + +from bitarray import bitarray +from bitarray.util import zeros, ones, int2ba, parity + +from bitarray.test_bitarray import Util, urandom_2, skipIf, PTRSIZE + +# --------------------- internal C-level debug tests ------------------------ + +from bitarray._util import ( + _setup_table, _zlw, # defined in bitarray.h + _cfw, _read_n, _write_n, _sc_rts, _SEGSIZE, # defined in _util.h +) +SEGBITS = 8 * _SEGSIZE + +# ---------------------------- bitarray.h ----------------------------------- + +class SetupTableTests(unittest.TestCase): + + def test_common(self): + for kop in 'aAsSxXcpr': + table = _setup_table(kop) + self.assertEqual(type(table), bytes) + self.assertEqual(len(table), 256) + self.assertEqual(table[0], 0) # all tables start with 0 + + def test_add(self): + table = _setup_table('a') + self.assertEqual(max(table), 28) + self.assertTrue(table[255] == sum(range(8)) == 28) + self.assertEqual(table[15], 0+1+2+3) + + table = _setup_table('A') + self.assertEqual(table[15], 4+5+6+7) + + for kop, endian in ('a', 'little'), ('A', 'big'): + t = _setup_table(kop) + for i in range(256): + a = int2ba(i, 8, endian) + self.assertEqual(t[i], sum(i for i, v in enumerate(a) if v)) + + def test_add_sqr(self): + table = _setup_table('s') + self.assertEqual(max(table), 140) + for kop, endian in ('s', 'little'), ('S', 'big'): + t = _setup_table(kop) + for i in range(256): + a = int2ba(i, 8, endian) + self.assertEqual(t[i], + sum(i * i for i, v in enumerate(a) if v)) + + def test_xor(self): + table = _setup_table('x') + self.assertEqual(max(table), 7) # max index is 7 + self.assertTrue(table[255] == 0^1^2^3^4^5^6^7 == 0) + self.assertEqual(table[2], 1) + self.assertTrue(table[29] == table[0b11101] == 0^2^3^4 == 5) + self.assertTrue(table[34] == table[0b100010] == 1^5 == 4) + self.assertTrue(table[157] == table[0b10011101] == 2^3^4^7 == 2) + + table = _setup_table('X') + self.assertEqual(table[2], 6) + self.assertTrue(table[157] == 3^4^5^7 == 5) + + for kop, endian in ('x', 'little'), ('X', 'big'): + t = _setup_table(kop) + for i in range(256): + a = int2ba(i, 8, endian) + c = 0 + for j, v in enumerate(a): + c ^= j * v + self.assertEqual(t[i], c) + + def test_count(self): + table = _setup_table('c') + self.assertEqual(max(table), 8) # 8 active bits the most + self.assertEqual(table[255], 8) + self.assertTrue(table[29] == table[0b11101] == 4) + for endian in 'little', 'big': + for i in range(256): + a = int2ba(i, 8, endian) + self.assertEqual(table[i], a.count()) + + def test_parity(self): + table = _setup_table('p') + self.assertEqual(max(table), 1) + self.assertEqual(table[254], 1) + self.assertEqual(table[255], 0) + for endian in 'little', 'big': + for i in range(256): + a = int2ba(i, 8, endian) + self.assertEqual(table[i], parity(a)) + + def test_reverse(self): + table = _setup_table('r') + self.assertEqual(max(table), 255) + self.assertEqual(table[255], 255) # reversed is still 255 + self.assertEqual(table[1], 128) + for i in range(256): + j = table[i] + self.assertEqual(table[j], i) + self.assertEqual(int2ba(i, 8, 'little'), int2ba(j, 8, 'big')) + + def test_endian(self): + reverse_trans = _setup_table('r') + for kop1, kop2 in 'aA', 'xX': + a = _setup_table(kop1) + b = _setup_table(kop2) + for i in range(256): + j = reverse_trans[i] + self.assertEqual(a[i], b[j]) + + +class ZLW_Tests(unittest.TestCase, Util): + + def test_zeros(self): + for n in range(200): + a = zeros(n, self.random_endian()) + self.assertEqual(_zlw(a), zeros(64)) + + def test_ones(self): + for n in range(200): + a = ones(n, self.random_endian()) + b = _zlw(a) + r = n % 64 + self.assertEqual(b, ones(r) + zeros(64 - r)) + + def test_random(self): + for n in range(200): + a = urandom_2(n) + b = _zlw(a) + self.assertEqual(len(b), 64) + self.assertEqual(a.endian, b.endian) + self.assertEqual(b[63], 0) # last bit is always 0 + q, r = divmod(n, 64) + self.assertEqual(b, a[64 * q:] + zeros(64 - r)) + +# ---------------------------- _bitarray.c -------------------------------- + +class ShiftR8_Tests(unittest.TestCase, Util): + + def test_empty(self): + a = bitarray() + a._shift_r8(0, 0, 3) + self.assertEqual(a, bitarray()) + + def test_explicit(self): + x = bitarray('11000100 11111111 11100111 10111111 00001000') + y = bitarray('11000100 00000111 11111111 00111101 00001000') + x._shift_r8(1, 4, 5) + self.assertEqual(x, y) + x._shift_r8(2, 1, 5) # start > stop -- do nothing + self.assertEqual(x, y) + x._shift_r8(0, 5, 0) # shift = 0 -- do nothing + self.assertEqual(x, y) + + x = bitarray('11000100 11110') + y = bitarray('00011000 10011') + x._shift_r8(0, 2, 3) + self.assertEqual(x, y) + + x = bitarray('1100011') + y = bitarray('0110001') + x._shift_r8(0, 1, 1) + self.assertEqual(x, y) + + def test_random(self): + for _ in range(2000): + n = randrange(200) + x = urandom_2(n) + a = randint(0, x.nbytes) + b = randint(a, x.nbytes) + k = randrange(8) + y = x.copy() + y[8 * a : 8 * b] >>= k + s = x.to01() + if a < b: + s = s[:8 * a] + k * "0" + s[8 * a : 8 * b - k] + s[8 * b:] + if 8 * b > n: + s = s[:n] + x._shift_r8(a, b, k) + self.assertEqual(x.to01(), s) + self.assertEqual(x, y) + self.assertEqual(x.endian, y.endian) + self.assertEqual(len(x), n) + + +class CopyN_Tests(unittest.TestCase, Util): + + def test_explicit(self): + x = bitarray('11000100 11110') + # ^^^^ ^ + y = bitarray('0101110001') + # ^^^^^ + x._copy_n(4, y, 1, 5) + self.assertEqual(x, bitarray('11001011 11110')) + # ^^^^ ^ + x = bitarray('10110111 101', 'little') + y = x.copy() + x._copy_n(3, x, 3, 7) # copy region of x onto x + self.assertEqual(x, y) + x._copy_n(3, bitarray(x, 'big'), 3, 7) # as before but other endian + self.assertEqual(x, y) + x._copy_n(5, bitarray(), 0, 0) # copy empty bitarray onto x + self.assertEqual(x, y) + + def test_example(self): + # example given in devel/copy_n.py + y = bitarray( + '00101110 11111001 01011101 11001011 10110000 01011110 011') + x = bitarray( + '01011101 11100101 01110101 01011001 01110100 10001010 01111011') + x._copy_n(21, y, 6, 31) + self.assertEqual(x, bitarray( + '01011101 11100101 01110101 11110010 10111011 10010111 01101011')) + + def check_copy_n(self, N, M, a, b, n): + x = urandom_2(N) + x_lst = x.tolist() + y = x if M < 0 else urandom_2(M) + y_lst = y.tolist() + x_lst[a:a + n] = y_lst[b:b + n] + x._copy_n(a, y, b, n) + self.assertEqual(x, bitarray(x_lst)) + self.assertEqual(len(x), N) + self.check_obj(x) + + if M < 0: + return + self.assertEqual(y, bitarray(y_lst)) + self.assertEqual(len(y), M) + self.check_obj(y) + + def test_random(self): + for _ in range(1000): + N = randrange(1000) + n = randint(0, N) + a = randint(0, N - n) + b = randint(0, N - n) + self.check_copy_n(N, -1, a, b, n) + + M = randrange(1000) + n = randint(0, min(N, M)) + a = randint(0, N - n) + b = randint(0, M - n) + self.check_copy_n(N, M, a, b, n) + + @staticmethod + def getslice(a, start, slicelength): + # this is the Python eqivalent of __getitem__ for slices with step=1 + b = bitarray(slicelength, a.endian) + b._copy_n(0, a, start, slicelength) + return b + + def test_getslice(self): + for a in self.randombitarrays(): + a_lst = a.tolist() + n = len(a) + i = randint(0, n) + j = randint(i, n) + b = self.getslice(a, i, j - i) + self.assertEqual(b.tolist(), a_lst[i:j]) + self.assertEQUAL(b, a[i:j]) + + +class Overlap_Tests(unittest.TestCase, Util): + + def check_overlap(self, a, b, res): + r1 = a._overlap(b) + r2 = b._overlap(a) + self.assertTrue(r1 is r2 is res) + self.check_obj(a) + self.check_obj(b) + + def test_empty(self): + a = bitarray() + self.check_overlap(a, a, False) + b = bitarray() + self.check_overlap(a, b, False) + + def test_distinct(self): + for a in self.randombitarrays(): + # buffers overlaps with itself, unless buffer is NULL + self.check_overlap(a, a, bool(a)) + b = a.copy() + self.check_overlap(a, b, False) + + def test_shared(self): + a = bitarray(64) + b = bitarray(buffer=a) + self.check_overlap(b, a, True) + + c = bitarray(buffer=memoryview(a)[2:4]) + self.check_overlap(c, a, True) + + d = bitarray(buffer=memoryview(a)[5:]) + self.check_overlap(d, c, False) + self.check_overlap(d, b, True) + + e = bitarray(buffer=memoryview(a)[3:3]) + self.check_overlap(e, c, False) + self.check_overlap(e, d, False) + + def test_shared_random(self): + n = 100 # buffer size in bytes + a = bitarray(8 * n) + for _ in range(1000): + i1 = randint(0, n) + j1 = randint(i1, n) + b1 = bitarray(buffer=memoryview(a)[i1:j1]) + + i2 = randint(0, n) + j2 = randint(i2, n) + b2 = bitarray(buffer=memoryview(a)[i2:j2]) + + r1, r2 = range(i1, j1), range(i2, j2) + res = bool(r1) and bool(r2) and (i2 in r1 or i1 in r2) + self.check_overlap(b1, b2, res) + + +# -------------------------------- _util.c ---------------------------------- + +class CountFromWord_Tests(unittest.TestCase, Util): + + def test_ones_zeros_empty(self): + for _ in range(1000): + n = randrange(1024) + a = ones(n) # ones + i = randrange(20) + self.assertEqual(_cfw(a, i), max(0, n - i * 64)) + a.setall(0) # zeros + self.assertEqual(_cfw(a, i), 0) + a.clear() # empty + self.assertEqual(_cfw(a, i), 0) + + def test_random(self): + for _ in range(1000): + n = randrange(1024) + a = urandom_2(n) + i = randrange(20) + res = _cfw(a, i) + self.assertEqual(res, a[64 * i:].count()) + + +class RTS_Tests(unittest.TestCase): + + # _sc_rts() (running totals debug test) + + def test_segsize(self): + self.assertEqual(type(_SEGSIZE), int) + self.assertTrue(_SEGSIZE in [8, 16, 32]) + + def test_empty(self): + rts = _sc_rts(bitarray()) + self.assertEqual(len(rts), 1) + self.assertEqual(rts, [0]) + + @skipIf(SEGBITS != 256) + def test_example(self): + # see example before sc_calc_rts() in _util.c + a = zeros(987) + a[:5] = a[512:515] = a[768:772] = 1 + self.assertEqual(a.count(), 12) + rts = _sc_rts(a) + self.assertEqual(type(rts), list) + self.assertEqual(len(rts), 5) + self.assertEqual(rts, [0, 5, 5, 8, 12]) + + @staticmethod + def nseg(a): # number of segments, see also SegmentTests in tricks.py + return (a.nbytes + _SEGSIZE - 1) // _SEGSIZE + + def test_ones(self): + for n in range(1000): + a = ones(n) + rts = _sc_rts(a) + self.assertEqual(len(rts), self.nseg(a) + 1) + self.assertEqual(rts[0], 0) + self.assertEqual(rts[-1], n) + for i, v in enumerate(rts): + self.assertEqual(v, min(SEGBITS * i, n)) + + def test_random(self): + for _ in range(200): + a = urandom_2(randrange(10000)) + rts = _sc_rts(a) + self.assertEqual(len(rts), self.nseg(a) + 1) + self.assertEqual(rts[0], 0) + self.assertEqual(rts[-1], a.count()) + for i in range(self.nseg(a)): + seg_pop = a.count(1, SEGBITS * i, SEGBITS * (i + 1)) + self.assertEqual(rts[i + 1] - rts[i], seg_pop) + + +class ReadN_WriteN_Tests(unittest.TestCase, Util): + + # Regardless of machine byte-order, read_n() and write_n() use + # little endian byte-order. + + def test_explicit(self): + for blob, x in [(b"", 0), + (b"\x00", 0), + (b"\x01", 1), + (b"\xff", 255), + (b"\xff\x00", 255), + (b"\xaa\xbb\xcc", 0xccbbaa)]: + n = len(blob) + self.assertEqual(_read_n(iter(blob), n), x) + self.assertEqual(_write_n(n, x), blob) + + def test_zeros(self): + for n in range(PTRSIZE): + blob = n * b"\x00" + self.assertEqual(_read_n(iter(blob), n), 0) + self.assertEqual(_write_n(n, 0), blob) + + def test_max(self): + blob = (PTRSIZE - 1) * b"\xff" + b"\x7f" + self.assertEqual(_read_n(iter(blob), PTRSIZE), sys.maxsize) + self.assertEqual(_write_n(PTRSIZE, sys.maxsize), blob) + + def test_round_trip_random(self): + for _ in range(1000): + n = randint(1, PTRSIZE - 1); + blob = os.urandom(n) + i = _read_n(iter(blob), n) + self.assertEqual(_write_n(n, i), blob) + + def test_read_n_untouch(self): + it = iter(b"\x00XY") + self.assertEqual(_read_n(it, 1), 0) + self.assertEqual(next(it), ord('X')) + self.assertEqual(_read_n(it, 0), 0) + self.assertEqual(next(it), ord('Y')) + self.assertRaises(StopIteration, _read_n, it, 1) + + def test_read_n_item_errors(self): + for v in -1, 256: + self.assertRaises(ValueError, _read_n, iter([3, v]), 2) + + for v in None, "F", Ellipsis, []: + self.assertRaises(TypeError, _read_n, iter([3, v]), 2) + + def test_read_n_negative(self): + it = iter(PTRSIZE * b"\xff") + self.assertRaisesMessage( + ValueError, + "read %d bytes got negative value: -1" % PTRSIZE, + _read_n, it, PTRSIZE) + +# --------------------------------------------------------------------------- + +if __name__ == '__main__': + unittest.main() diff -pruN 2.9.2-1/devel/test_random.py 3.6.1-1/devel/test_random.py --- 2.9.2-1/devel/test_random.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/test_random.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,865 @@ +""" +Statistical Tests for Random Functions in bitarray.util +------------------------------------------------------- + +These are statistical tests. They do not test any basic functionality of +random functions. Those are already tested in the regular utility tests. +Therefore, and because these tests take longer to run, we decided to put +them in a separate file. + +In addition, this file contains some important verification tests that don't +test actual functionality in random_p(), but rather verify some of the logic +and establish some tricky equations. +""" +import sys +import unittest +from copy import deepcopy +from collections import Counter +from itertools import pairwise +from math import comb, fmod, sqrt +from statistics import fmean, stdev, pstdev +from random import randint, randrange, random, binomialvariate + +from bitarray import bitarray, frozenbitarray +from bitarray.util import ( + zeros, ones, urandom, random_k, random_p, sum_indices, + int2ba, count_and, count_or, count_xor, parity, +) +from bitarray.util import _Random # type: ignore + + +HEAVY = False # set True for heavy testing + + +_r = _Random() +M = _r.M +K = _r.K +limit = 1.0 / (K + 1) # lower limit for p +SMALL_P = _r.SMALL_P + + +def count_each_index(arrays): + """ + Given an iterable of bitarrays, count the sums of all bits at each + index and return those counts in a Counter object. + For example, for a returned Counter c, c[2] = 4 means that a sum of 2 + across all bitarrays occurs at 4 indices. + """ + b = bitarray() + n = None # length of each bitarray + for a in arrays: + if n is None: + n = len(a) + elif len(a) != n: + raise ValueError("bitarrays of same length expected") + b.extend(a) + if n is None: + return Counter() + return Counter(b.count(1, i, len(b), n) for i in range(n)) + + +class CountEachIndexTests(unittest.TestCase): + + def test_example(self): + arrays = [bitarray("0011101"), + bitarray("1010100"), + bitarray("1011001")] + # sums: 2032202 + c = count_each_index(arrays) + self.assertEqual(c.total(), 7) # length of each bitarray + self.assertEqual(c[0], 2) + self.assertEqual(c[1], 0) + self.assertEqual(c[2], 4) + self.assertEqual(c[3], 1) + + def test_random(self): + for _ in range(1_000): + m = randrange(10) + n = randrange(10) if m else 0 + arrays = [urandom(n) for _ in range(m)] + c = count_each_index(arrays) + self.assertEqual(c.total(), n) + for j in range(m + 1): + self.assertTrue(0 <= c[j] <= n) + + c2 = Counter(sum(arrays[j][i] for j in range(m)) + for i in range(n)) + self.assertEqual(c, c2) + + # generator + gen = (arrays[j] for j in range(m)) + self.assertEqual(count_each_index(gen), c) + self.assertEqual(list(gen), []) + + def test_empty(self): + arrays = [] + for m in range(10): + self.assertEqual(count_each_index(arrays), Counter()) + arrays.append(bitarray()) + + def test_zeros_ones(self): + for _ in range(1_000): + m = randrange(10) + n = randrange(10) if m else 0 + c = count_each_index(zeros(n) for _ in range(m)) + self.assertEqual(c[0], n) + + c = count_each_index(ones(n) for _ in range(m)) + self.assertEqual(c[m], n) + + def test_errors(self): + C = count_each_index + self.assertRaises(ValueError, C, "ABC") + self.assertRaises(TypeError, C, [0, 1]) + self.assertRaises(ValueError, C, [bitarray("01"), bitarray("1")]) + + +def create_masks(m): + """ + Create a list with m masks. Each mask has a length of 2**m bits. + """ + masks = [] + for i in range(m): + j = 1 << i + mask = zeros(j) + ones(j) + mask *= 1 << (m - i - 1) + masks.append(mask) + return masks + + +class CreateMasksTests(unittest.TestCase): + + def test_explict(self): + C = create_masks + self.assertEqual(C(0), []) + + self.assertEqual(C(1), [bitarray("01")]) + + self.assertEqual(C(2), [bitarray("0101"), + bitarray("0011")]) + + self.assertEqual(C(3), [bitarray("01010101"), + bitarray("00110011"), + bitarray("00001111")]) + + def test_11(self): + m = 11 + masks = create_masks(m) + n = 1 << m + self.assertEqual(len(masks), m) + self.assertEqual(count_each_index(masks), + Counter(int2ba(i).count() for i in range(n))) + for i in range(m): + a = masks[i] + self.assertEqual(len(a), n) + self.assertEqual(a.count(), n // 2) + for j in range(i): + b = masks[j] + self.assertEqual(count_and(a, b), n // 4) + self.assertEqual(count_or(a, b), 3 * n // 4) + self.assertEqual(count_xor(a, b), n // 2) + + +class Util(unittest.TestCase): + + def check_binomial_dist(self, n, p, x): + mu = n * p + sigma = sqrt(n * p * (1.0 - p)) + msg = "n=%d p=%f mu=%f sigma=%f x=%f" % (n, p, mu, sigma, x) + self.assertTrue(abs(x - mu) < 10.0 * sigma, msg) + + def check_probability(self, a, p): + n = len(a) + c = a.count() + if p == 0: + self.assertEqual(c, 0) + elif p == 1: + self.assertEqual(c, n) + else: + self.check_binomial_dist(n, p, c) + + +class UtilTests(Util): + + def test_check_probability(self): + C = self.check_probability + N = 1_000_000 + + a = zeros(N) + C(a, 0.0) + + a.setall(1) + C(a, 1.0) + + a[::2] = 0 + self.assertEqual(a.count(), N // 2) + C(a, 0.501) + C(a, 0.499) + self.assertRaises(AssertionError, C, a, 0.506) + self.assertRaises(AssertionError, C, a, 0.494) + + +class URandomTests(Util): + + def test_count(self): + a = urandom(10_000_000) + self.check_probability(a, 0.5) + + def test_stat(self): + for c in [ + Counter(urandom(100).count() for _ in range(100_000)), + count_each_index(urandom(100_000) for _ in range(100)), + ]: + self.assertTrue(set(c) <= set(range(101))) + self.assertEqual(c.total(), 100_000) + x = sum(c[k] for k in range(40, 51)) + # p = 0.522195 mean = 52219.451858 stdev = 157.958033 + self.assertTrue(abs(x - 52_219) <= 1_580) + + +class Random_K_Tests(Util): + + def test_mean(self): + M = 100_000 # number of trails + N = 1_000 # bitarray length + K = 500 # sample size + C = Counter() + ranges = [0.0, 500.0, 510.0, 520.0, 1000.0] + for _ in range(M): + x = sum_indices(random_k(N, K)) / K + for i, (x1, x2) in enumerate(pairwise(ranges)): + if x1 <= x < x2: + C[i] += 1 + + self.assertEqual(C.total(), M) + # python random/sample.py 100_000 1000 500 0 500 510 520 1000 + self.assertTrue(abs(C[0] - 52_183) <= 1_580) # p = 0.521829 + self.assertTrue(abs(C[1] - 35_303) <= 1_511) # p = 0.353025 + self.assertTrue(abs(C[2] - 11_275) <= 1_000) # p = 0.112747 + self.assertTrue(abs(C[3] - 1_240) <= 350) # p = 0.012399 + + def test_mean_2(self): + M = 100_000 # number of trails + N = 500 # bitarray length + K = 400 # sample size + C = Counter() + ranges = [200.0, 249.5, 251.0, 255.0, 260.0, 300.0] + for _ in range(M): + x = sum_indices(random_k(N, K)) / K + for i, (x1, x2) in enumerate(pairwise(ranges)): + if x1 <= x < x2: + C[i] += 1 + + self.assertEqual(C.total(), M) + # python random/sample.py 100_000 500 400 200 249.5 251 255 260 300 + self.assertTrue(abs(C[0] - 50_000) <= 1_581) # p = 0.500000 + self.assertTrue(abs(C[1] - 17_878) <= 1_212) # p = 0.178781 + self.assertTrue(abs(C[2] - 27_688) <= 1_415) # p = 0.276879 + self.assertTrue(abs(C[3] - 4_376) <= 647) # p = 0.043762 + + def test_apply_masks(self): + Na = 25_000 # number of bitarrays to test against masks + Nm = 12 # number of masks + n = 1 << Nm # length of each mask + # Create masks for selecting half elements in random bitarray a. + # For example, masks[0] selects all odd elements, and masks[-1] + # selects the upper half of a. + masks = create_masks(Nm) + cm = Nm * [0] # counter for each mask + for _ in range(Na): + k = randrange(1, n, 2) # k is odd + a = random_k(n, k) + self.assertEqual(len(a), n) + self.assertTrue(parity(a)) # count is odd + for i in range(Nm): + c1 = count_and(a, masks[i]) + c0 = k - c1 + # counts cannot be equal because k is odd + self.assertNotEqual(c0, c1) + # the probability for having more, e.g. even than + # odd (masks[0]) elements should be 1/2, or having more bits + # in upper vs lower half (mask(-1)) + if c0 > c1: + cm[i] += 1 + + for c in cm: # for each mask, check counter + self.check_binomial_dist(Na, 0.5, c) + + def test_random_masks(self): + Na = 10 # number of arrays to test + Nm = 500_000 if HEAVY else 25_000 # number of masks + n = 7000 # bitarray length + # count for each array + ka = [randrange(1, n, 2) for _ in range(Na)] + arrays = [random_k(n, k) for k in ka] + + for k, a in zip(ka, arrays): # sanity check arrays + self.assertEqual(len(a), n) + self.assertEqual(a.count(), k) + self.assertTrue(parity(a)) + + ca = Na * [0] # counter for each array + for _ in range(Nm): + # each mask has exactly half elements set to 1 + mask = random_k(n, n//2) + self.assertEqual(mask.count(0), mask.count(1)) + + # test each array against this masks + for i in range(Na): + c1 = count_and(arrays[i], mask) + c0 = ka[i] - c1 + # counts cannot be equal because k is odd + self.assertNotEqual(c0, c1) + if c0 > c1: + ca[i] += 1 + + for c in ca: # for each array, check counter + self.check_binomial_dist(Nm, 0.5, c) + + def test_elements_uniform(self): + arrays = [random_k(100_000, 30_000) for _ in range(100)] + for a in arrays: + # for each bitarray check sample size k + self.assertEqual(a.count(), 30_000) + + c = count_each_index(arrays) + self.assertTrue(abs(c[30] - 8_678) <= 890) + x = sum(c[k] for k in range(20, 31)) + # p = 0.540236 mean = 54023.639245 stdev = 157.601089 + self.assertTrue(abs(x - 54_024) <= 1_576) + self.assertEqual(c.total(), 100_000) + + def test_all_bits_active(self): + for _ in range(100): + n = randrange(10, 10_000) + cum = zeros(n) + for _ in range(10_000): + k = n // 7 + a = random_k(n, k) + self.assertEqual(len(a), n) + self.assertEqual(a.count(), k) + cum |= a + if cum.all(): + break + else: + self.fail() + + def test_combinations(self): + # for entire range of 0 <= k <= n, validate that random_k() + # generates all possible combinations + n = 12 + total = 0 + for k in range(n + 1): + expected = comb(n, k) + combs = set() + for _ in range(100_000): + a = random_k(n, k) + self.assertEqual(a.count(), k) + combs.add(frozenbitarray(a)) + if len(combs) == expected: + total += expected + break + else: + self.fail() + self.assertEqual(total, 2 ** n) + + def test_evenly(self): + # Calculate random_k(n, k) N times, and count each specific outcome. + # We know that there are m=comb(n, k) possible outcomes, so each one + # has a probability 1/m and the mean of the count should be N/m. + N = 100_000 + n = 9 + k = 3 + m = comb(n, k) + c = Counter() + for _ in range(N): + a = frozenbitarray(random_k(n, k)) + c[a] += 1 + self.assertEqual(c.total(), N) + self.assertEqual(len(c), m) + p = 1.0 / m + self.assertAlmostEqual(fmean(c.values()), N * p) + if 0: + print(m) + print(N * p) + print(sqrt(N * p * (1.0 - p))) + print(stdev(c.values())) + for x in c.values(): + self.check_binomial_dist(N, p, x) + + def random_p_alt(self, n, p=0.5): + """ + Alternative implementation of random_p(). While the performance is + about the same for large n, we found that for smaller n the handling + of special cases leads to better overall performance in the current + implementation. + """ + k = binomialvariate(n, p) + self.assertTrue(0 <= k <= n) + a = random_k(n, k) + self.assertEqual(len(a), n) + self.assertEqual(a.count(), k) + return a + + def test_random_p_alt(self): + n = 1_000_000 + for _ in range(100): + p = random() + a = self.random_p_alt(n, p) + self.check_probability(a, p) + + +class Random_P_Tests(Util): + + def test_apply_masks(self): + M = 12 # number of masks + # Create masks for selecting half elements in the random bitarray a. + # For example, masks[0] selects all odd elements, and masks[-1] + # selects the upper half of a. + masks = create_masks(M) + n = M * [0] # sample size for each mask + c = M * [0] # count for each mask + for _ in range(25_000): + p = 1.5 * SMALL_P * random() + a = random_p(1 << M, p) + tot = a.count() + for i in range(M): + c1 = count_and(a, masks[i]) + c0 = tot - c1 + if c0 == c1: # counts are equal -> + continue # ignore this mask for this bitarray a + n[i] += 1 + # counts are not equal, the probability for having more, + # e.g. even than odd (masks[0]) elements should be 1/2, + # or having more bits in upper vs lower half (mask(-1)) + if c0 > c1: + c[i] += 1 + + for i in range(M): + self.assertTrue(n[i] > 20_000, n[i]) + self.check_binomial_dist(n[i], 0.5, c[i]) + + def test_elements_uniform(self): + arrays = [random_p(100_000, 0.3) for _ in range(100)] + for a in arrays: + # for each bitarray see if population is within expectation + self.check_probability(a, 0.3) + + c = count_each_index(arrays) + self.assertTrue(abs(c[30] - 8_678) <= 890) + x = sum(c[k] for k in range(20, 31)) + # p = 0.540236 mean = 54023.639245 stdev = 157.601089 + self.assertTrue(abs(x - 54_024) <= 1_576) + self.assertEqual(c.total(), 100_000) + + def test_tiny_p(self): + for n in 4, 10, 1000: + for p in 1e-9, 1e-12, 1e-15, 1e-18: + a = random_p(n, p) + self.assertTrue(a.count() <= 1) + + def test_literal(self): + # test "literal definition" case, n = 5 + M = 250_000 # number of trails + C = Counter(random_p(5, 0.3).count() for _ in range(M)) + self.assertEqual(C.total(), M) + # python random/binomial.py 250_000 5 0.3 + self.assertTrue(abs(C[0] - 42_017) <= 1_870) # p = 0.168070 + self.assertTrue(abs(C[1] - 90_037) <= 2_400) # p = 0.360150 + self.assertTrue(abs(C[2] - 77_175) <= 2_310) # p = 0.308700 + self.assertTrue(abs(C[3] - 33_075) <= 1_694) # p = 0.132300 + self.assertTrue(abs(C[4] - 7_087) <= 830) # p = 0.028350 + + def test_small_p(self): + # test small p case + C = Counter(random_p(50, p=0.005).count() for _ in range(100_000)) + self.assertEqual(C.total(), 100_000) + # python random/binomial.py 100_000 50 .005 + self.assertTrue(abs(C[0] - 77_831) <= 1_314) # p = 0.778313 + self.assertTrue(abs(C[1] - 19_556) <= 1_254) # p = 0.195556 + + def test_small_p_symmetry(self): + # same as above - exploiting symmetry + C = Counter(random_p(50, p=0.995).count() for _ in range(100_000)) + self.assertEqual(C.total(), 100_000) + self.assertTrue(abs(C[49] - 19_556) <= 1_254) + self.assertTrue(abs(C[50] - 77_831) <= 1_314) + + def test_small_p_uniform(self): + C = count_each_index(random_p(100_000, 0.005) for _ in range(50)) + self.assertEqual(C.total(), 100_000) + self.assertTrue(abs(C[0] - 77_831) <= 1_314) + self.assertTrue(abs(C[1] - 19_556) <= 1_254) + + def test_p375(self): + # test .combine_half() + M = 100_000 # number of trails + C = Counter(random_p(100, 0.375).count() for _ in range(M)) + self.assertEqual(C.total(), M) + # python random/binomial.py 100_000 100 .375 37..48 + self.assertTrue(abs(C[36] - 7_898) <= 853) # p = 0.078977 + self.assertTrue(abs(C[37] - 8_196) <= 867) # p = 0.081965 + self.assertTrue(abs(C[38] - 8_153) <= 865) # p = 0.081533 + self.assertTrue(abs(C[39] - 7_777) <= 847) # p = 0.077770 + x = sum(C[k] for k in range(37, 49)) + self.assertTrue(abs(x - 56_614) <= 1_567) # p = 0.566139 + + def test_ne5(self): + M = 25_000 # number of trails + C = Counter(random_p(100_000, 0.5).count() for _ in range(M)) + self.assertEqual(C.total(), M) + # python binomial.py 25_000 100_000 .5 48_000..50_000 50_000..50_200 + x = sum(C[k] for k in range(48000, 50001)) + self.assertTrue(abs(x - 12_532) <= 791) # p = 0.501262 + x = sum(C[k] for k in range(50000, 50201)) + self.assertTrue(abs(x - 9_972) <= 774) # p = 0.398876 + + def test_probabilities(self): + n = 100_000_000 + special_p = [ + 65 / 257 - 1e-9, # largest x for OR + 65 / 257 + 1e-9, # smallest x for AND + 0.0, 1e-12, 0.25, 1/3, 3/8, 127/257, 0.5, + ] + for j in range(100 if HEAVY else 2): + sys.stdout.write('.') + sys.stdout.flush() + try: + p = special_p[j] + except IndexError: + p = random() + + a = random_p(n, p) + self.check_probability(a, p) + + +class VerificationTests(Util): + + def test_uniform_stdev(self): + # verify that the standard deviation of a uniform distribution + # of population size n is given by: n / sqrt(12) + for _ in range(100): + n = randrange(10, 10_000) + pop = list(range(n)) + self.assertEqual(fmean(pop), (n - 1) / 2) + self.assertAlmostEqual(pstdev(pop), n / sqrt(12), delta=0.1) + + def test_operations(self): + C = self.check_probability + n = 1_000_000 + values = [i / 16.0 for i in range(17)] + arrays0, arrays1 = ([(random_p(n, p), p) for p in values] + for _ in range(2)) + + for a, p in arrays0: + C(a, p) + C(~a, 1.0 - p) # invert + + for b, q in arrays1: + C(b, q) + + C(a & b, p * q) # AND + C(a | b, p + q - p * q) # OR + C(a ^ b, p + q - 2 * p * q) # XOR + + for b, q in arrays0: + C(b, q) + for a, p in deepcopy(arrays1): + C(a, p) + + a &= b # in-place AND + p *= q + C(a, p) + + for a, p in deepcopy(arrays1): + C(a, p) + + a |= b # in-place OR + p += q * (1.0 - p) + C(a, p) + + for a, p in deepcopy(arrays1): + C(a, p) + + a ^= b # in-place XOR + p += q * (1.0 - 2 * p) + C(a, p) + + # ---------------- verifications relevant for random_k() ---------------- + + def test_decide_on_sequence(self): + N = 100_000 + cdiff = Counter() + + for _ in range(N): + n = randrange(1, 10_000) + k = randint(0, n // 2) + self.assertTrue(0 <= k <= n // 2) + + if k < 16 or k * K < 3 * n: + # for small k, we increase the count of a zeros(n) bitarray + i = 0 + else: + # We could simply have `i = int(k / n * K)`. However, + # when k is small, many reselections are required to + # decrease the count. On the other hand, for k near n/2, + # increasing and decreasing the count is equally expensive. + p = k / n # p <= 0.5 + # Numerator: f(p)=(1-2*p)*c -> f(0)=c, f(1/2)=0 + # As the standard deviation of the .combine_half() bitarrays + # gets smaller with larger n, we divide by sqrt(n). + p -= (0.2 - 0.4 * p) / sqrt(n) + # Note that we divide by K+1. This will round towards the + # nearest probability as we get closer to p = 1/2. + i = int(p * (K + 1)) + + if i < 3: + # a = zeros(n), count is 0 + diff = -k + else: + self.assertTrue(k >= 16) + self.assertTrue(n >= 32) + self.assertTrue(3 <= i <= K // 2) + # a = self.combine_half(self.op_seq(i)) + # count is given by binomialvariate(n, i / K) + diff = binomialvariate(n, i / K) - k + + cdiff[diff] += 1 + + self.assertEqual(cdiff.total(), N) + # count the number of cases where the count needs to be decreased + above = sum(cdiff[i] for i in range(1, max(cdiff) + 1)) + self.assertTrue(M != 8 or 0.28 < above / N < 0.34) + + # ---------------- verifications relevant for random_p() ---------------- + + def test_equal_x(self): + """ + Verify that the probabilities p for which final AND and OR result in + equal x are: p = j / (K + 1) j in range(1, K) + Also, verify these x are all: x = 1 / (K + 1) = limit + These are also the maximal x. + """ + for j in range(1, K): + # probabilities p for which final AND and OR result in equal x + p = j / (K + 1) + i = int(p * K) + self.assertEqual(i, j - 1) # as K / (K + 1) < 1 + self.assertEqual(p * (K + 1), i + 1) + q = i / K + x1 = (p - q) / (1.0 - q) # OR + x2 = 1.0 - p / (q + 1.0 / K) # AND x2 = 1 - p / next q + self.assertAlmostEqual(x1, x2, delta=1e-14) + self.assertAlmostEqual(x1, limit, delta=1e-14) + + def special_p(self): + """ + generate special test values of p < 0.5 + """ + EPS = 1e-12 + + for j in range(1, K // 2 + 1): + # probabilities for which final AND and OR result in equal x + p = j / (K + 1) + for e in -EPS, EPS: + yield p + e + + for j in range(1, K // 2): + # probabilities for which no final AND or OR is not necessary + p = j / K + for e in -EPS, 0.0, EPS: + yield p + e + + for p in 0.0, EPS, 0.5 - EPS: + yield p + + for e in -EPS, 0.0, EPS: + yield SMALL_P + e + + for _ in range(10_000): + yield 0.5 * random() + + def test_decide_on_operation(self): + """ + Verify that `x1 > x2` equates to `p * (K + 1) > i + 1`. + """ + for p in self.special_p(): + self.assertTrue(0 <= p < 0.5, p) + + i = int(p * K) + q = i / K + self.assertTrue(q <= p) + x1 = (p - q) / (1.0 - q) # OR + x2 = 1.0 - p / (q + 1.0 / K) # AND x2 = 1 - p / next q + # decided whether to use next i (next q) + self.assertEqual(x1 > x2, + p * (K + 1) > i + 1) + + def test_decision_limit(self): + """ + Verify that decision operation works as desired, and that resulting + probability q is within limit of p. + """ + # limit = 1/(K+1) is slightly smaller than 1/K: + self.assertEqual(limit, 1.0 / K - 1.0 / (K * (K + 1))) + self.assertTrue(1.0 / K - limit < K ** -2 == 1.0 / (1 << (2 * M))) + + for p in self.special_p(): + i = int(p * K) + q0 = i / K + q1 = (i + 1) / K + self.assertTrue(q0 <= p < q1) + self.assertTrue(q1 - q0 == 1.0 / K > limit) + self.assertTrue(q0 + 0.5 * limit < q1 - 0.5 * limit) + + if p * (K + 1) > i + 1: + self.assertTrue(q1 - 0.5 * limit < p < q1) + # implies: + self.assertNotEqual(q0, p) + q = q1 + self.assertTrue(q > p) # use AND operation + else: + self.assertTrue(q0 <= p < q0 + limit) + q = q0 + self.assertTrue(q <= p) # use OR operation + + self.assertTrue(p - limit < q < p + 0.5 * limit) + self.assertTrue(abs(p - q) < limit) + self.assertEqual(bool(q != p), bool(fmod(p, 1.0 / K))) + + def test_final_op(self): + """ + Verify final operation always gives us the correct probability. + """ + for p in self.special_p(): + i = int(p * K) + if p * (K + 1) > i + 1: # see above + i += 1 + + if p > limit: + self.assertNotEqual(i, 0) + # Note that all the below handles this case fine. + # However, rather than extending .op_seq() and .combine_half() + # to handle i=0, we decided to "filter out" i=0 by the small p + # case (see test below). + self.assertTrue(0 <= i <= K // 2) + + q = i / K + self.assertTrue(abs(p - q) < limit) # see above + + if q < p: # increase probability - OR + x = (p - q) / (1.0 - q) + # ensure small p case is called + self.assertTrue(0.0 < x < limit) + q += x * (1.0 - q) # OR + elif q > p: # decrease probability - AND + x = p / q + # ensure small p case is called (after symmetry is exploited) + self.assertTrue(0.0 < 1.0 - x < limit) + q *= x # AND + self.assertEqual(q, p) + + def test_i_not_0(self): + """ + Verify that for `p > limit`, we always get `i > 0`. + This is important, as the small p case has to "filter out" `i = 0`, + as the sequence of operations do not handle `i = 0`. + """ + p = limit + 1e-12 + i = int(p * K) + self.assertEqual(i, 0) # as K / (K + 1) < 1 + if p * (K + 1) > i + 1: + i += 1 + # So for i be non-zero we must have: + # p * (K + 1) > 1 + # or + # p > 1 / (K + 1) = limit q.e.d. + self.assertEqual(i, 1) + + def dummy_random_p(self, p=0.5, verbose=False): + """ + Unlike random_p(), this function returns the desired probability q + itself, and not a random bitarray. The point of this function is to + illustrate how random_p() essentially works. + Instead of actual bitarray operations, we change q accordingly. + This method is neither concerned with the bitarray length n nor + endianness. + """ + # error check inputs and handle edge cases + if p <= 0.0 or p == 0.5 or p >= 1.0: + if p in (0.0, 0.5, 1.0): + return p + raise ValueError("p must be in range 0.0 <= p <= 1.0, got %f", p) + + # exploit symmetry to establish: p < 0.5 + if p > 0.5: + return 1.0 - self.dummy_random_p(1.0 - p, verbose) + + # for small p set randomly individual bits, which is much faster + if p < SMALL_P: + return p # random.binomialvariate() and .random_pop() + + # calculate operator sequence + i = int(p * K) + if p * (K + 1) > i + 1: + i += 1 + self.assertTrue(0 < i <= K // 2) + a = bitarray(i.to_bytes(2, byteorder="little"), "little") + seq = a[a.index(1) + 1 : M] + + # combine random bitarrays using bitwise AND and OR operations + q = 0.5 # start with randbytes() + for k in seq: + if k: + q += 0.5 * (1.0 - q) # OR + else: + q *= 0.5 # AND + self.assertEqual(q, i / K) + + x = 0.0 + if q < p: # increase probability + x = (p - q) / (1.0 - q) + self.assertTrue(0.0 < x < SMALL_P) + q += x * (1.0 - q) # OR + elif q > p: # decrease probability + x = p / q + self.assertTrue(0.0 < 1.0 - x < SMALL_P) + q *= x # AND + + if verbose: + print("%15.9f %9d %9d %15.9f" % (p, len(seq) + 1, i, x)) + self.assertEqual(q, p) + return q + + def test_dummy_random_p(self): + for p in self.special_p(): + self.assertEqual(self.dummy_random_p(p), p) + + # test 0 <= p < 1; self.special_p() only gives us 0 <= p < 0.5 + for _ in range(10_000): + p = random() + self.assertEqual(self.dummy_random_p(p), p) + +def disp(): + i = sys.argv.index('--disp') + args = sys.argv[i + 1:] + if args: + plist = [float(eval(s)) for s in args] + else: + plist = [1/4, 1/8, 1/16, 1/32, 1/64, 3/128, 127/256, + SMALL_P, 0.1, 0.2, 0.3, 0.4, + 65/257, 127/257 + 1e-9, 0.5 - 1e-9] + print(" p k i x") + print(55 * '-') + for p in plist: + VerificationTests().dummy_random_p(p, True) + + +if __name__ == '__main__': + if '--disp' in sys.argv: + disp() + sys.exit() + if "--heavy" in sys.argv: + HEAVY = True + sys.argv.remove("--heavy") + unittest.main() diff -pruN 2.9.2-1/devel/test_sum_indices.py 3.6.1-1/devel/test_sum_indices.py --- 2.9.2-1/devel/test_sum_indices.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/test_sum_indices.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,274 @@ +""" +In both ssqi() (in _util.c) and sum_indices() (in util.py), we divide our +bitarray into equally sized blocks in order to calculate the sum of active +indices. We use the same trick but for different reasons: + + (a) in ssqi(), we want to loop over bytes (blocks of 8 bits) and use + lookup tables (for sum z_j [**2]) + + (b) in sum_indices() we want to loop over blocks of smaller bitarrays + in order to keep the summation in ssqi() from overflowing + +The trick is to write + + x_j = y_j + z_j where y_j = y : if bit j is active + 0 : otherwise + +for each block. Here, j is the index within each block. +That is, j is in range(block size). +Using the above, we get: + + sum x_j = k * y + sum z_j + +where k is the bit count (per block). And: + + sum x_j**2 = k * y**2 + 2 * sum z_j * y + sum z_j**2 + +These are the sums for each block and their sum (over all blocks) is what +we are interested in. + + (a) ssqi() (b) sum_indices() +------------------------------------------------------------ +block c (char) block (bitarray) +block size 8 n +i byte index block index +y 8 * i n * i +k count_table[c] block.count() +z1 = sum z_j sum_table[c] _ssqi(block) +z2 = sum z_j**2 sum_sqr_table[c] _ssqi(block, 2) +""" +import unittest +from random import getrandbits, randint, randrange, sample + +from bitarray.util import zeros, ones, urandom, _ssqi, sum_indices +from bitarray.test_util import SumIndicesUtil + + +N19 = 1 << 19 # 512 Kbit = 64 KB +N20 = 1 << 20 # 1 Mbit = 128 KB +N21 = 1 << 21 # 2 Mbit = 256 KB +N22 = 1 << 22 # 4 Mbit = 512 KB +N23 = 1 << 23 # 8 Mbit = 1 MB +N28 = 1 << 28 # 256 Mbit = 32 MB +N30 = 1 << 30 # 1 Gbit = 128 MB +N31 = 1 << 31 # 2 Gbit = 256 MB +N32 = 1 << 32 # 4 Gbit = 512 MB +N33 = 1 << 33 # 8 Gbit = 1 GB + +MAX_UINT64 = (1 << 64) - 1 + + +def sum_range(n): + "Return sum(range(n))" + return n * (n - 1) // 2 + +def sum_sqr_range(n): + "Return sum(i * i for i in range(n))" + return n * (n - 1) * (2 * n - 1) // 6 + + +class SumRangeTests(unittest.TestCase): + + def test_sum_range(self): + for n in range(1000): + self.assertEqual(sum_range(n), sum(range(n))) + + def test_sum_sqr_range(self): + for n in range(1000): + self.assertEqual(sum_sqr_range(n), sum(i * i for i in range(n))) + + def test_mode(self): + for n in range(1000): + for mode, f in [(1, sum_range), + (2, sum_sqr_range)]: + sum_ones = 3 if mode == 1 else 2 * n - 1 + sum_ones *= n * (n - 1) + sum_ones //= 6 + self.assertEqual(sum_ones, f(n)) + + def test_o2(self): + for n in range(1000): + o1 = n * (n - 1) // 2 + o2, r = divmod(o1 * (2 * n - 1), 3) + self.assertEqual(r, 0) + self.assertEqual(o2, sum_sqr_range(n)) + + +class ExampleImplementationTests(unittest.TestCase): + + def sum_indices(self, a, mode=1): + n = 503 # block size in bits + nblocks = (len(a) + n - 1) // n # number of blocks + sm = 0 + for i in range(nblocks): + y = n * i + block = a[y : y + n] + + k = block.count() + z1 = _ssqi(block) + self.assertEqual( + # Note that j are indices within each block. + # Also note that we use len(block) instead of block_size, + # as the last block may be truncated. + z1, sum(j for j in range(len(block)) if block[j])) + + if mode == 1: + x = k * y + z1 + else: + z2 = _ssqi(block, 2) + x = (k * y + 2 * z1) * y + z2 + + # x is the sum [of squares] of indices for each block + self.assertEqual( + # Note that here t are indices of the full bitarray a. + x, sum(t ** mode for t in range(y, y + len(block)) if a[t])) + + sm += x + + return sm + + def test_sum_indices(self): + for _ in range(100): + n = randrange(10_000) + a = urandom(n) + mode = randint(1, 2) + self.assertEqual(self.sum_indices(a, mode), sum_indices(a, mode)) + + +class SSQI_Tests(SumIndicesUtil): + + # Note carefully that the limits that are calculated and tested here + # are limits used in internal function _ssqi(). + # The public Python function sum_indices() does NOT impose any limits + # on the size of bitarrays it can compute. + + def test_calculate_limits(self): + # calculation of limits used in ssqi() (in _util.c) + for f, limit in [(sum_range, 6_074_001_000), + (sum_sqr_range, 3_810_778)]: + lo = 0 + hi = MAX_UINT64 + while hi > lo + 1: + n = (lo + hi) // 2 + if f(n) > MAX_UINT64: + hi = n + else: + lo = n + self.assertTrue(f(n) < MAX_UINT64) + self.assertTrue(f(n + 1) > MAX_UINT64) + self.assertEqual(n, limit) + + def test_overflow(self): + # _ssqi() is limited to bitarrays of about 6 Gbit (4 Mbit mode=2). + # This limit is never reached because sum_indices() uses + # a much smaller block size for practical reasons. + for mode, f, n in [(1, sum_range, 6_074_001_000), + (2, sum_sqr_range, 3_810_778)]: + a = ones(n) + self.assertTrue(f(len(a)) <= MAX_UINT64) + self.assertEqual(_ssqi(a, mode), f(n)) + a.append(1) + self.assertTrue(f(len(a)) > MAX_UINT64) + self.assertRaises(OverflowError, _ssqi, a, mode) + + def test_sparse(self): + for _ in range(500): + n = randint(2, 3_810_778) + k = randrange(min(1_000, n // 2)) + mode = randint(1, 2) + freeze = getrandbits(1) + inv = getrandbits(1) + self.check_sparse(_ssqi, n, k, mode, freeze, inv) + + +class SumIndicesTests(SumIndicesUtil): + + def test_urandom(self): + self.check_urandom(sum_indices, 1_000_003) + + def test_random_sample(self): + n = N31 + for k in 1, 31, 503: + mode = randint(1, 2) + freeze = getrandbits(1) + inv = getrandbits(1) + self.check_sparse(sum_indices, n, k, mode, freeze, inv) + + def test_ones(self): + for m in range(19, 32): + n = randrange(1 << m) + mode = randint(1, 2) + freeze = getrandbits(1) + self.check_sparse(sum_indices, n, 0, mode, freeze, inv=True) + + def test_sum_random(self): + for _ in range(50): + n = randrange(1 << randrange(19, 32)) + k = randrange(min(1_000, n // 2)) + mode = randint(1, 2) + freeze = getrandbits(1) + inv = getrandbits(1) + self.check_sparse(sum_indices, n, k, mode, freeze, inv) + + +class VarianceTests(unittest.TestCase): + + def variance(self, a, mu=None): + si = sum_indices(a) + k = a.count() + if mu is None: + mu = si / k + return (sum_indices(a, 2) - 2 * mu * si) / k + mu * mu + + def variance_values(self, values, mu=None): + k = len(values) + if mu is None: + mu = sum(values) / k + return sum((x - mu) ** 2 for x in values) / k + + def test_variance(self): + for _ in range(1_000): + n = randrange(1, 1_000) + k = randint(1, max(1, n // 2)) + indices = sample(range(n), k) + a = zeros(n) + a[indices] = 1 + mean = sum(indices) / len(indices) + self.assertAlmostEqual(self.variance(a), + self.variance_values(indices)) + self.assertAlmostEqual(self.variance(a, mean), + self.variance_values(indices, mean)) + mean = 20.5 + self.assertAlmostEqual(self.variance(a, mean), + self.variance_values(indices, mean)) + + +def test_ones(): + + for n in [3_810_778, + 3_810_779, + 6_074_001_000, + 6_074_001_001, + N33, 2 * N33]: + a = ones(n) + print("n = %32d %6.2f Gbit %6.2f GB" % (n, n / N30, n / N33)) + print("2^64 = %32d" % (1 << 64)) + res = sum_indices(a) + print("sum = %32d" % res) + assert res == sum_range(n) + + res = sum_indices(a, 2) + print("sum2 = %32d" % res) + assert res == sum_sqr_range(n) + + print() + + print("OK") + + +if __name__ == "__main__": + import sys + if '--ones' in sys.argv: + test_ones() + sys.exit() + unittest.main() diff -pruN 2.9.2-1/devel/tricks.py 3.6.1-1/devel/tricks.py --- 2.9.2-1/devel/tricks.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/devel/tricks.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,249 @@ +""" +This file contains some little tricks and verifications for some code which +is used in the C implementation of bitarray. +""" +from random import randint +import unittest + + +# ---------------------------- Range checks --------------------------------- + +class RangeTests(unittest.TestCase): + + def test_check_simple(self): + r = range(0, 256) + for k in range(-10, 300): + self.assertEqual(k < 0 or k > 0xff, bool(k >> 8)) + self.assertEqual(k not in r, bool(k >> 8)) + + def test_check(self): + # used in various places in C code + for i in range(0, 11): + m = 1 << i + for k in range(-10, 2000): + res1 = k not in range(0, m) + res2 = k < 0 or k >= m + self.assertEqual(res1, res2) + # simply shift i to right and see if anything remains + res3 = bool(k >> i) + self.assertEqual(res1, res3) + + def test_check_2(self): + # this is used in _util.c in set_count() + for i in range(0, 11): + m = 1 << i + for k in range(-10, 2000): + res1 = k not in range(0, m + 1) + res2 = k < 0 or k > m + self.assertEqual(res1, res2) + # same as above but combined with k substracted by 1 + res3 = bool(k >> i) and bool((k - 1) >> i) + self.assertEqual(res1, res3) + +# ------------------------------ Slicing ------------------------------------ + +def adjust_step_positive(slicelength, start, stop, step): + """ + This is the adjust_step_positive() implementation from bitarray.h. + """ + if step < 0: + stop = start + 1 + start = stop + step * (slicelength - 1) - 1 + step = -step + + assert start >= 0 and stop >= 0 + assert step > 0 + assert slicelength >= 0 + if slicelength == 0: + assert stop <= start + elif step == 1: + assert stop - start == slicelength + + return start, stop, step + + +def slicelength(start, stop, step): + """ + This is the slicelength implementation from PySlice_AdjustIndices(). + + a / b does integer division. If either a or b is negative, the result + depends on the compiler (rounding can go toward 0 or negative infinity). + Therefore, we are careful that both a and b are always positive. + """ + if step < 0: + if stop < start: + return (start - stop - 1) // (-step) + 1 + else: + if start < stop: + return (stop - start - 1) // step + 1 + return 0 + + +class ListSliceTests(unittest.TestCase): + + def random_slices(self, max_len=100, repeat=10_000): + for _ in range(repeat): + n = randint(0, max_len) + s = slice(randint(-n - 2, n + 2), + randint(-n - 2, n + 2), + randint(-5, 5) or 1) + yield n, s, range(n)[s] + + def test_basic(self): + for n, s, r in self.random_slices(): + self.assertEqual(range(*s.indices(n)), r) + + def test_indices(self): + for n, s, r in self.random_slices(): + start, stop, step = s.indices(n) + self.assertEqual(start, r.start) + self.assertEqual(stop, r.stop) + self.assertEqual(step, r.step) + + self.assertNotEqual(step, 0) + if step > 0: + self.assertTrue(0 <= start <= n) + self.assertTrue(0 <= stop <= n) + else: + self.assertTrue(-1 <= start < n) + self.assertTrue(-1 <= stop < n) + self.assertEqual(range(start, stop, step), r) + + def test_list_get(self): + for n, s, r in self.random_slices(): + a = list(range(n)) + b = a[s] + self.assertEqual(len(b), len(r)) + self.assertEqual(b, list(r)) + + def test_list_set(self): + for n, s, r in self.random_slices(20): + a = n * [None] + b = list(a) + a[s] = range(len(r)) + for i, j in enumerate(r): + b[j] = i + self.assertEqual(a, b) + + def test_list_del(self): + for n, s, r in self.random_slices(): + a = list(range(n)) + b = list(a) + del a[s] + self.assertEqual(len(a), n - len(r)) + for i in sorted(r, reverse=True): + del b[i] + self.assertEqual(a, b) + + def test_adjust_step_positive(self): + for n, s, r in self.random_slices(): + if s.step < 0: + r = r[::-1] + + start, stop, step = adjust_step_positive(len(r), *s.indices(n)) + + self.assertEqual(range(start, stop, step), r) + self.assertTrue(step > 0) + if r: + self.assertTrue(0 <= start < n) + self.assertTrue(0 < stop <= n) + + def test_slicelength(self): + for n, s, r in self.random_slices(): + self.assertEqual(slicelength(r.start, r.stop, r.step), len(r)) + +# ------------------------- Modular Arithmetic ------------------------------ + +class ModularTests(unittest.TestCase): + + def test_remainder(self): + for _ in range(1000): + a = randint(-20, 20) + b = randint(1, 20) + # integer division in Python returns the floor of the result + # instead of truncating towards zero like C + q = a // b + if a < 0: + self.assertTrue(q < 0) + r = a % b + self.assertEqual(b * q + r, a) + self.assertTrue(0 <= r < b) + + def test_avoid_neg_numerator(self): + # + # equality: a % b = (b - (-a) % b) % b + # + for _ in range(1000): + a = randint(-20, 20) + b = randint(1, 20) + r = a % b + # Note that even though a may be negative, the remainder is + # always positive: + self.assertTrue(r >= 0) + # The following equality: + s = (b - (-a) % b) % b + self.assertEqual(s, r) + # can be used to implement a % b in C when a <= 0 + if a <= 0: + # here % always operates on positive numerator + self.assertTrue(-a >= 0) + self.assertTrue(b - (-a) % b > 0) + +# ----------------------------- Segments ------------------------------------ + +class SegmentTests(unittest.TestCase): + + def test_nseg(self): + SEGSIZE = 32 # segment size in bytes + SEGBITS = 8 * SEGSIZE + for nbits in range(1000): + nbytes = (nbits + 7) // 8 + + # number of segments in terms of bytes + nseg = (nbytes + SEGSIZE - 1) // SEGSIZE + + # and in terms of bits + self.assertEqual((nbits + SEGBITS - 1) // SEGBITS, nseg) + + # number of complete segments + cseg = nbits // SEGBITS + self.assertTrue(cseg <= nseg) + + # The number of complete segments cannot be calculated in terms + # of bytes, as it isn't possible to tell how many bits are + # actually used within the last byte of each segment. + if (nbits % SEGBITS > SEGBITS - 8): + self.assertNotEqual(nbytes // SEGSIZE, cseg) + else: + self.assertEqual(nbytes // SEGSIZE, cseg) + + # remaining bits + rbits = nbits % SEGBITS + self.assertEqual(cseg * SEGBITS + rbits, nbits) + if cseg == nseg: + self.assertEqual(rbits, 0) + self.assertEqual(nbytes % SEGSIZE, 0) + else: + self.assertEqual(nseg, cseg + 1) + self.assertTrue(rbits > 0) + +# ------------------------ Variable Length Format --------------------------- + +class VLFTests(unittest.TestCase): + + def test_padding(self): + LEN_PAD_BITS = 3 + for nbits in range(1000): + n = (nbits + LEN_PAD_BITS + 6) // 7 # number of resulting bytes + padding = 7 * n - LEN_PAD_BITS - nbits + self.assertTrue(0 <= padding < 7) + self.assertEqual(divmod(nbits + padding + LEN_PAD_BITS, 7), + (n, 0)) + + # alternative equation for padding + padding_2 = (7 - (nbits + LEN_PAD_BITS) % 7) % 7 + self.assertEqual(padding_2, padding) + + +if __name__ == '__main__': + unittest.main() diff -pruN 2.9.2-1/doc/bitarray3.rst 3.6.1-1/doc/bitarray3.rst --- 2.9.2-1/doc/bitarray3.rst 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/doc/bitarray3.rst 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,27 @@ +Bitarray 3 transition +===================== + +The bitarray version 3 release is bitarray's farewell to Python 2. +Apart from removing Python 2 support, this release also migrates +bitarray's ``.decode()`` and ``.search()`` methods to return iterators. +This is similar to how Python's ``dict.keys()``, ``.values()`` +and ``.items()`` methods were revamped in the Python 2 to 3 transition. + +In the following table, ``a`` is assumed to a bitarray object. + ++----------------------+----------------------+ +| before version 3 | version 3 | ++======================+======================+ +| ``a.iterdecode()`` | ``a.decode()`` | ++----------------------+----------------------+ +| ``a.decode()`` | ``list(a.decode()`` | ++----------------------+----------------------+ +| ``a.itersearch()`` | ``a.search()`` | ++----------------------+----------------------+ +| ``a.search()`` | ``list(a.search()`` | ++----------------------+----------------------+ + +Aside from these changes which will make bitarray 3 more pythonic, there +are a few other minor changes (see changelog). +It should be emphasized that in most common use cases the bitarray 3 +transition will require only minor code changes, or no changes at all. diff -pruN 2.9.2-1/doc/canonical.rst 3.6.1-1/doc/canonical.rst --- 2.9.2-1/doc/canonical.rst 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/doc/canonical.rst 2025-08-12 08:35:42.000000000 +0000 @@ -63,7 +63,7 @@ Encode a message using this code: >>> a.encode(codedict, msg) >>> a bitarray('01011001110011110101100') - >>> assert ''.join(a.iterdecode(codedict)) == msg + >>> assert ''.join(a.decode(codedict)) == msg And now decode using not ``codedict``, but the canonical decoding tables ``count`` and ``symbol`` instead: diff -pruN 2.9.2-1/doc/changelog.rst 3.6.1-1/doc/changelog.rst --- 2.9.2-1/doc/changelog.rst 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/doc/changelog.rst 2025-08-12 08:35:42.000000000 +0000 @@ -1,6 +1,166 @@ Change log ========== +**3.6.1** (2025-08-12): + +* add development files for statistical tests in ``devel/random/`` +* optimize ``util.sum_indices()`` +* fix RecursionError in ``util.random_k()``, see `#239 `__ +* add ``devel/test_sum_indices.py`` + + +**3.6.0** (2025-07-29): + +* add ``util.random_k()``, see `#237 `__ +* add ``util.sum_indices()`` +* optimize ``util.xor_indices()`` +* move development files from ``examples/`` to ``devel/`` + + +**3.5.2** (2025-07-21): + +* optimize ``util.random_p()`` by also using bitwise AND in final step +* fix DeprecationWarning regarding ``u`` type code +* add `verification tests <../devel/test_random.py>`__ for internals + of ``util.random_p()`` + + +**3.5.1** (2025-07-14): + +* optimize ``util.random_p()`` for ``n < 100`` +* add `Random Bitarrays `__ documentation +* add `statistical tests for random functions <../devel/test_random.py>`__ + + +**3.5.0** (2025-07-06): + +* add ``util.random_p()`` +* improve sparse compression testing + + +**3.4.3** (2025-06-23): + +* minor updates to documentation +* C-level: + - simplify and speedup ``extend_unicode01()`` + - customize ``resize_lite()`` - avoid unused code + - use ``PyTypeObject`` for bitarray type object in ``_util.c`` to + be consistent with ``_bitarray.c`` + - add and improve comments to implementation of sparse bitarray + compression + - simplify ``sc_count()`` + + +**3.4.2** (2025-05-21): + +* extend documentation of + `compression of sparse bitarrays `__ +* ``util.sc_decode()`` and ``util.vl_decode()`` now raise ``StopIteration`` + instead of ``ValueError`` when unexpected end of stream is encountered +* add debug mode tests for ``read_n()``, ``write_n()`` and ``count_from_word()`` + + +**3.4.1** (2025-05-15): + +* add ``pyproject.toml``, see `#233 `__ +* implement ``bits2bytes()`` in C +* optimize ``delslice()`` when ``step`` is larger than about 5 +* consistently name ``*_span()`` and ``*_range()`` in C for + invert, set and count +* organize and add tests (including debug mode tests for ``zlw()``) + + +**3.4.0** (2025-05-06): + +* remove ``.endian()`` method in favor of data descriptor ``.endian`` +* allow bitarray initializer ``bytes`` or ``bytearray`` to set buffer directly +* allow calling ``.extend()`` with ``bytes`` object (although the only + valid bytes are 0x00 and 0x01) +* add ``util.byteswap()`` +* add ``util.correspond_all()`` +* fix ``.reverse()`` for imported buffer +* drop Python 3.5 support +* add tests + + +**3.3.2** (2025-05-02): + +* fix off-by-one-error in check for length of count argument + in ``util.canonical_decode()`` +* simplify ``util.int2ba()`` +* add tests +* add `masked indexing example <../examples/masked.py>`__ +* add `tricks example <../devel/tricks.py>`__ + + +**3.3.1** (2025-04-04): + +* remove ``License`` classifier in favor of a SPDX license expression, `#231 `__ +* reorganize and cleanup many tests + + +**3.3.0** (2025-03-30): + +* add optional ``group`` and ``sep`` arguments' to ``.to01()``, `#230 `__ - + as well as ``util.ba2hex()`` and ``util.ba2base()`` +* ignore whitespace in ``util.base2ba()`` and ``util.hex2ba()`` +* check for embedded nul characters when extending (and initializing) + bitarray from string +* improve testing +* add `double precision floating point number example <../examples/double.py>`__ + + +**3.2.0** (2025-03-19): + +* add ``util.xor_indices()``, `#229 `__ +* add `Hamming code example <../examples/hamming.py>`__ + + +**3.1.1** (2025-03-06): + +* updated ``pythoncapi_compat.h`` for pypy3.11 support, see `#227 `__ +* use ``__builtin_parityll()`` when available in ``util.parity()`` +* add ``parity_64()`` to header +* simplify some tests +* add `LFSR example <../examples/lfsr.py>`__ + + +**3.1.0** (2025-02-19): + +* allow mask assignment to bitarrays, see `#225 `__ +* add missing masked operations to pyi-file +* refactor ``resize()`` and avoid overallocation when downsizing buffer +* update ``build_wheels.yml`` +* fix some typos +* minor simplifications +* rename ``growth/`` example to ``resize/`` and add tests for ``resize()`` +* update gene example +* add comments + + +**3.0.0** (2024-10-15): + +* see `Bitarray 3 transition `__ +* remove Python 2.7 support +* ``.decode()`` now returns iterator (equivalent to past ``.iterdecode()``) +* ``.search()`` now returns iterator (equivalent to past ``.itersearch()``) +* remove ``.iterdecode()`` and ``.itersearch()`` +* remove ``util.rindex()``, use ``.index(..., right=1)`` instead, + deprecated since 2.9 +* remove ``util.make_endian()``, use ``bitarray(..., endian=...)`` instead, + deprecated since 2.9 +* remove hackish support for ``bitarray()`` handling unpickling, + see detailed explaination in `#207 `__ - closes `#206 `__ + + +**2.9.3** (2024-10-10): + +* add official Python 3.13 support +* update cibuildwheel to 2.21.3 +* minor simplifications +* fix some typos + + **2.9.2** (2024-01-01): * optimize initialization from strings by not constantly resizing buffer @@ -48,7 +208,7 @@ Change log **2.8.4** (2023-12-04): * simplify ``copy_n()`` (remove special cases), see `#d2d6fd53 `__ -* add `word shift example C program <../examples/shift_r8.c>`__, +* add `word shift example C program <../devel/shift_r8.c>`__, and simplify ``shift_r8()`` * improve documentation and testing * add `roadmap `__ @@ -168,10 +328,10 @@ Change log **2.6.0** (2022-07-19): -* add data descriptions: ``.nbytes``, ``.padbits``, ``.readonly`` +* add data descriptors: ``.nbytes``, ``.padbits``, ``.readonly`` * allow optional ``endian`` argument to be ``None`` when creating bitarrays * fix type annotation for ``canonical_decode()``, `#178 `__ -* frozenbitarray's padbits are now guaranteed to be zero +* frozenbitarray's pad bits are now guaranteed to be zero * add tests @@ -196,7 +356,7 @@ Change log * optimize ``delslice()`` for cases like ``del a[1:17:2]`` when ``a`` is large * fix ``examples/huffman/compress.py`` to handle files with 0 or 1 characters, see also `#172 `__ -* add ``skipIF`` decorator for skipping tests +* add ``skipIf`` decorator for skipping tests * add tests @@ -284,7 +444,7 @@ Change log a speedup for ``.find()``, ``.index()``, ``.search()`` and ``util.rindex()`` * add optional start and stop arguments to ``.bytereverse()`` * add example to illustrate how - `unaligned copying <../examples/copy_n.py>`__ works internally + `unaligned copying <../devel/copy_n.py>`__ works internally * add documentation * add tests diff -pruN 2.9.2-1/doc/random_p.rst 3.6.1-1/doc/random_p.rst --- 2.9.2-1/doc/random_p.rst 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/doc/random_p.rst 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,119 @@ +Random Bitarrays +================ + +Bitarray 3.5 introduced the utility function ``util.random_p(n, p=0.5)``. +It returns a pseudo-random bitarray (of length ``n``) for which each bit has +probability ``p`` of being one. This is mathematically equivalent to: + +.. code-block:: python + + bitarray(random() < p for _ in range(n)) + +While this expression work well for small ``n``, it is quite slow when ``n`` +is large. In the following we focus on the case of ``n`` being large. + +When ``p`` is small, a fast implementation of ``random_p()`` is to (a) +calculate the population of the bitarray, and then (b) set the required +number of bits, using ``random.randrange()`` for each bit. +Python 3.12 introduced ``random.binomialvariate()`` which is exactly what we +need to determine the bitarray's population. + +When ``p == 0.5``, we use ``random.randbytes()`` to initialize our bitarray +buffer. It should be noted that ``util.urandom()`` uses ``os.urandom()``, +but since ``util.random_p()`` is designed to give reproducible pseudo-random +bitarrays, it uses ``randbytes()``. + +Taking two (independent) such bitarrays and combining them +using the bitwise AND operation, gives us a random bitarray with +probability 1/4. +Likewise, taking two bitwise OR operation gives us probability 3/4. +Without going into too much further detail, it is possible to combine +more than two "randbytes" bitarray to get probabilities ``i / 2**M``, +where ``M`` is the maximal number of "randbytes" bitarrays we combine, +and ``i`` is an integer. +The required sequence of AND and OR operations is calculated from +the desired probability ``p`` and ``M``. + +Once we have calculated our sequence, and obtained a bitarray with +probability ``q = i / 2**M``, we perform a final OR or AND operation with +a random bitarray of probability ``x``. +In order to arrive at exactly the requested probability ``p``, it can +be verified that: + +.. code-block:: python + + x = (p - q) / (1.0 - q) # OR + x = p / q # AND + +It should be noted that ``x`` is always small (once symmetry is applied in +case of AND) such that it always uses the "small p" case. +Unlike the combinations, this gives us a bitarray +with exact probability ``x``. Therefore, the requested probability ``p`` +is exactly obtained. +For more details, see ``VerificationTests`` in the +additional `random tests <../devel/test_random.py>`__. + + +Speedup +------- + +The speedup is largest, when the number of number of random numbers our +algorithm uses is smallest. +In the following, let ``k`` be the number of calls to ``randbytes()``. +For example, when ``p=0.5`` we have ``k=1``. +When ``p`` is below our limit for using the procedure of setting individual +bits, we call this limit ``small_p``, we have ``k=0``. + +In our implementation, we are using ``M=8`` and ``small_p=0.01``. +These parameters have carefully been selected to optimize the average (with +respect to ``p``) execution time. +The following table shows execution times (in milliseconds) of ``random_p()`` +for different values of ``p`` for ``n=100_000_000``: + +.. code-block:: + + p t/ms k notes + ----------------------------------------------------------------------- + edge cases: + 0.0 0.4 0 + 0.5 21.7 1 + 1.0 0.4 0 + + pure combinations: + 1/4 44.6 2 + 1/8 65.2 3 + 1/16 88.7 4 + 1/32 108.6 5 + 1/64 132.4 6 + 3/128 151.9 7 p = 1/128 < small_p, so we take different p + 127/256 174.9 8 priciest pure combinations case(s) + + small p: + 0.0001 2.2 0 + 0.001 18.7 0 + 0.003891051 72.9 0 p = 1/257 - largest x in mixed case + 0.009999999 192.3 0 priciest small p case + + mixed: x (final operation) + 0.01 194.3 7 0.002204724 OR smallest p for mixed case + 0.1 223.4 8 0.002597403 OR + 0.2 194.7 8 0.000975610 OR + 0.3 213.7 8 0.997402597 AND + 0.4 203.3 7 0.002597403 OR + 0.252918288 118.7 2 0.003891051 OR p=65/257 + 0.494163425 249.5 8 0.996108951 AND priciest mixed case(s) + 0.499999999 22.4 1 0.999999998 AND cheapest mixed case + + literal: + any 3740.2 - bitarray(random() < p for _ in range(n)) + + +Using the literal definition one always uses ``n`` calls to ``random()``, +regardless of ``p``. +For 1000 random values of ``p`` (between 0 and 1), we get an average speedup +of about 19. + +In summary: Even in the worst cases ``random_p()`` performs about 15 times +better than the literal definition for large ``n``, while on average we get +a speedup of almost 20. For very small ``p``, and for special values of ``p`` +the speedup is significantly higher. diff -pruN 2.9.2-1/doc/reference.rst 3.6.1-1/doc/reference.rst --- 2.9.2-1/doc/reference.rst 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/doc/reference.rst 2025-08-12 08:35:42.000000000 +0000 @@ -1,7 +1,7 @@ Reference ========= -bitarray version: 2.9.2 -- `change log `__ +bitarray version: 3.6.1 -- `change log `__ In the following, ``item`` and ``value`` are usually a single bit - an integer 0 or 1. @@ -14,15 +14,12 @@ The bitarray object: ``bitarray(initializer=0, /, endian='big', buffer=None)`` -> bitarray Return a new bitarray object whose items are bits initialized from - the optional initial object, and endianness. - The initializer may be of the following types: - - ``int``: Create a bitarray of given integer length. The initial values are - all ``0``. - - ``str``: Create bitarray from a string of ``0`` and ``1``. - - ``iterable``: Create bitarray from iterable or sequence of integers 0 or 1. + the optional initializer, and bit-endianness. + The initializer may be one of the following types: + a.) ``int`` bitarray, initialized to zeros, of given length + b.) ``bytes`` or ``bytearray`` to initialize buffer directly + c.) ``str`` of 0s and 1s, ignoring whitespace and "_" + d.) iterable of integers 0 or 1. Optional keyword arguments: @@ -34,7 +31,9 @@ The bitarray object: cannot be present (or has to be ``None``). The imported buffer may be read-only or writable, depending on the object type. - New in version 2.3: optional ``buffer`` argument. + New in version 2.3: optional ``buffer`` argument + + New in version 3.4: allow initializer ``bytes`` or ``bytearray`` to set buffer directly bitarray methods: @@ -59,7 +58,7 @@ bitarray methods: 0. memory address of buffer 1. buffer size (in bytes) - 2. bit-endianness as a string + 2. bit-endianness as a Unicode string 3. number of pad bits 4. allocated memory for the buffer (in bytes) 5. memory is read-only @@ -71,16 +70,16 @@ bitarray methods: For each byte in byte-range(start, stop) reverse bits in-place. The start and stop indices are given in terms of bytes (not bits). Also note that this method only changes the buffer; it does not change the - endianness of the bitarray object. Padbits are left unchanged such that - two consecutive calls will always leave the bitarray unchanged. + bit-endianness of the bitarray object. Pad bits are left unchanged such + that two consecutive calls will always leave the bitarray unchanged. - New in version 2.2.5: optional start and stop arguments. + New in version 2.2.5: optional start and stop arguments ``clear()`` Remove all items from the bitarray. - New in version 1.4. + New in version 1.4 ``copy()`` -> bitarray @@ -95,16 +94,21 @@ bitarray methods: The ``value`` may also be a sub-bitarray. In this case non-overlapping occurrences are counted within ``[start:stop]`` (``step`` must be 1). - New in version 1.1.0: optional start and stop arguments. + New in version 1.1.0: optional start and stop arguments - New in version 2.3.7: optional step argument. + New in version 2.3.7: optional step argument - New in version 2.9: add non-overlapping sub-bitarray count. + New in version 2.9: add non-overlapping sub-bitarray count -``decode(code, /)`` -> list +``decode(code, /)`` -> iterator Given a prefix code (a dict mapping symbols to bitarrays, or ``decodetree`` - object), decode content of bitarray and return it as a list of symbols. + object), decode content of bitarray and return an iterator over + corresponding symbols. + + See also: `Bitarray 3 transition `__ + + New in version 3.0: returns iterator (equivalent to past ``.iterdecode()``) ``encode(code, iterable, /)`` @@ -113,15 +117,13 @@ bitarray methods: with corresponding bitarray for each symbol. -``endian()`` -> str - Return the bit-endianness of the bitarray as a string (``little`` or ``big``). - - ``extend(iterable, /)`` - Append all items from ``iterable`` to the end of the bitarray. - If the iterable is a string, each ``0`` and ``1`` are appended as + Append items from to the end of the bitarray. + If ``iterable`` is a Unicode string, each ``0`` and ``1`` are appended as bits (ignoring whitespace and underscore). + New in version 3.4: allow ``bytes`` object + ``fill()`` -> int Add zeros to the end of the bitarray, such that the length will be @@ -133,25 +135,25 @@ bitarray methods: is found, such that sub_bitarray is contained within ``[start:stop]``. Return -1 when sub_bitarray is not found. - New in version 2.1. + New in version 2.1 - New in version 2.9: add optional keyword argument ``right``. + New in version 2.9: add optional keyword argument ``right`` ``frombytes(bytes, /)`` Extend bitarray with raw bytes from a bytes-like object. Each added byte will add eight bits to the bitarray. - New in version 2.5.0: allow bytes-like argument. + New in version 2.5.0: allow bytes-like argument ``fromfile(f, n=-1, /)`` Extend bitarray with up to ``n`` bytes read from file object ``f`` (or any other binary stream what supports a ``.read()`` method, e.g. ``io.BytesIO``). - Each read byte will add eight bits to the bitarray. When ``n`` is omitted or - negative, all bytes until EOF are read. When ``n`` is non-negative but - exceeds the data available, ``EOFError`` is raised (but the available data - is still read and appended). + Each read byte will add eight bits to the bitarray. When ``n`` is omitted + or negative, reads and extends all data until EOF. + When ``n`` is non-negative but exceeds the available data, ``EOFError`` is + raised. However, the available data is still read and extended. ``index(sub_bitarray, start=0, stop=, /, right=False)`` -> int @@ -159,7 +161,7 @@ bitarray methods: is found, such that sub_bitarray is contained within ``[start:stop]``. Raises ``ValueError`` when the sub_bitarray is not present. - New in version 2.9: add optional keyword argument ``right``. + New in version 2.9: add optional keyword argument ``right`` ``insert(index, value, /)`` @@ -170,23 +172,7 @@ bitarray methods: Invert all bits in bitarray (in-place). When the optional ``index`` is given, only invert the single bit at index. - New in version 1.5.3: optional index argument. - - -``iterdecode(code, /)`` -> iterator - Given a prefix code (a dict mapping symbols to bitarrays, or ``decodetree`` - object), decode content of bitarray and return an iterator over - the symbols. - - -``itersearch(sub_bitarray, start=0, stop=, /, right=False)`` -> iterator - Return iterator over indices where sub_bitarray is found, such that - sub_bitarray is contained within ``[start:stop]``. - The indices are iterated in ascending order (from lowest to highest), - unless ``right=True``, which will iterate in descending oder (starting with - rightmost match). - - New in version 2.9: optional start and stop arguments - add optional keyword argument ``right``. + New in version 1.5.3: optional index argument ``pack(bytes, /)`` @@ -198,7 +184,7 @@ bitarray methods: transfer of data between bitarray objects to other Python objects (for example NumPy's ndarray object) which have a different memory view. - New in version 2.5.0: allow bytes-like argument. + New in version 2.5.0: allow bytes-like argument ``pop(index=-1, /)`` -> item @@ -215,11 +201,18 @@ bitarray methods: Reverse all bits in bitarray (in-place). -``search(sub_bitarray, limit=, /)`` -> list - Searches for given sub_bitarray in self, and return list of start - positions. - The optional argument limits the number of search results to the integer - specified. By default, all search results are returned. +``search(sub_bitarray, start=0, stop=, /, right=False)`` -> iterator + Return iterator over indices where sub_bitarray is found, such that + sub_bitarray is contained within ``[start:stop]``. + The indices are iterated in ascending order (from lowest to highest), + unless ``right=True``, which will iterate in descending order (starting with + rightmost match). + + See also: `Bitarray 3 transition `__ + + New in version 2.9: optional start and stop arguments - add optional keyword argument ``right`` + + New in version 3.0: returns iterator (equivalent to past ``.itersearch()``) ``setall(value, /)`` @@ -231,9 +224,13 @@ bitarray methods: Sort all bits in bitarray (in-place). -``to01()`` -> str - Return a string containing '0's and '1's, representing the bits in the - bitarray. +``to01(group=0, sep=' ')`` -> str + Return bitarray as Unicode string of '0's and '1's. + The bits are grouped into ``group`` bits (default is no grouping). + When grouped, the string ``sep`` is inserted between groups + of ``group`` characters, default is a space. + + New in version 3.3: optional ``group`` and ``sep`` arguments ``tobytes()`` -> bytes @@ -245,8 +242,8 @@ bitarray methods: ``tolist()`` -> list - Return bitarray as list of integer items. - ``a.tolist()`` is equal to ``list(a)``. + Return bitarray as list of integers. + ``a.tolist()`` equals ``list(a)``. Note that the list object being created will require 32 or 64 times more memory (depending on the machine architecture) than the bitarray object, @@ -254,7 +251,7 @@ bitarray methods: ``unpack(zero=b'\x00', one=b'\x01')`` -> bytes - Return bytes containing one character for each bit in the bitarray, + Return bytes that contain one byte for each bit in the bitarray, using specified mapping. @@ -263,6 +260,12 @@ bitarray data descriptors: Data descriptors were added in version 2.6. +``endian`` -> str + bit-endianness as Unicode string + + New in version 3.4: replaces former ``.endian()`` method + + ``nbytes`` -> int buffer size in bytes @@ -283,14 +286,14 @@ Other objects: object is initialized. A ``frozenbitarray`` is immutable and hashable, and may therefore be used as a dictionary key. - New in version 1.1. + New in version 1.1 ``decodetree(code, /)`` -> decodetree Given a prefix code (a dict mapping symbols to bitarrays), - create a binary tree object to be passed to ``.decode()`` or ``.iterdecode()``. + create a binary tree object to be passed to ``.decode()``. - New in version 1.6. + New in version 1.6 Functions defined in the `bitarray` module: @@ -301,11 +304,11 @@ Functions defined in the `bitarray` modu ``get_default_endian()`` -> str - Return the default endianness for new bitarray objects being created. - Unless ``_set_default_endian('little')`` was called, the default endianness - is ``big``. + Return the default bit-endianness for new bitarray objects being created. + Unless ``_set_default_endian('little')`` was called, the default + bit-endianness is ``big``. - New in version 1.3. + New in version 1.3 ``test(verbosity=1)`` -> TextTestResult @@ -317,74 +320,95 @@ Functions defined in `bitarray.util` mod This sub-module was added in version 1.2. -``zeros(length, /, endian=None)`` -> bitarray - Create a bitarray of length, with all values 0, and optional - endianness, which may be 'big', 'little'. +``any_and(a, b, /)`` -> bool + Efficient implementation of ``any(a & b)``. + New in version 2.7 -``ones(length, /, endian=None)`` -> bitarray - Create a bitarray of length, with all values 1, and optional - endianness, which may be 'big', 'little'. - New in version 2.9. +``ba2base(n, bitarray, /, group=0, sep=' ')`` -> str + Return a string containing the base ``n`` ASCII representation of + the bitarray. Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. + The bitarray has to be multiple of length 1, 2, 3, 4, 5 or 6 respectively. + For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the + standard base 64 alphabet is used. + When grouped, the string ``sep`` is inserted between groups + of ``group`` characters, default is a space. + See also: `Bitarray representations `__ -``urandom(length, /, endian=None)`` -> bitarray - Return a bitarray of ``length`` random bits (uses ``os.urandom``). + New in version 1.9 - New in version 1.7. + New in version 3.3: optional ``group`` and ``sep`` arguments -``pprint(bitarray, /, stream=None, group=8, indent=4, width=80)`` - Prints the formatted representation of object on ``stream`` (which defaults - to ``sys.stdout``). By default, elements are grouped in bytes (8 elements), - and 8 bytes (64 elements) per line. - Non-bitarray objects are printed by the standard library - function ``pprint.pprint()``. +``ba2hex(bitarray, /, group=0, sep=' ')`` -> hexstr + Return a string containing the hexadecimal representation of + the bitarray (which has to be multiple of 4 in length). + When grouped, the string ``sep`` is inserted between groups + of ``group`` characters, default is a space. - New in version 1.8. + New in version 3.3: optional ``group`` and ``sep`` arguments -``make_endian(bitarray, /, endian)`` -> bitarray - When the endianness of the given bitarray is different from ``endian``, - return a new bitarray, with endianness ``endian`` and the same elements - as the original bitarray. - Otherwise (endianness is already ``endian``) the original bitarray is returned - unchanged. +``ba2int(bitarray, /, signed=False)`` -> int + Convert the given bitarray to an integer. + The bit-endianness of the bitarray is respected. + ``signed`` indicates whether two's complement is used to represent the integer. - New in version 1.3. - New in version 2.9: deprecated - use ``bitarray()``. +``base2ba(n, asciistr, /, endian=None)`` -> bitarray + Bitarray of base ``n`` ASCII representation. + Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. + For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the + standard base 64 alphabet is used. Whitespace is ignored. + See also: `Bitarray representations `__ -``rindex(bitarray, sub_bitarray=1, start=0, stop=, /)`` -> int - Return rightmost (highest) index where sub_bitarray (or item - defaults - to 1) is found in bitarray (``a``), such that sub_bitarray is contained - within ``a[start:stop]``. - Raises ``ValueError`` when the sub_bitarray is not present. + New in version 1.9 - New in version 2.3.0: optional start and stop arguments. + New in version 3.3: ignore whitespace - New in version 2.9: deprecated - use ``.index(..., right=1)``. +``byteswap(a, /, n=)`` + Reverse every ``n`` consecutive bytes of ``a`` in-place. + By default, all bytes are reversed. Note that ``n`` is not limited to 2, 4 + or 8, but can be any positive integer. + Also, ``a`` may be any object that exposes a writable buffer. + Nothing about this function is specific to bitarray objects. -``strip(bitarray, /, mode='right')`` -> bitarray - Return a new bitarray with zeros stripped from left, right or both ends. - Allowed values for mode are the strings: ``left``, ``right``, ``both`` + New in version 3.4 -``count_n(a, n, value=1, /)`` -> int - Return lowest index ``i`` for which ``a[:i].count(value) == n``. - Raises ``ValueError`` when ``n`` exceeds total count (``a.count(value)``). +``canonical_decode(bitarray, count, symbol, /)`` -> iterator + Decode bitarray using canonical Huffman decoding tables + where ``count`` is a sequence containing the number of symbols of each length + and ``symbol`` is a sequence of symbols in canonical order. - New in version 2.3.6: optional value argument. + See also: `Canonical Huffman Coding `__ + New in version 2.5 -``parity(a, /)`` -> int - Return parity of bitarray ``a``. - ``parity(a)`` is equivalent to ``a.count() % 2`` but more efficient. - New in version 1.9. +``canonical_huffman(dict, /)`` -> tuple + Given a frequency map, a dictionary mapping symbols to their frequency, + calculate the canonical Huffman code. Returns a tuple containing: + + 0. the canonical Huffman code as a dict mapping symbols to bitarrays + 1. a list containing the number of symbols of each code length + 2. a list of symbols in canonical order + + Note: the two lists may be used as input for ``canonical_decode()``. + + See also: `Canonical Huffman Coding `__ + + New in version 2.5 + + +``correspond_all(a, b, /)`` -> tuple + Return tuple with counts of: ~a & ~b, ~a & b, a & ~b, a & b + + New in version 3.4 ``count_and(a, b, /)`` -> int @@ -392,6 +416,13 @@ This sub-module was added in version 1.2 as no intermediate bitarray object gets created. +``count_n(a, n, value=1, /)`` -> int + Return lowest index ``i`` for which ``a[:i].count(value) == n``. + Raises ``ValueError`` when ``n`` exceeds total count (``a.count(value)``). + + New in version 2.3.6: optional value argument + + ``count_or(a, b, /)`` -> int Return ``(a | b).count()`` in a memory efficient manner, as no intermediate bitarray object gets created. @@ -404,17 +435,39 @@ This sub-module was added in version 1.2 This is also known as the Hamming distance. -``any_and(a, b, /)`` -> bool - Efficient implementation of ``any(a & b)``. +``deserialize(bytes, /)`` -> bitarray + Return a bitarray given a bytes-like representation such as returned + by ``serialize()``. - New in version 2.7. + See also: `Bitarray representations `__ + New in version 1.8 -``subset(a, b, /)`` -> bool - Return ``True`` if bitarray ``a`` is a subset of bitarray ``b``. - ``subset(a, b)`` is equivalent to ``a | b == b`` (and equally ``a & b == a``) but - more efficient as no intermediate bitarray object is created and the buffer - iteration is stopped as soon as one mismatch is found. + New in version 2.5.0: allow bytes-like argument + + +``hex2ba(hexstr, /, endian=None)`` -> bitarray + Bitarray of hexadecimal representation. hexstr may contain any number + (including odd numbers) of hex digits (upper or lower case). + Whitespace is ignored. + + New in version 3.3: ignore whitespace + + +``huffman_code(dict, /, endian=None)`` -> dict + Given a frequency map, a dictionary mapping symbols to their frequency, + calculate the Huffman code, i.e. a dict mapping those symbols to + bitarrays (with given bit-endianness). Note that the symbols are not limited + to being strings. Symbols may be any hashable object. + + +``int2ba(int, /, length=None, endian=None, signed=False)`` -> bitarray + Convert the given integer to a bitarray (with given bit-endianness, + and no leading (big-endian) / trailing (little-endian) zeros), unless + the ``length`` of the bitarray is provided. An ``OverflowError`` is raised + if the integer is not representable with the given number of bits. + ``signed`` determines whether two's complement is used to represent the integer, + and requires ``length`` to be provided. ``intervals(bitarray, /)`` -> iterator @@ -422,76 +475,72 @@ This sub-module was added in version 1.2 iterator over tuples ``(value, start, stop)``. The intervals are guaranteed to be in order, and their size is always non-zero (``stop - start > 0``). - New in version 2.7. + New in version 2.7 -``ba2hex(bitarray, /)`` -> hexstr - Return a string containing the hexadecimal representation of - the bitarray (which has to be multiple of 4 in length). +``ones(n, /, endian=None)`` -> bitarray + Create a bitarray of length ``n``, with all values ``1``, and optional + bit-endianness (``little`` or ``big``). + New in version 2.9 -``hex2ba(hexstr, /, endian=None)`` -> bitarray - Bitarray of hexadecimal representation. hexstr may contain any number - (including odd numbers) of hex digits (upper or lower case). +``parity(a, /)`` -> int + Return parity of bitarray ``a``. + ``parity(a)`` is equivalent to ``a.count() % 2`` but more efficient. -``ba2base(n, bitarray, /)`` -> str - Return a string containing the base ``n`` ASCII representation of - the bitarray. Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. - The bitarray has to be multiple of length 1, 2, 3, 4, 5 or 6 respectively. - For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the - standard base 64 alphabet is used. - - See also: `Bitarray representations `__ - - New in version 1.9. - + New in version 1.9 -``base2ba(n, asciistr, /, endian=None)`` -> bitarray - Bitarray of base ``n`` ASCII representation. - Allowed values for ``n`` are 2, 4, 8, 16, 32 and 64. - For ``n=32`` the RFC 4648 Base32 alphabet is used, and for ``n=64`` the - standard base 64 alphabet is used. - See also: `Bitarray representations `__ +``pprint(bitarray, /, stream=None, group=8, indent=4, width=80)`` + Prints the formatted representation of object on ``stream`` (which defaults + to ``sys.stdout``). By default, elements are grouped in bytes (8 elements), + and 8 bytes (64 elements) per line. + Non-bitarray objects are printed by the standard library + function ``pprint.pprint()``. - New in version 1.9. + New in version 1.8 -``ba2int(bitarray, /, signed=False)`` -> int - Convert the given bitarray to an integer. - The bit-endianness of the bitarray is respected. - ``signed`` indicates whether two's complement is used to represent the integer. +``random_k(n, /, k, endian=None)`` -> bitarray + Return (pseudo-) random bitarray of length ``n`` with ``k`` elements + set to one. Mathematically equivalent to setting (in a bitarray of + length ``n``) all bits at indices ``random.sample(range(n), k)`` to one. + The random bitarrays are reproducible when giving Python's ``random.seed()`` + with a specific seed value. + This function requires Python 3.9 or higher, as it depends on the standard + library function ``random.randbytes()``. Raises ``NotImplementedError`` + when Python version is too low. -``int2ba(int, /, length=None, endian=None, signed=False)`` -> bitarray - Convert the given integer to a bitarray (with given endianness, - and no leading (big-endian) / trailing (little-endian) zeros), unless - the ``length`` of the bitarray is provided. An ``OverflowError`` is raised - if the integer is not representable with the given number of bits. - ``signed`` determines whether two's complement is used to represent the integer, - and requires ``length`` to be provided. + New in version 3.6 -``serialize(bitarray, /)`` -> bytes - Return a serialized representation of the bitarray, which may be passed to - ``deserialize()``. It efficiently represents the bitarray object (including - its bit-endianness) and is guaranteed not to change in future releases. +``random_p(n, /, p=0.5, endian=None)`` -> bitarray + Return (pseudo-) random bitarray of length ``n``, where each bit has + probability ``p`` of being one (independent of any other bits). Mathematically + equivalent to ``bitarray((random() < p for _ in range(n)), endian)``, but much + faster for large ``n``. The random bitarrays are reproducible when giving + Python's ``random.seed()`` with a specific seed value. - See also: `Bitarray representations `__ + This function requires Python 3.12 or higher, as it depends on the standard + library function ``random.binomialvariate()``. Raises ``NotImplementedError`` + when Python version is too low. - New in version 1.8. + See also: `Random Bitarrays `__ + New in version 3.5 -``deserialize(bytes, /)`` -> bitarray - Return a bitarray given a bytes-like representation such as returned - by ``serialize()``. - See also: `Bitarray representations `__ +``sc_decode(stream)`` -> bitarray + Decompress binary stream (an integer iterator, or bytes-like object) of a + sparse compressed (``sc``) bitarray, and return the decoded bitarray. + This function consumes only one bitarray and leaves the remaining stream + untouched. Use ``sc_encode()`` for compressing (encoding). - New in version 1.8. + See also: `Compression of sparse bitarrays `__ - New in version 2.5.0: allow bytes-like argument. + New in version 2.7 ``sc_encode(bitarray, /)`` -> bytes @@ -501,28 +550,42 @@ This sub-module was added in version 1.2 See also: `Compression of sparse bitarrays `__ - New in version 2.7. + New in version 2.7 -``sc_decode(stream)`` -> bitarray - Decompress binary stream (an integer iterator, or bytes-like object) of a - sparse compressed (``sc``) bitarray, and return the decoded bitarray. - This function consumes only one bitarray and leaves the remaining stream - untouched. Use ``sc_encode()`` for compressing (encoding). +``serialize(bitarray, /)`` -> bytes + Return a serialized representation of the bitarray, which may be passed to + ``deserialize()``. It efficiently represents the bitarray object (including + its bit-endianness) and is guaranteed not to change in future releases. - See also: `Compression of sparse bitarrays `__ + See also: `Bitarray representations `__ - New in version 2.7. + New in version 1.8 -``vl_encode(bitarray, /)`` -> bytes - Return variable length binary representation of bitarray. - This representation is useful for efficiently storing small bitarray - in a binary stream. Use ``vl_decode()`` for decoding. +``strip(bitarray, /, mode='right')`` -> bitarray + Return a new bitarray with zeros stripped from left, right or both ends. + Allowed values for mode are the strings: ``left``, ``right``, ``both`` + + +``subset(a, b, /)`` -> bool + Return ``True`` if bitarray ``a`` is a subset of bitarray ``b``. + ``subset(a, b)`` is equivalent to ``a | b == b`` (and equally ``a & b == a``) but + more efficient as no intermediate bitarray object is created and the buffer + iteration is stopped as soon as one mismatch is found. - See also: `Variable length bitarray format `__ - New in version 2.2. +``sum_indices(a, /)`` -> int + Return sum of indices of all active bits in bitarray ``a``. + Equivalent to ``sum(i for i, v in enumerate(a) if v)``. + + New in version 3.6 + + +``urandom(n, /, endian=None)`` -> bitarray + Return random bitarray of length ``n`` (uses ``os.urandom()``). + + New in version 1.7 ``vl_decode(stream, /, endian=None)`` -> bitarray @@ -532,38 +595,29 @@ This sub-module was added in version 1.2 See also: `Variable length bitarray format `__ - New in version 2.2. - - -``huffman_code(dict, /, endian=None)`` -> dict - Given a frequency map, a dictionary mapping symbols to their frequency, - calculate the Huffman code, i.e. a dict mapping those symbols to - bitarrays (with given endianness). Note that the symbols are not limited - to being strings. Symbols may may be any hashable object (such as ``None``). - + New in version 2.2 -``canonical_huffman(dict, /)`` -> tuple - Given a frequency map, a dictionary mapping symbols to their frequency, - calculate the canonical Huffman code. Returns a tuple containing: - 0. the canonical Huffman code as a dict mapping symbols to bitarrays - 1. a list containing the number of symbols of each code length - 2. a list of symbols in canonical order +``vl_encode(bitarray, /)`` -> bytes + Return variable length binary representation of bitarray. + This representation is useful for efficiently storing small bitarray + in a binary stream. Use ``vl_decode()`` for decoding. - Note: the two lists may be used as input for ``canonical_decode()``. + See also: `Variable length bitarray format `__ - See also: `Canonical Huffman Coding `__ + New in version 2.2 - New in version 2.5. +``xor_indices(a, /)`` -> int + Return xor reduced indices of all active bits in bitarray ``a``. + This is essentially equivalent to + ``reduce(operator.xor, (i for i, v in enumerate(a) if v))``. -``canonical_decode(bitarray, count, symbol, /)`` -> iterator - Decode bitarray using canonical Huffman decoding tables - where ``count`` is a sequence containing the number of symbols of each length - and ``symbol`` is a sequence of symbols in canonical order. + New in version 3.2 - See also: `Canonical Huffman Coding `__ - New in version 2.5. +``zeros(n, /, endian=None)`` -> bitarray + Create a bitarray of length ``n``, with all values ``0``, and optional + bit-endianness (``little`` or ``big``). diff -pruN 2.9.2-1/doc/represent.rst 3.6.1-1/doc/represent.rst --- 2.9.2-1/doc/represent.rst 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/doc/represent.rst 2025-08-12 08:35:42.000000000 +0000 @@ -9,7 +9,7 @@ advantages and disadvantages. Binary representation --------------------- -The most common representation of bitarrays is it's native binary string +The most common representation of bitarrays is its native binary string representation, which is great for interactively analyzing bitarray objects: .. code-block:: python @@ -50,7 +50,7 @@ adds this information to a header byte: >>> x b'\x13\xce\r\x1cxJ\xf1\xe0' >>> b = deserialize(x) - >>> assert a == b and a.endian() == b.endian() + >>> assert a == b and a.endian == b.endian The header byte is structured the following way: @@ -95,11 +95,11 @@ endianness changes: .. code-block:: python - >>> a.endian() + >>> a.endian 'big' >>> b = bitarray(a, 'little') >>> assert a == b - >>> b.endian() + >>> b.endian 'little' >>> ba2hex(b) '3785c1f' @@ -147,6 +147,20 @@ Variable length representation In some cases, it is useful to represent bitarrays in a binary format that is "self terminating" (in the same way that C strings are NUL terminated). -That is, when a bitarray of unknown length is encountered in a stream of -binary data, the format lets you know when the end of a bitarray is reached. +That is, when an encoded bitarray of unknown length is encountered in a +stream of binary data, the format lets us know when the end of the encoded +bitarray is reached. See `variable length format <./variable_length.rst>`__ for this representation. + + +Compressed sparse bitarrays +--------------------------- + +Another representation +is `compressed sparse bitarrays <./sparse_compression.rst>`__, +whose format is also "self terminating". This, format actually uses different +representations dependent on how sparsely the population of the bitarray (even +sections of the bitarray) is. +For large sparse bitarrays, the format reduces (compresses) the amount of data +very efficiently, while only requiring a very tiny overhead for non-sparsely +populated bitarrays. diff -pruN 2.9.2-1/doc/sparse_compression.rst 3.6.1-1/doc/sparse_compression.rst --- 2.9.2-1/doc/sparse_compression.rst 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/doc/sparse_compression.rst 2025-08-12 08:35:42.000000000 +0000 @@ -1,8 +1,18 @@ Compression of sparse bitarrays =============================== -The two utility functions ``sc_encode()`` and ``sc_decode()`` provide -functionality to efficiently compress and decompress sparse bitarrays. +In a ``bitarray`` object each byte in memory represents eight bits. +While this representation is very compact and efficient when dealing with +most data, there are situations when this representation is inefficient. +One such situation are sparsely populated bitarray. +That is, bitarray in which only a few bits are 1, but most bits are 0. +In this situation, one might consider using a data structure which stores +the indices of the 1 bits and not use the ``bitarray`` object at all. +However, having all of bitarray's functionality is very convenient. +It may be desired to convert ``bitarray`` objects into a more compact (index +based) format when storing objects on disk or sending them over the network. +This is the use case of the utility functions ``sc_encode()`` +and ``sc_decode()``. The lower the population count, the more efficient the compression will be: .. code-block:: python @@ -17,6 +27,33 @@ The lower the population count, the more >>> assert sc_decode(blob) == a +How it works +------------ + +Consider a ``bitarray`` of length 256, that is 32 bytes of memory. +If we represent this object by the indices of 1 bits as one byte each, +the object will be represent more efficiently when the population (number +of 1 bits) is less than 32. Based on the population, the +function ``sc_encode()`` chooses to represent the object as either raw bytes +or as bytes of indices of 1 bits. These are the block types 0 and 1. + +Next, we consider a ``bitarray`` of length 65536. When each section of 256 +bits has a population below 32, it would be stored as 256 blocks of type 1. +That is, we need 256 block headers and one (index) byte for each 1 bit. +However, when the total population is below 256, we could also introduce +a new block type 2 in which each index is represented by two bytes and +represent the entire bitarray as a single block (of type 2). +This saves us the 256 block headers (of type 1). +Similarly, with even less populated bitarrays, it will become more efficient +to move to blocks representing each index using 3 or more bytes. + +The encoding algorithm starts at the front of the ``bitarray``, inspects +the population and decides which block type to use to encode the following +bits. Once the first block is written, the algorithm moves on to inspecting +the remaining population, and so on. +This way, a large bitarray with densely and sparsely populated areas will +be compressed efficiently using different block types. + The binary blob consists of a header which encodes the bit-endianness and the total length of the bitarray, i.e. the number of bits. The header is followed by an arbitrary number of blocks. There are 5 block types. Each block starts @@ -67,35 +104,30 @@ ratio, and the number of blocks of each p ratio raw type 1 type 2 type 3 type 4 ------------------------------------------------------------------------- - 0.00000001 0.00000048 0 0 0 0 1 - 0.00000002 0.00000072 0 0 0 0 1 - 0.00000003 0.00000119 0 0 0 0 1 - 0.00000006 0.00000203 0 0 0 0 1 - 0.00000010 0.00000358 0 0 0 0 1 - 0.00000019 0.00000560 0 0 0 16 0 - 0.00000034 0.00000927 0 0 0 16 0 - 0.00000061 0.00001580 0 0 0 16 0 - 0.00000110 0.00002751 0 0 0 16 0 - 0.00000198 0.00004870 0 0 0 16 0 - 0.00000357 0.00008678 0 0 0 16 0 - 0.00000643 0.00015536 0 0 0 16 0 - 0.00001157 0.00027874 0 0 0 16 0 - 0.00002082 0.00057423 0 0 3914 1 0 - 0.00003748 0.00084394 0 0 4085 1 0 - 0.00006747 0.00132376 0 0 4096 0 0 - 0.00012144 0.00218719 0 0 4096 0 0 - 0.00021859 0.00374150 0 0 4096 0 0 - 0.00039346 0.00653845 0 0 4096 0 0 - 0.00070824 0.01157218 0 0 4096 0 0 - 0.00127482 0.02062941 0 0 4096 0 0 - 0.00229468 0.03691334 0 0 4096 0 0 - 0.00413043 0.06410542 0 634074 1620 0 0 - 0.00743477 0.09051070 0 1048576 0 0 0 - 0.01338259 0.13759786 0 1048576 0 0 0 - 0.02408866 0.22166717 0 1048576 0 0 0 - 0.04335959 0.37071779 1 1048575 0 0 0 - 0.07804726 0.65549386 6157 1042384 0 0 0 - 0.14048506 0.98861155 176021 223348 0 0 0 - 0.25287311 1.00024432 8192 0 0 0 0 - 0.45517160 1.00024432 8192 0 0 0 0 - 0.81930887 1.00024432 8192 0 0 0 0 + 1.00000000 1.00024432 8192 0 0 0 0 + 0.50000000 1.00024432 8192 0 0 0 0 + 0.25000000 1.00024432 8192 0 0 0 0 + 0.12500000 0.95681512 261581 494833 0 0 0 + 0.06250000 0.53125548 176 1048400 0 0 0 + 0.03125000 0.28118369 0 1048576 0 0 0 + 0.01562500 0.15618166 0 1048576 0 0 0 + 0.00781250 0.09370023 0 1048576 0 0 0 + 0.00390625 0.06188744 0 281719 2996 0 0 + 0.00195312 0.03140587 0 0 4096 0 0 + 0.00097656 0.01582754 0 0 4096 0 0 + 0.00048828 0.00804502 0 0 4096 0 0 + 0.00024414 0.00415075 0 0 4096 0 0 + 0.00012207 0.00220025 0 0 4096 0 0 + 0.00006104 0.00122154 0 0 4096 0 0 + 0.00003052 0.00072917 0 0 3958 1 0 + 0.00001526 0.00038746 0 0 817 13 0 + 0.00000763 0.00018182 0 0 0 16 0 + 0.00000381 0.00009358 0 0 0 16 0 + 0.00000191 0.00004664 0 0 0 16 0 + 0.00000095 0.00002545 0 0 0 16 0 + 0.00000048 0.00001267 0 0 0 16 0 + 0.00000024 0.00000739 0 0 0 16 0 + 0.00000012 0.00000426 0 0 0 16 0 + 0.00000006 0.00000226 0 0 0 0 1 + 0.00000003 0.00000107 0 0 0 0 1 + 0.00000001 0.00000060 0 0 0 0 1 diff -pruN 2.9.2-1/doc/variable_length.rst 3.6.1-1/doc/variable_length.rst --- 2.9.2-1/doc/variable_length.rst 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/doc/variable_length.rst 2025-08-12 08:35:42.000000000 +0000 @@ -3,8 +3,9 @@ Variable length bitarray format In some cases, it is useful to represent bitarrays in a binary format that is "self terminating" (in the same way that C strings are NUL terminated). -That is, when a bitarray of unknown length is encountered in a stream of -binary data, the format lets you know when the end of a bitarray is reached. +That is, when an encoded bitarray of unknown length is encountered in a +stream of binary data, the format lets us know when the end of the encoded +bitarray is reached. Such a "variable length format" (most memory efficient for small bitarrays) is implemented in ``vl_encode()`` and ``vl_decode()``: diff -pruN 2.9.2-1/examples/README 3.6.1-1/examples/README --- 2.9.2-1/examples/README 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/README 2025-08-12 08:35:42.000000000 +0000 @@ -1,12 +1,7 @@ All files under this 'examples/' directory are unsupported. -While bitarray itself still supports Python 2.7 and Python 3.5+, the examples -usually require Python 3. - - -base-n.py - Simple illustration of ASCII encoding bitarrays to bases - of 2, 4, 8, 16, 32 and 64. +While bitarray itself supports Python 3.6+, some examples may require +a later Python versoin. bloom.py @@ -14,17 +9,16 @@ bloom.py http://en.wikipedia.org/wiki/Bloom_filter -copy_n.py - Illustrate how copy_n() in _bitarray.c works. This is essentially - a Python implementation of copy_n() with output of the different stages - of the bitarray we copy into. - - distance.py Implementation of distance functions and comparison to the corresponding functions in the scipy.spatial.distance module. +double.py + Functionality to analyze double precision floating point numbers + (IEEE 754 binary64). + + extend_json.py Demonstrates how to construct a json encoder and decoder (using the 'json' standard library) which can handle extended Python data structures @@ -36,9 +30,9 @@ gene.py represented by bitarrays. -growth/ - Things to study the bitarray growth pattern, including tests for the - current implementation. +hamming.py + Implementation of Hamming codes for error correction with send + and receive functionality. huffman/ @@ -51,8 +45,8 @@ lexico.py lexicographical permutations of bitarrays. -mapped-file.py - Demonstrates how to memory map a file into a bitarray. +lfsr.py + Linear Feedback Shift Register mandel.py @@ -63,6 +57,14 @@ mandel.py Requires: numba +masked.py + Illustrate masked indexing, i.e. using bitarrays to select indices + + +mmapped-file.py + Demonstrates how to memory map a file into a bitarray. + + ndarray.py Demonstrates how to efficiently convert boolean data from a bitarray to a numpy.ndarray of dtype bool. @@ -80,10 +82,6 @@ puff/ https://github.com/madler/zlib/blob/master/contrib/puff/puff.c -shift_r8.c - C program is to illustrate and document shift_r8() - - sieve.py Demonstrates the "Sieve of Eratosthenes" algorithm for finding all prime numbers up to a specified integer. diff -pruN 2.9.2-1/examples/base-n.py 3.6.1-1/examples/base-n.py --- 2.9.2-1/examples/base-n.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/base-n.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -from __future__ import print_function -from bitarray.util import urandom, pprint, ba2base, base2ba - - -a = urandom(60) - -for m in range(1, 7): - n = 1 << m - print("----- length: %d ----- base: %d ----- " % (m, n)) - pprint(a, group=m) - rep = ba2base(n, a) - print("representation:", rep) - print() - assert base2ba(n, rep) == a diff -pruN 2.9.2-1/examples/bloom.py 3.6.1-1/examples/bloom.py --- 2.9.2-1/examples/bloom.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/bloom.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,13 +1,13 @@ import hashlib from math import ceil, exp, log, log2 -from bitarray.util import zeros +from bitarray import bitarray class BloomFilter(object): """ Implementation of a Bloom filter. An instance is initialized by - it's capacity `n` and error rate `p`. The capacity tells how many + its capacity `n` and error rate `p`. The capacity tells how many elements can be stored while maintaining no more than `p` false positives. """ @@ -18,7 +18,7 @@ class BloomFilter(object): self.k = ceil(-log2(p)) # size of array self.m = ceil(-n * log2(p) / log(2)) - self.array = zeros(self.m) + self.array = bitarray(self.m) def calculate_p(self): """ @@ -38,6 +38,12 @@ class BloomFilter(object): return 0.0 return -self.m / self.k * log(1 - x / self.m) + def population(self): + """ + Return the current relative array population as a float. + """ + return self.array.count() / self.m + def add(self, key): self.array[list(self._hashes(key))] = 1 @@ -52,7 +58,7 @@ class BloomFilter(object): n = 1 << self.m.bit_length() h = hashlib.new('sha1') h.update(str(key).encode()) - x = int.from_bytes(h.digest(), 'little') + x = 0 i = 0 while i < self.k: if x < n: @@ -74,17 +80,17 @@ def test_bloom(n, p): for i in range(n): b.add(i) assert i in b - print("population: %.2f%%" % (100.0 * b.array.count() / b.m)) + print("population: %.2f%%" % (100.0 * b.population())) print("approx_items(): %.2f" % b.approx_items()) print("calculate_p(): %.3f%%" % (100.0 * b.calculate_p())) - N = 100000 + N = 100_000 false_pos = sum(i in b for i in range(n, n + N)) print("experimental : %.3f%%\n" % (100.0 * false_pos / N)) if __name__ == '__main__': - test_bloom(5000, 0.05) - test_bloom(10000, 0.01) - test_bloom(50000, 0.005) - test_bloom(100000, 0.002) + test_bloom( 5_000, 0.05) + test_bloom( 10_000, 0.01) + test_bloom( 50_000, 0.005) + test_bloom(100_000, 0.002) diff -pruN 2.9.2-1/examples/copy_n.py 3.6.1-1/examples/copy_n.py --- 2.9.2-1/examples/copy_n.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/copy_n.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,231 +0,0 @@ -""" -The purpose of this script is to illustrate how copy_n() in _bitarray.c works. -This is essentially a Python implementation of copy_n() with output of the -different stages of the bitarray we copy into. - -Sample output: -a = 21 -b = 6 -n = 31 -p1 = 2 -p2 = 6 -p3 = 0 -sa = 5 -sb = -6 - -> p3 = 1 - -> sb = 2 -other -bitarray('00101110 11111001 01011101 11001011 10110000 01011110 011') -b..b+n ^^ ^^^^^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^^ - ======== ======== ======== ======== - 33 -self -bitarray('01011101 11100101 01110101 01011001 01110100 10001010 01111011') -a..a+n ^^^ ^^^^^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^ - 11111 - 2222 -memmove 4 - ======== ======== ======== ======== -bitarray('01011101 11100101 11111001 01011101 11001011 10110000 01111011') -rshift 7 - >>>>>>>> >>>>>>>> >>>>>>>> >>>>>>>> >>>>>>>> -bitarray('01011101 11100101 00000001 11110010 10111011 10010111 01100000') - = ======== ======== ======== ======== - 11111 - 2222 - 33 -bitarray('01011101 11100101 01110101 11110010 10111011 10010111 01101011') -""" -from io import StringIO - -from bitarray import bitarray, bits2bytes -from bitarray.util import pprint, urandom - - -verbose = False - -def mark_range_n(i, n, c, text=''): - a = bitarray(i * '0' + n * '1') - f = StringIO() - pprint(a, stream=f) - s = f.getvalue() - print("%-10s" % text + ''.join(c if e == '1' else ' ' for e in s[10:])) - - -def mark_range(i, j, c, text=''): - mark_range_n(i, j - i, c, text) - - -def shift_r8(self, a, b, n): - assert 0 <= n < 8 - assert 0 <= a <= self.nbytes - assert 0 <= b <= self.nbytes - if n == 0 or a >= b: - return - self[8 * a : 8 * b] >>= n - -def is_be(self): - return self.endian() == 'big' - -bitmask_table = [ - [0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80], # little endian - [0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01], # big endian -] - -ones_table = [ - [0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f], # little endian - [0x00, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe], # big endian -] - -def copy_n(self, a, other, b, n): - """ - copy n bits from other (starting at b) onto self (starting at a) - """ - p1 = a // 8 # first byte to be copied to - p2 = (a + n - 1) // 8 # last byte to be copied to - p3 = b // 8 # first byte to be memmoved from - sa = a % 8 - sb = -(b % 8) - t3 = 0 - - if verbose: - print('a =', a) - print('b =', b) - print('n =', n) - print('p1 =', p1) - print('p2 =', p2) - print('p3 =', p3) - print('sa =', sa) - print('sb =', sb) - - assert 0 <= n <= min(len(self), len(other)) - assert 0 <= a <= len(self) - n - assert 0 <= b <= len(other) - n - if n == 0 or (self is other and a == b): - return - - if sa + sb < 0: - # In order to keep total right shift (sa + sb) positive, we - # increase the first byte to be copied from (p3) by one byte, - # such that memmove() will move all bytes one extra to the left. - - # As other may be self, we need to store this byte as its memory - # location may be overwritten or changed by memmove or rshift. - t3 = memoryview(other)[p3] - p3 += 1 - sb += 8 - if verbose: - print(' -> p3 =', p3) - print(' -> sb =', sb) - - assert a - sa == 8 * p1 - assert b + sb == 8 * p3 - assert p1 <= p2 - assert 8 * p2 < a + n <= 8 * (p2 + 1) - - if verbose: - print('other') - pprint(other) - mark_range_n(b, n, '^', 'b..b+n') - if n > sb: - mark_range_n(8 * p3, 8 * bits2bytes(n - sb), '=') - mark_range_n(b, sb, '3') - print('self') - pprint(self) - mark_range_n(a, n, '^', 'a..a+n') - if n > sb: - mark_range(8 * p1, a, '1') - mark_range(a + n, 8 * p2 + 8, '2') - - if n > sb: - m = bits2bytes(n - sb) # number of bytes memmoved - table = ones_table[is_be(self)] - m1 = table[sa] - m2 = table[(a + n) % 8] - t1 = memoryview(self)[p1] - t2 = memoryview(self)[p2] - - assert p1 + m in [p2, p2 + 1] - assert p1 + m <= self.nbytes and p3 + m <= other.nbytes - - # aligned copy -- copy first sb bits (if any) later - memoryview(self)[p1:p1 + m] = memoryview(other)[p3:p3 + m] - if self.endian() != other.endian(): - self.bytereverse(p1, p1 + m) - - if verbose: - print('memmove', m) - mark_range_n(8 * p1, 8 * m, '=') - pprint(self) - print('rshift', sa + sb) - mark_range(8 * p1, 8 * (p2 + 1), '>') - - shift_r8(self, p1, p2 + 1, sa + sb) # right shift - if verbose: - pprint(self) - mark_range(8 * p1 + sa + sb, 8 * (p2 + 1), '=') - - if m1: # restore bits at p1 - if verbose: - mark_range(8 * p1, a, '1') - memoryview(self)[p1] = (memoryview(self)[p1] & ~m1) | (t1 & m1) - - if m2: # restore bits at p2 - if verbose: - mark_range(a + n, 8 * p2 + 8, '2') - memoryview(self)[p2] = (memoryview(self)[p2] & m2) | (t2 & ~m2) - - if verbose: - mark_range_n(a, sb, '3') - for i in range(min(sb, n)): # copy first sb bits - self[i + a] = bool(t3 & bitmask_table[is_be(other)][(i + b) % 8]) - - if verbose: - pprint(self) - - -def test_copy_n(): - from random import getrandbits, randrange, randint - - def random_endian(): - return ['little', 'big'][getrandbits(1)] - - max_size = 56 - - for _ in range(10000): - N = randrange(max_size) - M = randrange(max_size) - n = randint(0, min(N, M)) - a = randint(0, N - n) - b = randint(0, M - n) - x = urandom(N, random_endian()) - y = urandom(M, random_endian()) - z = x.copy() - copy_n(x, a, y, b, n) - z[a:a + n] = y[b:b + n] - assert x == z - - for _ in range(10000): - N = randrange(max_size) - n = randint(0, N) - a = randint(0, N - n) - b = randint(0, N - n) - x = urandom(N, random_endian()) - z = x.copy() - copy_n(x, a, x, b, n) - z[a:a + n] = z[b:b + n] - assert x == z - - -if __name__ == '__main__': - test_copy_n() - verbose = True - other = bitarray( - '00101110 11111001 01011101 11001011 10110000 01011110 011') - self = bitarray( - '01011101 11100101 01110101 01011001 01110100 10001010 01111011') - copy_n(self, 21, other, 6, 31) - assert self == bitarray( - '01011101 11100101 01110101 11110010 10111011 10010111 01101011') - #copy_n(self, 2, other, 12, 1) - #copy_n(self, 9, other, 17, 23) diff -pruN 2.9.2-1/examples/distance.py 3.6.1-1/examples/distance.py --- 2.9.2-1/examples/distance.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/distance.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,11 +1,11 @@ """ In this module, we implement distance functions and compare them to the corresponding functions in the scipy.spatial.distance module. -The functions in this module are typically around 10 to 50 times faster. +The functions using bitarray are typically around 50 to 200 times faster. """ from time import perf_counter -from bitarray.util import _correspond_all, count_and, count_xor, urandom +from bitarray.util import correspond_all, count_and, count_xor, urandom import numpy import scipy.spatial.distance as distance # type: ignore @@ -42,7 +42,7 @@ def sokalsneath(u, v): return R / (count_and(u, v) + R) def yule(u, v): - nff, nft, ntf, ntt = _correspond_all(u, v) + nff, nft, ntf, ntt = correspond_all(u, v) half_R = ntf * nft if half_R == 0: return 0.0 diff -pruN 2.9.2-1/examples/double.py 3.6.1-1/examples/double.py --- 2.9.2-1/examples/double.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/examples/double.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,180 @@ +from struct import pack, unpack + +from bitarray import bitarray +from bitarray.util import ba2int, int2ba + + +class Double: + + def __init__(self, x=0.0): + if isinstance(x, (float, int)): + self.from_float(float(x)) + elif isinstance(x, str): + self.from_string(x) + else: + raise TypeError("float or str expected") + + def __float__(self): + a = self.to_bitarray() + return unpack(" %r" % x) + + +# --------------------------------------------------------------------------- + +from math import pi, inf, nan, isnan +from random import getrandbits, randint +import unittest + +from bitarray.util import urandom + + +EXAMPLES = [ + ( 0.0, "0 00000000000 " + 52 * "0"), + ( 1.0, "0 01111111111 " + 52 * "0"), + ( 1.5, "0 01111111111 1" + 51 * "0"), + ( 2.0, "0 10000000000 " + 52 * "0"), + ( 5.0, "0 10000000001 01" + 50 * "0"), + (-5.0, "1 10000000001 01" + 50 * "0"), + # smallest number > 1 + (1.0000000000000002, "0 01111111111 " + 51 * "0" + "1"), + # minimal subnormal double + (4.9406564584124654e-324, "0 00000000000 " + 51 * "0" + "1"), + # maximal subnormal double + (2.2250738585072009e-308, "0 00000000000 " + 52 * "1"), + # minimal normal double + (2.2250738585072014e-308, "0 00000000001 " + 52 * "0"), + # maximal (normal) double + (1.7976931348623157e+308, "0 11111111110 " + 52 * "1"), + ( inf, "0 11111111111 " + 52 * "0"), + (-inf, "1 11111111111 " + 52 * "0"), + ( 1/3, "0 01111111101 " + 26 * "01"), + ( pi, "0 10000000000 " + "1001001000011111101101010100010001000010110100011000"), + # largest number exactly representated as integer + (2 ** 53 - 1, "0 10000110011 " + 52 * "1"), +] + +class DoubleTests(unittest.TestCase): + + def test_zero(self): + d = Double() + self.assertEqual(float(d), 0.0) + self.assertEqual(d.sign, 0) + self.assertEqual(d.exponent, -1023) + self.assertEqual(d.fraction, bitarray(52)) + + def test_examples(self): + for x, s in EXAMPLES: + for d in Double(x), Double(s): + self.assertEqual(float(d), x) + self.assertEqual(str(d), s) + + def test_nan(self): + s = "0 11111111111 1" + 51 * "0" + for x in nan, s: + d = Double(x) + self.assertEqual(str(d), s) + self.assertTrue(isnan(float(d))) + + def test_nan_msg(self): + msg = urandom(52) + d = Double() + d.exponent = 1024 + d.fraction = msg + x = float(d) + self.assertEqual(type(x), float) + self.assertTrue(isnan(x)) + e = Double(x) + self.assertEqual(e.exponent, 1024) + self.assertEqual(e.fraction, msg) + + def test_exponent52(self): + for _ in range(1000): + d = Double() + d.fraction = urandom(52, endian="little") + d.exponent = 52 + d.sign = getrandbits(1) + i = (1 << 52) + ba2int(d.fraction) + if d.sign: + i = -i + self.assertEqual(float(d), i) + + def test_exact_ints(self): + for _ in range(1000): + i = getrandbits(randint(1, 53)) + if i == 0: + continue + + d = Double(i) + self.assertEqual(d.sign, 0) + + a = int2ba(i, endian="little") + a.pop() + n = len(a) + self.assertEqual(d.exponent, n) + + a = bitarray(52 - n, endian="little") + a + self.assertEqual(d.fraction, a) + +if __name__ == '__main__': + import sys + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + d = Double(eval(arg)) + d.info() + else: + unittest.main() diff -pruN 2.9.2-1/examples/gene.py 3.6.1-1/examples/gene.py --- 2.9.2-1/examples/gene.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/gene.py 2025-08-12 08:35:42.000000000 +0000 @@ -14,14 +14,14 @@ trans = { "C": bitarray("11") } -N = 10000 +N = 10_000 seq = [choice("ATGC") for _ in range(N)] arr = bitarray() arr.encode(trans, seq) -assert arr.decode(trans) == seq +assert list(arr.decode(trans)) == seq # decodage -t = timeit(lambda: arr.decode(trans), number=1000) +t = timeit(lambda: list(arr.decode(trans)), number=1000) print(t) diff -pruN 2.9.2-1/examples/growth/.gitignore 3.6.1-1/examples/growth/.gitignore --- 2.9.2-1/examples/growth/.gitignore 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/growth/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -resize -pattern-* diff -pruN 2.9.2-1/examples/growth/Makefile 3.6.1-1/examples/growth/Makefile --- 2.9.2-1/examples/growth/Makefile 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/growth/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -resize: resize.c - gcc -Wall resize.c -o resize - -test: resize - ./resize >pattern-c.txt - python growth.py >pattern-py.txt - diff pattern-c.txt pattern-py.txt - python test.py - -clean: - rm -f resize - rm -f pattern-* diff -pruN 2.9.2-1/examples/growth/README.md 3.6.1-1/examples/growth/README.md --- 2.9.2-1/examples/growth/README.md 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/growth/README.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,10 +0,0 @@ -The bitarray growth pattern -=========================== - -Running `python growth.py` will display the bitarray growth pattern. -This is done by appending one bit to a bitarray in a loop, and displaying -the allocated size of the bitarray object each time it changes. - -The program `resize.c` contains a distilled version of the `resize()` -function which contains the implementation of this growth pattern. -Running this C program gives exactly the same output. diff -pruN 2.9.2-1/examples/growth/growth.py 3.6.1-1/examples/growth/growth.py --- 2.9.2-1/examples/growth/growth.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/growth/growth.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,25 +0,0 @@ -from bitarray import bitarray - - -def show(a): - info = a.buffer_info() - size = info[1] - alloc = info[4] - print('%d %d' % (size, alloc)) - -a = bitarray() -prev = -1 -while len(a) < 2000: - alloc = a.buffer_info()[4] - if prev != alloc: - show(a) - prev = alloc - a.append(1) - -for i in 800_000, 400_000, 399_992, 0, 0, 80_000: - if len(a) < i: - a.extend(bitarray(i - len(a))) - else: - del a[i:] - assert len(a) == i - show(a) diff -pruN 2.9.2-1/examples/growth/resize.c 3.6.1-1/examples/growth/resize.c --- 2.9.2-1/examples/growth/resize.c 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/growth/resize.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,80 +0,0 @@ -#include -#include - - -typedef struct { - int size; - int nbits; - int allocated; -} bitarrayobject; - - -/* number of bytes necessary to store given bits */ -#define BYTES(bits) (((bits) + 7) >> 3) - - -void resize(bitarrayobject *self, int nbits) -{ - int new_allocated, allocated = self->allocated, size = self->size; - int newsize; - - newsize = BYTES(nbits); - if (newsize == size) { - /* the memory size hasn't changed - bypass almost everything */ - self->nbits = nbits; - return; - } - - /* Bypass realloc() ... */ - if (allocated >= newsize && newsize >= (allocated >> 1)) { - self->size = newsize; - self->nbits = nbits; - return; - } - - if (newsize == 0) { - /* free(self->ob_item) */ - self->size = 0; - self->allocated = 0; - self->nbits = 0; - return; - } - - new_allocated = (newsize + (newsize >> 4) + - (newsize < 8 ? 3 : 7)) & ~(int) 3; - - if (newsize - size > new_allocated - newsize) - new_allocated = (newsize + 3) & ~(int) 3; - - /* realloc(self->ob_item) */ - self->size = newsize; - self->allocated = new_allocated; - self->nbits = nbits; -} - - -int main() -{ - int size, prev_alloc = -1; - bitarrayobject x; - - x.size = 0; - x.allocated = 0; - -#define SHOW printf("%d %d\n", x.size, x.allocated) - - for (size = 0; size < 2000; size++) { - if (prev_alloc != x.allocated) - SHOW; - prev_alloc = x.allocated; - resize(&x, size); - } - - resize(&x, 800000); SHOW; - resize(&x, 400000); SHOW; - resize(&x, 399992); SHOW; - resize(&x, 0); SHOW; - resize(&x, 0); SHOW; - resize(&x, 80000); SHOW; - return 0; -} diff -pruN 2.9.2-1/examples/growth/test.py 3.6.1-1/examples/growth/test.py --- 2.9.2-1/examples/growth/test.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/growth/test.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,51 +0,0 @@ -from bitarray import bitarray - - -def show(a): - info = a.buffer_info() - print('%18d %10d %10d' % (info[0], info[1], info[4])) - - -# make sure sequence of appends will always increase allocated size -a = bitarray() -prev = -1 -while len(a) < 1_000_000: - alloc = a.buffer_info()[4] - assert prev <= alloc - prev = alloc - a.append(1) - - -# ensure that when we start from a large array and delete part, we always -# get a decreasing allocation -a = bitarray(10_000_000) -prev = a.buffer_info()[4] -for _ in range(100): - del a[-100_000:] - alloc = a.buffer_info()[4] - assert alloc <= prev - prev = alloc - - -# initalizing a bitarray from a list or bitarray should not overallocate -for n in range(1000): - alloc = (n + 3) & ~3 - assert n <= alloc < n + 4 and alloc % 4 == 0 - a = bitarray(8 * n * [1]) - assert alloc == a.buffer_info()[4] - b = bitarray(a) - assert alloc == b.buffer_info()[4] - - -# starting from a large bitarray, make we sure we don't realloc each time -# we extend -a = bitarray(1_000_000) # no overallocation -assert a.buffer_info()[4] == 125_000 -a.extend(bitarray(8)) # overallocation happens here -alloc = a.buffer_info()[4] -for _ in range(1000): - a.extend(bitarray(8)) - assert a.buffer_info()[4] == alloc - - -print("OK") diff -pruN 2.9.2-1/examples/hamming.py 3.6.1-1/examples/hamming.py --- 2.9.2-1/examples/hamming.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/examples/hamming.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,122 @@ +# https://www.youtube.com/watch?v=b3NxrZOu_CE +# https://en.wikipedia.org/wiki/Hamming_code +from bitarray.util import xor_indices, int2ba, parity + + +class Hamming: + + def __init__(self, r): + self.r = r + self.n = 1 << r # block length + self.k = self.n - r - 1 # message length + + self.parity_bits = [0] # the 0th bit is to make overall parity 0 + i = 1 + while i < self.n: + self.parity_bits.append(i) + i <<= 1 + + def send(self, a): + "encode message inplace" + if len(a) != self.k: + raise ValueError("expected bitarray of message length %d" % self.k) + for i in self.parity_bits: + a.insert(i, 0) + + # prepare block + c = xor_indices(a) + a[self.parity_bits[1:]] = int2ba(c, length=self.r, endian="little") + a[0] = parity(a) + + def receive(self, a): + "decode inplace and return number of bit errors" + if len(a) != self.n: + raise ValueError("expected bitarray of block length %d" % self.n) + p = parity(a) + c = xor_indices(a) + a.invert(c) # fix bit error + del a[self.parity_bits] + + if p: # overall parity is wrong, so we have a 1 bit error + return 1 + if c: # overall parity is OK, but since we have wrong partial + # parities, there must have been 2 bit errors + return 2 + # overall parity as well as partial parities as fine, so no error + return 0 + +# --------------------------------------------------------------------------- + +from random import getrandbits, randint +import unittest + +from bitarray import bitarray +from bitarray.util import urandom, count_xor + + +class HammingTests(unittest.TestCase): + + def test_init(self): + for r, n, k in [ + (1, 2, 0), ( 8, 256, 247), + (2, 4, 1), (16, 65536, 65519), + (3, 8, 4), + (4, 16, 11), + (5, 32, 26), + (6, 64, 57), + ]: + h = Hamming(r) + self.assertEqual(h.r, r) + self.assertEqual(h.n, n) + self.assertEqual(h.k, k) + self.assertEqual(len(h.parity_bits), h.r + 1) + + def check_well_prepared(self, a): + n = len(a) + self.assertEqual(n & (n - 1), 0) # n is power of 2 + self.assertEqual(xor_indices(a), 0) # partial parity bits are 0 + self.assertEqual(parity(a), 0) # overall parity is 0 + + def test_example(self): + a = bitarray(" 0 010 111 0110") + # 012 4 8 + c = a.copy() + b = bitarray("1100 1010 1111 0110") + # 012 4 8 + h = Hamming(4) + self.check_well_prepared(b) + h.send(a) + self.assertEqual(a, b) + a.invert(10) + self.assertEqual(h.receive(a), 1) + self.assertEqual(a, c) + + def test_send(self): + for _ in range(1000): + h = Hamming(randint(2, 10)) + a = urandom(h.k) + h.send(a) + self.assertEqual(len(a), h.n) + self.check_well_prepared(a) + + def test_receive(self): + for _ in range(1000): + h = Hamming(randint(2, 10)) + a = urandom(h.k) + b = a.copy() + h.send(a) + t = a.copy() + for _ in range(randint(0, 2)): + a.invert(getrandbits(h.r)) + dist = count_xor(a, t) + self.assertTrue(0 <= dist <= 2) + + res = h.receive(a) + self.assertEqual(len(a), h.k) + if dist <= 1: + self.check_well_prepared(t) + self.assertEqual(a, b) + self.assertEqual(res, dist) + +if __name__ == '__main__': + unittest.main() diff -pruN 2.9.2-1/examples/hexadecimal.py 3.6.1-1/examples/hexadecimal.py --- 2.9.2-1/examples/hexadecimal.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/hexadecimal.py 2025-08-12 08:35:42.000000000 +0000 @@ -20,11 +20,11 @@ for k, v in CODEDICT['big'].items(): # t CODEDICT['little'][k] = v[::-1] # type: ignore def prefix_ba2hex(a): - return ''.join(a.iterdecode(CODEDICT[a.endian()])) + return ''.join(a.decode(CODEDICT[a.endian])) def prefix_hex2ba(s, endian=None): a = bitarray(0, endian or get_default_endian()) - a.encode(CODEDICT[a.endian()], s) + a.encode(CODEDICT[a.endian], s) return a # ----- test diff -pruN 2.9.2-1/examples/huffman/canonical.py 3.6.1-1/examples/huffman/canonical.py --- 2.9.2-1/examples/huffman/canonical.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/huffman/canonical.py 2025-08-12 08:35:42.000000000 +0000 @@ -40,7 +40,7 @@ def main(): a = bitarray() a.encode(code, plain) - assert bytearray(a.iterdecode(code)) == plain + assert bytearray(a.decode(code)) == plain assert bytearray(canonical_decode(a, count, symbol)) == plain diff -pruN 2.9.2-1/examples/huffman/compress.py 3.6.1-1/examples/huffman/compress.py --- 2.9.2-1/examples/huffman/compress.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/huffman/compress.py 2025-08-12 08:35:42.000000000 +0000 @@ -56,10 +56,10 @@ def decode(filename): with open(filename, 'rb') as fi: stream = iter(fi.read()) code = decode_code(stream) - a = deserialize(stream) + a = deserialize(bytes(stream)) with open(filename[:-5] + '.out', 'wb') as fo: - fo.write(bytearray(a.iterdecode(code))) + fo.write(bytearray(a.decode(code))) def main(): p = OptionParser("usage: %prog [options] FILE") diff -pruN 2.9.2-1/examples/huffman/decoding.py 3.6.1-1/examples/huffman/decoding.py --- 2.9.2-1/examples/huffman/decoding.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/huffman/decoding.py 2025-08-12 08:35:42.000000000 +0000 @@ -63,7 +63,7 @@ def main(): # Time the decode method which is implemented in C t0 = perf_counter() - res = bytearray(a.iterdecode(code)) + res = bytearray(a.decode(code)) C_time = perf_counter() - t0 print('C decode: %9.3f ms' % (1000.0 * C_time)) assert res == plain diff -pruN 2.9.2-1/examples/lexico.py 3.6.1-1/examples/lexico.py --- 2.9.2-1/examples/lexico.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/lexico.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,67 +1,108 @@ # issue 6 # http://www-graphics.stanford.edu/~seander/bithacks.html#NextBitPermutation -from bitarray import bitarray, get_default_endian -from bitarray.util import ba2int, int2ba, zeros - -from math import comb +from bitarray import bitarray +from bitarray.util import zeros, ba2int, int2ba def all_perm(n, k, endian=None): """all_perm(n, k, endian=None) -> iterator -Return an iterator over all bitarrays of length `n` with `k` bits set to 1 -in lexicographical order. +Return an iterator over all bitarrays of length `n` and +population count `k` in lexicographical order. """ - n = int(n) if n < 0: raise ValueError("length must be >= 0") - k = int(k) - if k < 0 or k > n: - raise ValueError("number of set bits must be in range(0, n + 1)") - - if k == 0: - yield zeros(n, endian) - return + + # error check inputs and handle edge cases + if k <= 0 or k > n: + if k == 0: + yield zeros(n, endian) + return + raise ValueError("k must be in range 0 <= k <= n, got %s" % k) v = (1 << k) - 1 - for _ in range(comb(n, k)): - yield int2ba(v, length=n, - endian=get_default_endian() if endian is None else endian) + while True: + try: + yield int2ba(v, length=n, endian=endian) + except OverflowError: + return t = (v | (v - 1)) + 1 v = t | ((((t & -t) // (v & -v)) >> 1) - 1) -def next_perm(a): - """next_perm(bitarray) -> bitarray +def next_perm(__a): + """next_perm(a, /) -> bitarray -Return the next lexicographical permutation. The length and the number -of 1 bits in the bitarray is unchanged. The integer value (`ba2int`) of the -next permutation will always increase, except when the cycle is completed (in -which case the lowest lexicographical permutation will be returned). +Return the next lexicographical permutation of bitarray `a`. The length +and population count of the result is that of `a`. The integer +value (`ba2int()`) of the next permutation will always increase, except +when the cycle is completed. In that case, the lowest lexicographical +permutation will be returned. """ - v = ba2int(a) + v = ba2int(__a) if v == 0: - return a + return __a + t = (v | (v - 1)) + 1 - w = t | ((((t & -t) // (v & -v)) >> 1) - 1) + v = t | ((((t & -t) // (v & -v)) >> 1) - 1) try: - return int2ba(w, length=len(a), endian=a.endian()) + return int2ba(v, length=len(__a), endian=__a.endian) except OverflowError: - return a[::-1] + return __a[::-1] # --------------------------------------------------------------------------- import unittest +from math import comb +from random import choice, getrandbits, randrange +from itertools import pairwise from bitarray import frozenbitarray -from bitarray.util import urandom -from bitarray.test_bitarray import Util +from bitarray.util import random_k -class PermTests(unittest.TestCase, Util): +class PermTests(unittest.TestCase): - def test_explicit_1(self): + def test_errors(self): + N = next_perm + self.assertRaises(TypeError, N) + self.assertRaises(TypeError, N, bitarray('1'), 1) + self.assertRaises(TypeError, N, '1') + self.assertRaises(ValueError, N, bitarray()) + + A = all_perm + self.assertRaises(TypeError, A) + self.assertRaises(TypeError, A, 4) + self.assertRaises(TypeError, next, A("4", 2)) + self.assertRaises(TypeError, next, A(1, "0.5")) + self.assertRaises(TypeError, A, 1, p=1) + self.assertRaises(TypeError, next, A(11, 5.5)) + self.assertRaises(ValueError, next, A(-1, 0)) + for k in -1, 11: # k is not 0 <= k <= n + self.assertRaises(ValueError, next, A(10, k)) + self.assertRaises(ValueError, next, A(10, 7, 'foo')) + self.assertRaises(ValueError, next, A(10, 7, endian='foo')) + + def test_zeros_ones(self): + for n in range(1, 30): + endian = choice(["little", "big"]) + v = getrandbits(1) + + lst = list(all_perm(n, v * n, endian)) + self.assertEqual(len(lst), 1) + a = lst[0] + c = a.copy() + self.assertEqual(a.endian, endian) + self.assertEqual(len(a), n) + if v: + self.assertTrue(a.all()) + else: + self.assertFalse(a.any()) + self.assertEqual(next_perm(a), a) + self.assertEqual(a, c) + + def test_next_perm_explicit(self): a = bitarray('00010011', 'big') for s in ['00010101', '00010110', '00011001', '00011010', '00011100', '00100011']: @@ -69,76 +110,44 @@ class PermTests(unittest.TestCase, Util) self.assertEqual(a.count(), 3) self.assertEqual(a, bitarray(s, 'big')) - def test_explicit_2(self): - for seq in (['0'], ['1'], ['00'], ['11'], ['01', '10'], - ['001', '010', '100'], ['011', '101', '110'], - ['0011', '0101', '0110', '1001', '1010', '1100']): - a = bitarray(seq[0], 'big') - for i in range(20): - self.assertEqual(a, bitarray(seq[i % len(seq)])) - a = next_perm(a) - - def test_all_same(self): - for endian in 'little', 'big': - for n in range(1, 30): - for v in 0, 1: - a = bitarray(n, endian) - a.setall(v) - self.assertEqual(next_perm(a), a) - - def test_turnover(self): + def test_next_perm_turnover(self): for a in [bitarray('11111110000', 'big'), bitarray('0000001111111', 'little')]: self.assertEqual(next_perm(a), a[::-1]) - def test_large(self): - a = bitarray('10010101010100100110010101110100111100101111', 'big') - b = next_perm(a) - c = bitarray('10010101010100100110010101110100111100110111') - self.assertEqual(b, c) - - def test_errors(self): - self.assertRaises(ValueError, next_perm, bitarray()) - self.assertRaises(TypeError, next_perm, '1') - - def check_all_perm(self, s): - s1 = s.count(1) - n = 0 - a = bitarray(s) + def test_next_perm_random(self): + for _ in range(100): + n = randrange(2, 1_000_000) + k = randrange(1, n) + a = random_k(n, k, endian=choice(["little", "big"])) + b = next_perm(a) + self.assertEqual(len(b), n) + self.assertEqual(b.count(), k) + self.assertEqual(b.endian, a.endian) + self.assertNotEqual(a, b) + if ba2int(a) > ba2int(b): + print(n) + c = a.copy() + c.sort(c.endian == 'big') + self.assertEqual(a, c) + self.assertEqual(b, a[::-1]) + + def check_perm_cycle(self, start): + n, k = len(start), start.count() + a = bitarray(start) coll = set() - while 1: + c = 0 + while True: a = next_perm(a) coll.add(frozenbitarray(a)) - self.assertEqual(len(a), len(s)) - self.assertEqual(a.count(), s1) - self.assertEqual(a.endian(), s.endian()) - n += 1 - if a == s: + self.assertEqual(len(a), n) + self.assertEqual(a.count(), k) + self.assertEqual(a.endian, start.endian) + c += 1 + if a == start: break - self.assertEqual(n, comb(len(s), s1)) - self.assertEqual(len(coll), n) - - def check_order(self, a): - i = -1 - for _ in range(comb(len(a), a.count())): - i, j = ba2int(a), i - self.assertTrue(i > j) - a = next_perm(a) - - def test_few(self): - for s in '0', '1', '00', '01', '111', '0011', '01011', '000000011': - for endian in 'little', 'big': - a = bitarray(s, endian) - self.check_all_perm(a) - a.sort(a.endian() == 'little') - self.check_order(a) - - def test_random(self): - for n in range(1, 10): - a = urandom(n, self.random_endian()) - self.check_all_perm(a) - a.sort(a.endian() == 'little') - self.check_order(a) + self.assertEqual(c, comb(n, k)) + self.assertEqual(len(coll), c) def test_all_perm_explicit(self): for n, k, res in [ @@ -147,25 +156,62 @@ class PermTests(unittest.TestCase, Util) (1, 1, ['1']), (2, 0, ['00']), (2, 1, ['01', '10']), + (2, 2, ['11']), + (3, 0, ['000']), + (3, 1, ['001', '010', '100']), (3, 2, ['011', '101', '110']), - ]: - self.assertEqual(list(all_perm(n, k, 'big')), - [bitarray(s) for s in res]) + (3, 3, ['111']), + (4, 2, ['0011', '0101', '0110', '1001', '1010', '1100']), + ]: + lst = list(all_perm(n, k, 'big')) + self.assertEqual(len(lst), comb(n, k)) + self.assertEqual(lst, [bitarray(s) for s in res]) + if n == 0: + continue + a = lst[0] + for i in range(20): + self.assertEqual(a, bitarray(res[i % len(lst)])) + a = next_perm(a) - def test_all_perm_1(self): - n, k = 10, 5 - c = 0 - s = set() - for a in all_perm(n, k, 'little'): - self.assertIsType(a, 'bitarray') + def test_all_perm(self): + n, k = 17, 5 + endian=choice(["little", "big"]) + + prev = None + cnt = 0 + coll = set() + for a in all_perm(n, k, endian): + self.assertEqual(type(a), bitarray) self.assertEqual(len(a), n) self.assertEqual(a.count(), k) - s.add(frozenbitarray(a)) - c += 1 - self.assertEqual(c, comb(n, k)) - self.assertEqual(len(s), comb(n, k)) + self.assertEqual(a.endian, endian) + coll.add(frozenbitarray(a)) + if prev is None: + first = a.copy() + c = a.copy() + c.sort(c.endian == "little") + self.assertEqual(a, c) + else: + self.assertNotEqual(a, first) + self.assertEqual(next_perm(prev), a) + self.assertTrue(ba2int(prev) < ba2int(a)) + prev = a + cnt += 1 + + self.assertEqual(cnt, comb(n, k)) + self.assertEqual(len(coll), cnt) + + # a is now the last permutation + last = a.copy() + self.assertTrue(ba2int(first) < ba2int(last)) + self.assertEqual(last, first[::-1]) + + def test_all_perm_order(self): + n, k = 10, 5 + for a, b in pairwise(all_perm(n, k, 'little')): + self.assertTrue(ba2int(b) > ba2int(a)) + self.assertEqual(next_perm(a), b) -# --------------------------------------------------------------------------- if __name__ == '__main__': unittest.main() diff -pruN 2.9.2-1/examples/lfsr.py 3.6.1-1/examples/lfsr.py --- 2.9.2-1/examples/lfsr.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/examples/lfsr.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,89 @@ +# Linear Feedback Shift Register +# https://www.youtube.com/watch?v=Ks1pw1X22y4 +# https://en.wikipedia.org/wiki/Linear-feedback_shift_register +from bitarray import bitarray +from bitarray.util import parity + + +def get_period(tabs, verbose=False): + "given a bitarray of tabs return period of lfsr" + n = len(tabs) + state0 = bitarray(n) + state0[0] = state0[-1] = 1 + state = state0.copy() + period = 0 + while True: + if verbose: + print(state[0], end='') + newbit = parity(state & tabs) + state <<= 1 + state[-1] = newbit + period += 1 + if state == state0: + break + if verbose: + print() + return period + +def simple(): + "example from computerphile" + state = 0b1001 + for _ in range(20): + print(state & 1, end='') + newbit = (state ^ (state >> 1)) & 1 + state = (state >> 1) | (newbit << 3) + print() + get_period(bitarray("1100"), True) + +def test_wiki(): + "test list of tabs shown on Wikipedia" + all_tabs = [ + "11", + "110", + "1100", + "10100", + "110000", + "1100000", + "10111000", + "100010000", + "1001000000", + "10100000000", + "111000001000", + "1110010000000", + "11100000000010", + "110000000000000", + "1101000000001000", + "10010000000000000", + "100000010000000000", + "1110010000000000000", + "10010000000000000000", + "101000000000000000000", + "1100000000000000000000", + "10000100000000000000000", + "111000010000000000000000", + ] + for tabs in all_tabs: + tabs = bitarray(tabs) + period = get_period(tabs) + print(period) + n = len(tabs) + assert period == 2 ** n - 1 + assert parity(tabs) == 0 + +def n128(): + tabs = bitarray(128) + tabs[[0, 1, 2, 7]] = 1 + + state = bitarray(128) + state[0] = state[-1] = 1 + + while True: + print(state[0], end='') + newbit = parity(state & tabs) + state <<= 1 + state[-1] = newbit + +if __name__ == "__main__": + simple() + test_wiki() + #n128() diff -pruN 2.9.2-1/examples/masked.py 3.6.1-1/examples/masked.py --- 2.9.2-1/examples/masked.py 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/examples/masked.py 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,17 @@ +# example to illustrate masked indexing +from bitarray import bitarray + + +a = bitarray('1110000') +b = bitarray('1100110') +# select bits from a where b is 1 +assert a[b] == bitarray('1100') + +# set bits in a where b is 1 +a[b] = bitarray('1010') +assert a == bitarray('1010100') + +# delete bits in a where b is 1 +del a[b] +assert a == bitarray('100') +print("Ok") diff -pruN 2.9.2-1/examples/mmapped-file.py 3.6.1-1/examples/mmapped-file.py --- 2.9.2-1/examples/mmapped-file.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/mmapped-file.py 2025-08-12 08:35:42.000000000 +0000 @@ -14,7 +14,7 @@ filesize = 10_000_000 with open(filename, 'wb') as fo: fo.write(filesize * b'\0') -# open the file in binary read-write mode for mapping into a bitarray +# open file in binary read-write mode for mapping into bitarray with open(filename, 'r+b') as f: mapping = mmap.mmap(f.fileno(), 0) a = bitarray(buffer=mapping, endian='little') diff -pruN 2.9.2-1/examples/ndarray.py 3.6.1-1/examples/ndarray.py --- 2.9.2-1/examples/ndarray.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/ndarray.py 2025-08-12 08:35:42.000000000 +0000 @@ -2,8 +2,6 @@ # This example illusatrates how binary data can be efficiently be passed # between a bitarray object and an ndarray with dtype bool # -from __future__ import print_function - import bitarray import numpy # type: ignore diff -pruN 2.9.2-1/examples/puff/_puff.c 3.6.1-1/examples/puff/_puff.c --- 2.9.2-1/examples/puff/_puff.c 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/puff/_puff.c 2025-08-12 08:35:42.000000000 +0000 @@ -112,7 +112,7 @@ append_byte(state_obj *s, int byte) { char *cp; - if (byte < 0 || byte > 0xff) { + if (byte >> 8) { PyErr_Format(PyExc_ValueError, "invalid byte: %d", byte); return -1; } @@ -131,7 +131,7 @@ append_byte(state_obj *s, int byte) static int dist_len_copy(state_obj *s, int dist, int len) { - char *out; + char *str; if (len < 0) { PyErr_SetString(PyExc_ValueError, "length cannot be negative"); @@ -150,9 +150,9 @@ dist_len_copy(state_obj *s, int dist, in return -1; } - out = PyByteArray_AS_STRING(s->out); + str = PyByteArray_AS_STRING(s->out); while (len--) { - out[s->outcnt] = out[s->outcnt - dist]; + str[s->outcnt] = str[s->outcnt - dist]; s->outcnt++; } @@ -265,23 +265,7 @@ codes(state_obj *s, const struct huffman /* ------------------------ State Python interface ------------------ */ /* set during module init */ -static PyObject *bitarray_type_obj; - -/* Return 0 if obj is bitarray. If not, return -1 and set an exception. */ -static int -ensure_bitarray(PyObject *obj) -{ - int t; - - t = PyObject_IsInstance(obj, bitarray_type_obj); - if (t < 0) - return -1; - if (t == 0) { - PyErr_SetString(PyExc_TypeError, "bitarray expected"); - return -1; - } - return 0; -} +static PyTypeObject *bitarray_type; /* create a new initialized canonical Huffman decode iterator object */ static PyObject * @@ -292,8 +276,11 @@ state_new(PyTypeObject *type, PyObject * if (!PyArg_ParseTuple(args, "OO:State", &in, &out)) return NULL; - if (ensure_bitarray(in) < 0) + + if (!PyObject_TypeCheck(in, bitarray_type)) { + PyErr_SetString(PyExc_TypeError, "bitarray expected"); return NULL; + } if (!PyByteArray_Check(out)) { PyErr_SetString(PyExc_TypeError, "bytearary expected"); return NULL; @@ -339,7 +326,7 @@ state_extend_block(state_obj *self, PyOb nbytes = PyNumber_AsSsize_t(value, NULL); if (nbytes == -1 && PyErr_Occurred()) return NULL; - if (nbytes < 0 || nbytes > 0xffff) + if (nbytes >> 16) return PyErr_Format(PyExc_ValueError, "invalid block size: %zd", nbytes); if (self->incnt % 8 != 0) { @@ -615,7 +602,7 @@ static PyMethodDef state_methods[] = { {"copy", (PyCFunction) state_copy, METH_VARARGS, 0}, {"get_incnt", (PyCFunction) state_get_incnt, METH_NOARGS, 0}, {"read_uint", (PyCFunction) state_read_uint, METH_O, 0}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; static void @@ -650,13 +637,13 @@ PyMODINIT_FUNC PyInit__puff(void) if ((bitarray_module = PyImport_ImportModule("bitarray")) == NULL) return NULL; - bitarray_type_obj = PyObject_GetAttrString(bitarray_module, "bitarray"); + bitarray_type = (PyTypeObject *) PyObject_GetAttrString(bitarray_module, + "bitarray"); Py_DECREF(bitarray_module); - if (bitarray_type_obj == NULL) + if (bitarray_type == NULL) return NULL; - m = PyModule_Create(&moduledef); - if (m == NULL) + if ((m = PyModule_Create(&moduledef)) == NULL) return NULL; Py_SET_TYPE(&state_type, &PyType_Type); diff -pruN 2.9.2-1/examples/puff/setup.py 3.6.1-1/examples/puff/setup.py --- 2.9.2-1/examples/puff/setup.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/puff/setup.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,9 +1,5 @@ -import sys from os.path import dirname -if sys.version_info[:2] < (3, 6): - sys.exit("This example requires Python 3.6 or higher") - try: from setuptools import setup, Extension except ImportError: diff -pruN 2.9.2-1/examples/puff/test_puff.py 3.6.1-1/examples/puff/test_puff.py --- 2.9.2-1/examples/puff/test_puff.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/puff/test_puff.py 2025-08-12 08:35:42.000000000 +0000 @@ -41,8 +41,7 @@ class TestState(unittest.TestCase): self.assertEqual(len(out), 0) # nothing in output def test_read_uint32(self): - a = bitarray(endian='little') - a.frombytes(b'\x7e\xae\xd4\xbb') + a = bitarray(b'\x7e\xae\xd4\xbb', endian='little') s = State(a, bytearray()) self.assertEqual(s.read_uint(32), 0xbbd4ae7e) self.assertEqual(s.get_incnt(), 32) @@ -77,8 +76,7 @@ class TestState(unittest.TestCase): self.assertEqual(bytes(out), b'\0\xff') def test_extend_block(self): - a = bitarray() - a.frombytes(b'ABCDEF') + a = bitarray(b'ABCDEF') b = bytearray() s = State(a, b) @@ -226,8 +224,7 @@ class TestPuff(unittest.TestCase): def round_trip(self, data, level=-1): compressed = zlib.compress(data, level=level) - a = bitarray(0, 'little') - a.frombytes(compressed) + a = bitarray(compressed, 'little') out = bytearray() p = Puff(a, out) # check zlib header diff -pruN 2.9.2-1/examples/shift_r8.c 3.6.1-1/examples/shift_r8.c --- 2.9.2-1/examples/shift_r8.c 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/shift_r8.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,149 +0,0 @@ -/* - The purpose of this C program is to illustrate the functions shift_r8le() - and shift_r8be(), which are called from shift_r8(). - These functions are symmetrical with the following replacements: - - PY_LITTLE_ENDIAN <-> PY_BIG_ENDIAN - <<= >>= - >> << - - Creating a macro from which both functions can be created is not possible, - unless one replaces the existing preprocessor introductions with ordinary - if statements. For the sake of simplicity we do not want to do this here, - even though it would avoid the spammish repetition. -*/ -#include -#include -#include - -#define Py_ssize_t ssize_t - -static inline uint64_t -builtin_bswap64(uint64_t word) -{ -#if (defined(__clang__) || \ - (defined(__GNUC__) \ - && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 3)))) - /* __builtin_bswap64() is available since GCC 4.3. */ -# define HAVE_BUILTIN_BSWAP64 1 - return __builtin_bswap64(word); -#elif defined(_MSC_VER) -# define HAVE_BUILTIN_BSWAP64 1 - return _byteswap_uint64(word); -#else -# define HAVE_BUILTIN_BSWAP64 0 - abort() -#endif -} - -/* machine byte-order */ -#define PY_LITTLE_ENDIAN 1 -#define PY_BIG_ENDIAN 0 - -/* bit-endianness */ -#define ENDIAN_LITTLE 0 -#define ENDIAN_BIG 1 - -#define BITMASK(endian, i) (((char) 1) << (endian == ENDIAN_LITTLE ? \ - ((i) % 8) : (7 - (i) % 8))) - -/* The following two functions operate on first n bytes in buffer. - Within this region, they shift all bits by k positions to right, - i.e. towards higher addresses. - They operate on little-endian and bit-endian bitarrays respectively. - As we shift right, we need to start with the highest address and loop - downwards such that lower bytes are still unaltered. - See also examples/shift_r8.c -*/ -static void -shift_r8le(unsigned char *buff, Py_ssize_t n, int k) -{ - Py_ssize_t w = 0; - -#if HAVE_BUILTIN_BSWAP64 || PY_LITTLE_ENDIAN /* use shift word */ - w = n / 8; /* number of words used for shifting */ - n %= 8; /* number of remaining bytes */ -#endif - while (n--) { /* shift in byte-range(8 * w, n) */ - Py_ssize_t i = n + 8 * w; - buff[i] <<= k; /* shift byte */ - if (n || w) /* add shifted next lower byte */ - buff[i] |= buff[i - 1] >> (8 - k); - } - while (w--) { /* shift in word-range(0, w) */ - uint64_t *p = ((uint64_t *) buff) + w; -#if HAVE_BUILTIN_BSWAP64 && PY_BIG_ENDIAN - *p = builtin_bswap64(*p); - *p <<= k; - *p = builtin_bswap64(*p); -#else - *p <<= k; -#endif - if (w) /* add shifted byte from next lower word */ - buff[8 * w] |= buff[8 * w - 1] >> (8 - k); - } -} - -static void -shift_r8be(unsigned char *buff, Py_ssize_t n, int k) -{ - Py_ssize_t w = 0; - -#if HAVE_BUILTIN_BSWAP64 || PY_BIG_ENDIAN /* use shift word */ - w = n / 8; /* number of words used for shifting */ - n %= 8; /* number of remaining bytes */ -#endif - while (n--) { /* shift in byte-range(8 * w, n) */ - Py_ssize_t i = n + 8 * w; - buff[i] >>= k; /* shift byte */ - if (n || w) /* add shifted next lower byte */ - buff[i] |= buff[i - 1] << (8 - k); - } - while (w--) { /* shift in word-range(0, w) */ - uint64_t *p = ((uint64_t *) buff) + w; -#if HAVE_BUILTIN_BSWAP64 && PY_LITTLE_ENDIAN - *p = builtin_bswap64(*p); - *p >>= k; - *p = builtin_bswap64(*p); -#else - *p >>= k; -#endif - if (w) /* add shifted byte from next lower word */ - buff[8 * w] |= buff[8 * w - 1] << (8 - k); - } -} - -/* display first nbits bytes of buffer given assumed bit-endianness - to one line in stdout */ -void display(unsigned char *buffer, Py_ssize_t nbits, int endian) -{ - Py_ssize_t i; - - for (i = 0; i < nbits; i++) - printf("%d", (buffer[i / 8] & BITMASK(endian, i)) ? 1 : 0); - - printf("\n"); -} - -int main() -{ -#define NBYTES 10 - unsigned char array[NBYTES] = {1, 15, 0, 131, 0, 255, 0, 7, 0, 1}; - ssize_t i; - - if ((PY_LITTLE_ENDIAN != (*((uint64_t *) "\xff\0\0\0\0\0\0\0") == 0xff)) - || - (PY_BIG_ENDIAN != (*((uint64_t *) "\0\0\0\0\0\0\0\xff") == 0xff))) { - printf("Error: machine byte-order\n"); - return 1; - } - for (i = 0; i < 15; i++) { - display(array, 77, ENDIAN_LITTLE); - shift_r8le(array, NBYTES, 1); - } - for (i = 0; i < 15; i++) { - display(array, 77, ENDIAN_BIG); - shift_r8be(array, NBYTES, 1); - } - return 0; -} diff -pruN 2.9.2-1/examples/sieve.py 3.6.1-1/examples/sieve.py --- 2.9.2-1/examples/sieve.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/sieve.py 2025-08-12 08:35:42.000000000 +0000 @@ -5,7 +5,7 @@ finding all prime numbers up to any give from math import isqrt from bitarray import bitarray -from bitarray.util import ones, count_n +from bitarray.util import ones, count_n, sum_indices N = 100_000_000 @@ -21,24 +21,30 @@ for i in range(2, isqrt(N) + 1): if a[i]: # i is prime, so all multiples are not a[i*i::i] = False -print('the first few primes are:') -it = a.itersearch(1) -print([next(it) for _ in range(20)]) +stop = 50 +print('primes up to %d are:' % stop) +print(list(a.search(1, 0, stop))) # There are 5,761,455 primes up to 100 million x = a.count() -print('there are %d primes up to %d' % (x, N)) +print('there are {:,d} primes up to {:,d}'.format(x, N)) assert x == 5_761_455 or N != 100_000_000 # The number of twin primes up to 100 million is 440,312 # we need to add 1 as .count() only counts non-overlapping sub_bitarrays # and (3, 5) as well as (5, 7) are both twin primes x = a.count(bitarray('101')) + 1 -print('number of twin primes up to %d is %d' % (N, x)) +print('number of twin primes up to {:,d} is {:,d}'.format(N, x)) assert x == 440_312 or N != 100_000_000 # The 1 millionth prime number is 15,485,863 m = 1_000_000 x = count_n(a, m) - 1 -print('the %dth prime is %d' % (m, x)) +print('the {:,d}-th prime is {:,d}'.format(m, x)) assert x == 15_485_863 + +# The sum of all prime numbers below one million is 37,550,402,023. +m = 1_000_000 +x = sum_indices(a[:m]) +print('the sum of prime numbers below {:,d} is {:,d}'.format(m, x)) +assert x == 37_550_402_023 diff -pruN 2.9.2-1/examples/sparse/compress.py 3.6.1-1/examples/sparse/compress.py --- 2.9.2-1/examples/sparse/compress.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/sparse/compress.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,183 +0,0 @@ -# Useful functions related to sparse bitarray compression. -# In particular the function sc_stats() which returns the -# frequency of each block type. - -import bz2 -import gzip -import sys -from time import perf_counter -from collections import Counter -from itertools import islice -from random import random, randrange - -from bitarray import bitarray -from bitarray.util import ( - zeros, urandom, - serialize, deserialize, - sc_encode, sc_decode, - vl_encode, vl_decode, -) - -def read_n(n, stream): - i = 0 - for j in range(n): - i |= next(stream) << 8 * j - if i < 0: - raise ValueError("read %d bytes got negative value: %d" % (n, i)) - return i - -def sc_decode_header(stream): - head = next(stream) - if head & 0xe0: - raise ValueError("invalid header: 0x%02x" % head) - endian = 'big' if head & 0x10 else 'little' - length = head & 0x0f - nbits = read_n(length, stream) - return endian, nbits - -def sc_decode_block(stream, stats): - head = next(stream) - if head == 0: # stop byte - return False - - if head < 0xa0: - n = 0 - k = head if head <= 32 else 32 * (head - 31) - elif head < 0xc0: - n = 1 - k = head - 0xa0 - elif 0xc2 <= head <= 0xc4: - n = head - 0xc0 - k = next(stream) - else: - raise ValueError("Invalid block head: 0x%02x" % head) - - stats['blocks'][n] += 1 - - # consume block data - size = max(1, n) * k # size of block data - next(islice(stream, size, size), None) - - return True - -def sc_stats(stream): - """sc_stats(stream) -> dict - -Decode a compressed byte stream (generated by `sc_encode()` and return -useful statistics. In particular, the frequency of each block type. -""" - stream = iter(stream) - endian, nbits = sc_decode_header(stream) - - stats = { - 'endian': endian, - 'nbits': nbits, - 'blocks': Counter() - } - - while sc_decode_block(stream, stats): - pass - - stop = False - try: - next(stream) - except StopIteration: - stop = True - assert stop - - return stats - -def test_sc_stat(): - a = bitarray(1<<33, 'little') - a.setall(0) - a[:1<<16] = 1 - a[:1<<18:1<<4] = 1 - a[:1<<22:1<<12] = 1 - a[:1<<30:1<<20] = 1 - assert a.count() == 79804 - b = sc_encode(a) - stat = sc_stats(b) - assert stat['endian'] == 'little' - assert stat['nbits'] == 1 << 33 - blocks = stat['blocks'] - for i, n in enumerate([2, 754, 46, 48, 1]): - print(" block type %d %8d" % (i, blocks[i])) - assert blocks[i] == n - if sys.version_info[:2] >= (3, 10): - print("total number of blocks %8d" % blocks.total()) - assert a == sc_decode(b) - -def random_array(n, p=0.5): - """random_array(n, p=0.5) -> bitarray - -Generate random bitarray of length n. -Each bit has a probability p of being 1. -""" - if p < 0.05: - # XXX what happens for small N? N=0 crashes right now. - # when the probability p is small, it is faster to randomly - # set p * n elements - a = zeros(n) - for _ in range(int(p * n)): - a[randrange(n)] = 1 - return a - - return bitarray((random() < p for _ in range(n))) - -def test_random_array(): - n = 10_000_000 - p = 1e-6 - while p < 1.0: - a = random_array(n, p) - cnt = a.count() - print("%10.7f %10.7f %10.7f" % (p, cnt / n, abs(p - cnt / n))) - p *= 1.4 - -def p_range(): - n = 1 << 28 - p = 1e-8 - print(" p ratio raw" - " type 1 type 2 type 3 type 4") - print(" " + 73 *'-') - while p < 1.0: - a = random_array(n, p) - b = sc_encode(a) - blocks = sc_stats(b)['blocks'] - print(' %11.8f %11.8f %8d %8d %8d %8d %8d' % ( - p, len(b) / (n / 8), - blocks[0], blocks[1], blocks[2], blocks[3], blocks[4])) - assert a == sc_decode(b) - p *= 1.8 - -def compare(): - n = 1 << 26 - # create random bitarray with p = 1 / 2^9 = 1 / 512 = 0.195 % - a = bitarray(n) - a.setall(1) - for i in range(10): - a &= urandom(n) - - raw = a.tobytes() - print(20 * ' ' + "compress (ms) decompress (ms) ratio") - print(70 * '-') - for name, f_e, f_d in [ - ('serialize', serialize, deserialize), - ('vl', vl_encode, vl_decode), - ('sc' , sc_encode, sc_decode), - ('gzip', gzip.compress, gzip.decompress), - ('bz2', bz2.compress, bz2.decompress)]: - x = a if name in ('serialize', 'vl', 'sc') else raw - t0 = perf_counter() - b = f_e(x) # compression - t1 = perf_counter() - c = f_d(b) # decompression - t2 = perf_counter() - print(" %-11s %16.3f %16.3f %16.4f" % - (name, 1000 * (t1 - t0), 1000 * (t2 - t1), len(b) / len(raw))) - assert c == x - -if __name__ == '__main__': - test_sc_stat() - #test_random_array() - compare() - p_range() diff -pruN 2.9.2-1/examples/sparse/tests.py 3.6.1-1/examples/sparse/tests.py --- 2.9.2-1/examples/sparse/tests.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/sparse/tests.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,20 +1,13 @@ import sys -from random import getrandbits, randint, randrange import unittest - -try: - from itertools import pairwise # type: ignore -except ImportError: - from itertools import tee - def pairwise(iterable): # type: ignore - a, b = tee(iterable) - next(b, None) - return zip(a, b) +from itertools import pairwise +from random import getrandbits, randint, randrange from bitarray import bitarray from bitarray.util import intervals from bitarray.test_bitarray import Util + if len(sys.argv) != 2 or sys.argv[1] not in ('flips', 'ones', '-'): sys.exit("Argument 'flips' or 'ones' expected.") MODE = sys.argv[1] diff -pruN 2.9.2-1/examples/utf-8.py 3.6.1-1/examples/utf-8.py --- 2.9.2-1/examples/utf-8.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/examples/utf-8.py 2025-08-12 08:35:42.000000000 +0000 @@ -7,9 +7,8 @@ from bitarray.util import ba2int, pprint def code_point(u): print('character:', u) b = u.encode('utf-8') - print('hexadecimal:', ' '.join('%02x' % i for i in bytearray(b))) - a = bitarray(endian='big') - a.frombytes(b) + print('hexadecimal:', ' '.join('%02x' % i for i in b)) + a = bitarray(b, endian='big') pprint(a) # calculate binary code point from binary UTF-8 representation diff -pruN 2.9.2-1/pyproject.toml 3.6.1-1/pyproject.toml --- 2.9.2-1/pyproject.toml 1970-01-01 00:00:00.000000000 +0000 +++ 3.6.1-1/pyproject.toml 2025-08-12 08:35:42.000000000 +0000 @@ -0,0 +1,4 @@ +# pyproject.toml +[build-system] +requires = ["setuptools >= 42.0.0"] +build-backend = "setuptools.build_meta" diff -pruN 2.9.2-1/setup.py 3.6.1-1/setup.py --- 2.9.2-1/setup.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/setup.py 2025-08-12 08:35:42.000000000 +0000 @@ -3,11 +3,11 @@ import sys import platform -if sys.version_info[0] == 2: - print("""\ +if sys.version_info[:3] < (3, 6, 1): + sys.exit("""\ **************************************************************************** -* Python 2 support of bitarray is deprecated (as of bitarray version 2.9) -* and will be removed in bitarray version 3.0. +* bitarray requires Python 3.6.1 or later. +* The last version supporting Python 2 is bitarray 2.9.3. **************************************************************************** """) @@ -23,10 +23,7 @@ except ImportError: kwds = {} -try: - kwds['long_description'] = open('README.rst').read() -except IOError: - pass +kwds['long_description'] = open('README.rst').read() # Read version from bitarray/bitarray.h pat = re.compile(r'#define\s+BITARRAY_VERSION\s+"(\S+)"', re.M) @@ -34,7 +31,7 @@ data = open('bitarray/bitarray.h').read( kwds['version'] = pat.search(data).group(1) macros = [] -if platform.python_implementation() == 'PyPy' or sys.version_info[0] == 2: +if platform.python_implementation() == 'PyPy': macros.append(("PY_LITTLE_ENDIAN", str(int(sys.byteorder == 'little')))) macros.append(("PY_BIG_ENDIAN", str(int(sys.byteorder == 'big')))) @@ -43,17 +40,13 @@ setup( author = "Ilan Schnell", author_email = "ilanschnell@gmail.com", url = "https://github.com/ilanschnell/bitarray", - license = "PSF", + license = "PSF-2.0", classifiers = [ - "License :: OSI Approved :: Python Software Foundation License", "Development Status :: 6 - Mature", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: C", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", @@ -61,6 +54,7 @@ setup( "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Utilities", ], description = "efficient arrays of booleans -- C extension", diff -pruN 2.9.2-1/update_doc.py 3.6.1-1/update_doc.py --- 2.9.2-1/update_doc.py 2024-01-01 18:50:09.000000000 +0000 +++ 3.6.1-1/update_doc.py 2025-08-12 08:35:42.000000000 +0000 @@ -1,8 +1,7 @@ -import sys -assert sys.version_info[0] == 3, "This program requires Python 3" - import re +import sys from doctest import testfile +from glob import glob from io import StringIO import bitarray.util @@ -11,39 +10,54 @@ import bitarray.util BASE_URL = "https://github.com/ilanschnell/bitarray" NEW_IN = { - 'bitarray': '2.3: optional `buffer` argument', + 'bitarray': ['2.3: optional `buffer` argument', + '3.4: allow initializer `bytes` or `bytearray` ' + 'to set buffer directly'], 'bitarray.bytereverse': '2.2.5: optional start and stop arguments', 'bitarray.clear': '1.4', 'bitarray.count': ['1.1.0: optional start and stop arguments', '2.3.7: optional step argument', '2.9: add non-overlapping sub-bitarray count'], + 'bitarray.decode': '3.0: returns iterator (equivalent to past ' + '`.iterdecode()`)', + 'bitarray.endian': '3.4: replaces former `.endian()` method', + 'bitarray.extend': '3.4: allow `bytes` object', 'bitarray.find': ['2.1', '2.9: add optional keyword argument `right`'], 'bitarray.frombytes': '2.5.0: allow bytes-like argument', 'bitarray.index': ['2.9: add optional keyword argument `right`'], - 'bitarray.itersearch': ['2.9: optional start and stop arguments - ' - 'add optional keyword argument `right`'], 'bitarray.invert': '1.5.3: optional index argument', 'bitarray.pack': '2.5.0: allow bytes-like argument', + 'bitarray.search': ['2.9: optional start and stop arguments - ' + 'add optional keyword argument `right`', + '3.0: returns iterator (equivalent to past ' + '`.itersearch()`)'], + 'bitarray.to01': '3.3: optional `group` and `sep` arguments', 'decodetree': '1.6', 'frozenbitarray': '1.1', 'get_default_endian': '1.3', 'util.any_and': '2.7', - 'util.ba2base': '1.9', - 'util.base2ba': '1.9', + 'util.byteswap': '3.4', + 'util.ba2base': ['1.9', + '3.3: optional `group` and `sep` arguments'], + 'util.base2ba': ['1.9', + '3.3: ignore whitespace'], + 'util.ba2hex': '3.3: optional `group` and `sep` arguments', + 'util.hex2ba': '3.3: ignore whitespace', + 'util.correspond_all': '3.4', 'util.count_n': '2.3.6: optional value argument', 'util.deserialize': ['1.8', '2.5.0: allow bytes-like argument'], 'util.intervals': '2.7', - 'util.make_endian': ['1.3', - '2.9: deprecated - use `bitarray()`'], 'util.ones': '2.9', 'util.parity': '1.9', + 'util.sum_indices': '3.6', + 'util.xor_indices': '3.2', 'util.pprint': '1.8', - 'util.rindex': ['2.3.0: optional start and stop arguments', - '2.9: deprecated - use `.index(..., right=1)`'], 'util.serialize': '1.8', 'util.urandom': '1.7', + 'util.random_k': '3.6', + 'util.random_p': '3.5', 'util.sc_encode': '2.7', 'util.sc_decode': '2.7', 'util.vl_decode': '2.2', @@ -53,18 +67,23 @@ NEW_IN = { } DOCS = { + 'ba3': ('Bitarray 3 transition', 'bitarray3.rst'), 'chc': ('Canonical Huffman Coding', 'canonical.rst'), 'rep': ('Bitarray representations', 'represent.rst'), + 'rnd': ('Random Bitarrays', 'random_p.rst'), 'sc': ('Compression of sparse bitarrays', 'sparse_compression.rst'), 'vlf': ('Variable length bitarray format', 'variable_length.rst'), } DOC_LINKS = { + 'bitarray.decode': 'ba3', + 'bitarray.search': 'ba3', 'util.canonical_huffman': 'chc', 'util.canonical_decode': 'chc', 'util.ba2base': 'rep', 'util.base2ba': 'rep', 'util.deserialize': 'rep', + 'util.random_p': 'rnd', 'util.serialize': 'rep', 'util.sc_decode': 'sc', 'util.sc_encode': 'sc', @@ -87,6 +106,7 @@ NOTES = { } GETSET = { + 'bitarray.endian': 'str', 'bitarray.nbytes': 'int', 'bitarray.padbits': 'int', 'bitarray.readonly': 'bool', @@ -120,7 +140,7 @@ def get_doc(name): if m is None: raise Exception("signature invalid: %r" % lines[0]) sig = '``%s``' % m.group(1) - assert m.group(2) == obj.__name__ + assert m.group(2) == obj.__name__, lines[0] if m.group(4): sig += ' -> %s' % m.group(4) assert lines[1] == '' @@ -148,7 +168,7 @@ def write_doc(fo, name): new_in = NEW_IN.get(name) if new_in: for line in new_in if isinstance(new_in, list) else [new_in]: - fo.write("\n New in version %s.\n" % line.replace('`', '``')) + fo.write("\n New in version %s\n" % line.replace('`', '``')) fo.write('\n\n') @@ -202,7 +222,7 @@ The bitarray object: fo.write("Functions defined in `bitarray.util` module:\n" "--------------------------------------------\n\n" "This sub-module was added in version 1.2.\n\n") - for func in bitarray.util.__all__: + for func in sorted(bitarray.util.__all__): write_doc(fo, 'util.%s' % func) for name in list(NEW_IN) + list(DOC_LINKS): @@ -276,12 +296,8 @@ def main(): write_changelog(fo) testfile('./README.rst') - testfile('./doc/buffer.rst') - testfile('./doc/canonical.rst') - testfile('./doc/indexing.rst') - testfile('./doc/represent.rst') - testfile('./doc/sparse_compression.rst') - testfile('./doc/variable_length.rst') + for path in glob("./doc/*.rst"): + testfile(path) if __name__ == '__main__':