From 00ef0d2ba6f99ad61ae3dc6219293fea0dc5d12f Mon Sep 17 00:00:00 2001 From: Riccardo Mori Date: Wed, 27 Sep 2023 15:49:58 +0200 Subject: [PATCH 1/7] Add venv to gitignore --- .gitignore | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a471030..63c08a9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,13 @@ site/ .pytest_cache/ dist/ docs/reference/python/ -doxygen_current.cfg \ No newline at end of file +doxygen_current.cfg + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ \ No newline at end of file From cb4addd3f1c64b7f73be3aab7a1ca24db66f5e1f Mon Sep 17 00:00:00 2001 From: Riccardo Mori Date: Wed, 27 Sep 2023 15:51:10 +0200 Subject: [PATCH 2/7] Add pycache to gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 63c08a9..3f8f52a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,11 @@ dist/ docs/reference/python/ doxygen_current.cfg +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + # Environments .env .venv From 7348e492a432b7bfacd03ec1c208c43fb15d86e7 Mon Sep 17 00:00:00 2001 From: Riccardo Mori Date: Wed, 27 Sep 2023 15:52:22 +0200 Subject: [PATCH 3/7] Add unit tests and coverage report to gitignore --- .gitignore | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitignore b/.gitignore index 3f8f52a..bb11179 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,18 @@ __pycache__/ *.py[cod] *$py.class +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + # Environments .env .venv From 4fc608756d50c5a6d7f59e161a68690d977a1c0b Mon Sep 17 00:00:00 2001 From: Riccardo Mori Date: Wed, 27 Sep 2023 15:54:45 +0200 Subject: [PATCH 4/7] Update gitignore --- .gitignore | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index bb11179..621537c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,7 @@ -build*/ third_party/idasdk*/ quokka.pb.h ci/ida*/ -.eggs/* site/ -.pytest_cache/ -dist/ docs/reference/python/ doxygen_current.cfg @@ -14,6 +10,25 @@ __pycache__/ *.py[cod] *$py.class +# Distribution / packaging +.Python +build*/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + # Unit test / coverage reports htmlcov/ .tox/ From 97188fc61a920451f4e1a4926b0b82c90f9be7bb Mon Sep 17 00:00:00 2001 From: Riccardo Mori Date: Wed, 27 Sep 2023 15:56:52 +0200 Subject: [PATCH 5/7] Decouple from pypcode as it might not be installed --- bindings/python/quokka/backends/__init__.py | 9 --------- bindings/python/quokka/block.py | 9 ++++++--- bindings/python/quokka/instruction.py | 10 +++++++--- bindings/python/quokka/program.py | 9 +++++++-- tests/python/tests/backends/test_pypcode.py | 20 ++++++++++---------- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/bindings/python/quokka/backends/__init__.py b/bindings/python/quokka/backends/__init__.py index 9103d48..dca7bc3 100644 --- a/bindings/python/quokka/backends/__init__.py +++ b/bindings/python/quokka/backends/__init__.py @@ -17,18 +17,9 @@ get_capstone_context, capstone_decode_instruction, ) -from quokka.backends.pypcode import ( - pypcode_decode_block, - pypcode_decode_instruction, - get_pypcode_context, -) __all__ = [ # From capstone.py "get_capstone_context", "capstone_decode_instruction", - # From pypcode.py - "get_pypcode_context", - "pypcode_decode_instruction", - "pypcode_decode_block", ] diff --git a/bindings/python/quokka/block.py b/bindings/python/quokka/block.py index 664fb63..bcb5e64 100644 --- a/bindings/python/quokka/block.py +++ b/bindings/python/quokka/block.py @@ -18,8 +18,7 @@ import logging import collections from functools import cached_property - -import pypcode +from typing import TYPE_CHECKING import quokka from quokka.types import ( @@ -35,6 +34,8 @@ Set, ) +if TYPE_CHECKING: + import pypcode logger: logging.Logger = logging.getLogger(__name__) @@ -257,4 +258,6 @@ def pcode_insts(self) -> List[pypcode.PcodeOp]: A list of PCode instructions """ - return quokka.backends.pypcode_decode_block(self) + from quokka.backend.pypcode import pypcode_decode_block + + return pypcode_decode_block(self) diff --git a/bindings/python/quokka/instruction.py b/bindings/python/quokka/instruction.py index 4c53d75..16a1a0b 100644 --- a/bindings/python/quokka/instruction.py +++ b/bindings/python/quokka/instruction.py @@ -20,7 +20,7 @@ from collections import defaultdict from functools import cached_property import capstone -import pypcode +from typing import TYPE_CHECKING import quokka from quokka.types import ( @@ -38,6 +38,9 @@ Union, ) +if TYPE_CHECKING: + import pypcode + logger: logging.Logger = logging.getLogger(__name__) @@ -188,7 +191,9 @@ def pcode_insts(self) -> Sequence[pypcode.PcodeOp]: Returns: A sequence of PCode instructions """ - return quokka.backends.pypcode_decode_instruction(self) + from quokka.backends.pypcode import pypcode_decode_instruction + + return pypcode_decode_instruction(self) @cached_property def string(self) -> Optional[str]: @@ -309,7 +314,6 @@ def constants(self) -> List[int]: for op_index in self.program.proto.instructions[self.proto_index].operand_index: operand: quokka.pb.Quokka.Operand = self.program.proto.operands[op_index] if operand.type == 5: - # FIX: This bug is due to IDA mislabelling operands for some # operations like ADRP on ARM where the operand points to a # memory area (2) but the type is CONSTANT (5). diff --git a/bindings/python/quokka/program.py b/bindings/python/quokka/program.py index 89f0415..1fb7049 100644 --- a/bindings/python/quokka/program.py +++ b/bindings/python/quokka/program.py @@ -26,9 +26,9 @@ import os import pathlib import subprocess +from typing import TYPE_CHECKING import capstone -import pypcode import networkx import quokka @@ -50,6 +50,9 @@ Union, ) +if TYPE_CHECKING: + import pypcode + class Program(dict): """Program @@ -209,7 +212,9 @@ def call_graph(self) -> networkx.DiGraph: @cached_property def pypcode(self) -> pypcode.Context: """Generate the Pypcode context.""" - return quokka.backends.get_pypcode_context(self.arch, self.endianness) + from quokka.backends.pypcode import get_pypcode_context + + return get_pypcode_context(self.arch, self.endianness) @cached_property def structures(self) -> List[quokka.Structure]: diff --git a/tests/python/tests/backends/test_pypcode.py b/tests/python/tests/backends/test_pypcode.py index 2a29832..c520005 100644 --- a/tests/python/tests/backends/test_pypcode.py +++ b/tests/python/tests/backends/test_pypcode.py @@ -2,25 +2,25 @@ import pytest import quokka -import quokka.backends +import quokka.backends.pypcode as pypcode_backend def test_pypcode_context(): - context = quokka.backends.get_pypcode_context(quokka.analysis.ArchX86) + context = pypcode_backend.get_pypcode_context(quokka.analysis.ArchX86) assert context.lang.id == "x86:LE:32:default" - context = quokka.backends.get_pypcode_context(quokka.analysis.ArchX64) + context = pypcode_backend.get_pypcode_context(quokka.analysis.ArchX64) assert context.lang.id == "x86:LE:64:default" - context = quokka.backends.get_pypcode_context(quokka.analysis.ArchARM64) + context = pypcode_backend.get_pypcode_context(quokka.analysis.ArchARM64) assert context.lang.id == "AARCH64:LE:64:v8A" - context = quokka.backends.get_pypcode_context(quokka.analysis.ArchARM) + context = pypcode_backend.get_pypcode_context(quokka.analysis.ArchARM) assert context.lang.id == "ARM:LE:32:v8" with pytest.raises(quokka.PypcodeError): - quokka.backends.get_pypcode_context(quokka.analysis.QuokkaArch) + pypcode_backend.get_pypcode_context(quokka.analysis.QuokkaArch) def test_pypcode_decode_instruction(mocker): @@ -28,11 +28,11 @@ def test_pypcode_decode_instruction(mocker): inst = mocker.MagicMock() inst.bytes = b"\x55" # push rbp inst.address = 0x1000 - inst.program.pypcode = quokka.backends.get_pypcode_context( + inst.program.pypcode = pypcode_backend.get_pypcode_context( quokka.analysis.ArchX64 ) - inst_decoded = quokka.backends.pypcode_decode_instruction(inst) + inst_decoded = pypcode_backend.pypcode_decode_instruction(inst) assert len(inst_decoded) == 3 assert isinstance(inst_decoded[0], pypcode.PcodeOp) @@ -49,9 +49,9 @@ def test_pypcode_decode_block(mocker): instructions = [mocker.MagicMock(size=len(inst_bytes)) for inst_bytes in block_bytes] block.instructions = instructions - block.program.pypcode = quokka.backends.get_pypcode_context( + block.program.pypcode = pypcode_backend.get_pypcode_context( quokka.analysis.ArchX64 ) - decoded_block = quokka.backends.pypcode_decode_block(block) + decoded_block = pypcode_backend.pypcode_decode_block(block) assert len(decoded_block) > 0 From fe701ba20e908c298f79c8f74d754a4cf9e19c5d Mon Sep 17 00:00:00 2001 From: Riccardo Mori Date: Wed, 27 Sep 2023 15:58:39 +0200 Subject: [PATCH 6/7] Switch to pyproject.toml and relax dependencies pypcode is now an optional dependency and the upper bound requirements for other packages have been remove --- pyproject.toml | 54 +++++++++++++++++++++++++++++++++++++++++++++ setup.py | 60 ++------------------------------------------------ 2 files changed, 56 insertions(+), 58 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d03dd2a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["setuptools", "protobuf_distutils"] +build-backend = "setuptools.build_meta" + +[project] +name = "quokka-project" +description = "Quokka: A Fast and Accurate Binary Exporter" +authors = [{ name = "Quarkslab", email = "diffing@quarkslab.com" }] +license = { text = "Apache Software License (Apache License, Version 2)" } +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.8" +dependencies = ["capstone>=4.0.2", "networkx>=2.4", "protobuf>=3.12.2"] +dynamic = ["version"] + +[project.urls] +Homepage = "https://github.com/quarkslab/quokka/" +Repository = "https://github.com/quarkslab/quokka/" +Documentation = "https://quarkslab.github.io/quokka/" +"Bug Tracker" = "https://github.com/quarkslab/quokka/issues" + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-mock", + "pytest-cov", + "coverage[toml]", + "pypcode>=1.1.1", +] +pypcode = ["pypcode>=1.1.1"] +doc = [ + "mkdocs", + "mkdocs-material", + "mkdocstrings", + "mkdocstrings-python", + "mkdocs-literate-nav", + "mkdocs-git-revision-date-localized-plugin", + "mkdocs-gen-files", + "mkdocs-simple-hooks", +] +dev = [ + "black", + "ipython", + "flake8", + "flake8-black", + "mypy", + "mypy-protobuf", + "nox", + "pypcode>=1.1.1", +] + +[tool.setuptools] +packages = ["quokka", "quokka.analysis", "quokka.backends"] +package-dir = { "" = "bindings/python/" } +package-data = { "quokka" = ["*.pyi", "*.typed"] } diff --git a/setup.py b/setup.py index b6d5b51..6b01dbc 100644 --- a/setup.py +++ b/setup.py @@ -17,73 +17,17 @@ from setuptools import setup from os.path import normpath -with open("README.md", "r") as fd: - readme = fd.read() - main_ns = {} -ver_path = normpath('bindings/python/quokka/version.py') +ver_path = normpath("bindings/python/quokka/version.py") with open(ver_path) as ver_file: exec(ver_file.read(), main_ns) setup( - name="quokka-project", - version=main_ns['__version__'], - author="Alexis Challande", - author_email="achallande@quarkslab.com", - url="https://github.com/quarkslab/quokka", - project_urls={ - "Documentation": "https://quarkslab.github.io/quokka/", - "Bug Tracker": "https://github.com/quarkslab/quokka/issues", - "Source": "https://github.com/quarkslab/quokka/", - }, - description="Quokka : A Fast and Accurate Binary Exporter", - long_description=readme, - long_description_content_type="text/markdown", - python_requires=">=3.8", - packages=["quokka", "quokka.analysis", "quokka.backends"], - package_dir={"": "bindings/python/"}, - package_data={"quokka": ["*.pyi", "*.typed"]}, - setup_requires=[ - "protobuf_distutils", - ], - license="Apache-2", + version=main_ns["__version__"], options={ "generate_py_protobufs": { "source_dir": "proto", "output_dir": "bindings/python/quokka", }, }, - install_requires=[ - "capstone>=4.0.2,<5", - "networkx>=2.4,<3", - "protobuf>=3.12.2,<4", - "pypcode>=1.1.1,<2", - ], - extras_require={ - "test": [ - "pytest", - "pytest-mock", - "pytest-cov", - "coverage[toml]", - ], - "doc": [ - "mkdocs", - "mkdocs-material", - "mkdocstrings", - "mkdocstrings-python", - "mkdocs-literate-nav", - "mkdocs-git-revision-date-localized-plugin", - "mkdocs-gen-files", - "mkdocs-simple-hooks", - ], - "dev": [ - "black", - "ipython", - "flake8", - "flake8-black", - "mypy", - "mypy-protobuf", - "nox", - ], - }, ) From e65dfd3ff54971152d4fc7c5a16d2082e5e987bb Mon Sep 17 00:00:00 2001 From: Riccardo Mori Date: Wed, 27 Sep 2023 16:02:53 +0200 Subject: [PATCH 7/7] Update gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 621537c..f0c0faf 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,4 @@ env/ venv/ ENV/ env.bak/ -venv.bak/ \ No newline at end of file +venv.bak/