Make the tables that are registered configurable (#15)

* Make the tables that are registered configurable Make the list of tables configurable in the API. Some of the tables are quite expensive to generate such a instructions or symbols. Allow the users to configure which tables to generate when they create a sql_engine. * Add RUNPATH example to README * Add new line to README
fzakaria · Sep 26, 2023 · ce12395 · ce12395
1 parent 5a1264d
commit ce12395
Show file tree

Hide file tree

Showing 6 changed files with 224 additions and 70 deletions.
diff --git a/Makefile b/Makefile
@@ -28,6 +28,7 @@ lint:             ## Run pep8, black, mypy linters.
 .PHONY: test
 test:             ## Run pytest primarily.
 	pytest
+	pytest -m "slow"
 
 .PHONY: coverage
 coverage:

diff --git a/README.md b/README.md
@@ -302,6 +302,35 @@ WHERE elf_dynamic_entries.tag = 'NEEDED'"
 ```
 </details>
 
+<details>
+<summary>Determine the RPATH/RINPATH entries for a program</summary>
+
+_This may be improved in the future but for now there is a little knowledge of the
+polymorphic nature of the dynamic entries needed_.
+
+The below uses a file built with [NixOS](https://nixos.org) as they all have RUNPATH set.
+
+A recursive query can further be used to split the row into multiple rows. 
+
+```console
+❯ sqlelf /nix/store/gjr9ylm023rl9di484g1wxcd1jp84xxv-nix-2.8.1/bin/nix \
+ --sql "SELECT elf_strings.path, elf_strings.value
+FROM elf_dynamic_entries
+INNER JOIN elf_strings ON elf_dynamic_entries.value = elf_strings.offset
+WHERE elf_dynamic_entries.tag = 'RUNPATH';"
+┌─────────────────────────────────────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│                      path                       │                                                                                 value                                                                                 │
+├─────────────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
+│ /nix/store/gjr9ylm023rl9di484g1wxcd1jp84xxv-    │ /nix/store/gjr9ylm023rl9di484g1wxcd1jp84xxv-                                                                                                                          │
+│ nix-2.8.1/bin/nix                               │ nix-2.8.1/lib:/nix/store/pkxyfwarcq081rybpbnprjmnkiy1cz6g-libsodium-1.0.18/lib:/nix/store/r6mrf9pz4dpax6rcszcmbyrpsk8j6saz-                                           │
+│                                                 │ editline-1.17.1/lib:/nix/store/ppm63lvkyfa58sgcnr2ddzh14dy1k9fn-boehm-gc-8.0.6/lib:/nix/store/sgw2i15l01rwxzj62745h30bsjmh7wc1-lowdown-0.11.1-                        │
+│                                                 │ lib/lib:/nix/store/bvy2z17rzlvkx2sj7fy99ajm853yv898-glibc-2.34-210/lib:/nix/store/gka59hya7l7qp26s0rydcgq8hj0d7v7k-gcc-11.3.0-lib/lib                                 │
+└─────────────────────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+```
+
+</details>
+
+
 ## Development
 
 You may want to install the package in _editable mode_ as well to make development easier

diff --git a/pyproject.toml b/pyproject.toml
@@ -61,7 +61,8 @@ skip = [".git", "result"]
 profile = "black"
 
 [tool.pytest.ini_options]
-addopts = ""
+addopts = "-m 'not slow' --strict-markers"
+markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]
 
 [tool.pyright]
 exclude = ["**/__pycache__", "sqlelf/_version.py"]

diff --git a/sqlelf/elf.py b/sqlelf/elf.py
@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Any, Callable, Iterator, Sequence, cast
+from enum import Flag, auto
+from typing import Any, Callable, Iterator, Optional, Sequence, cast
 
 import apsw
 import apsw.ext
@@ -35,7 +36,40 @@ def make_generator(
         return Generator(columns, apsw.ext.VTColumnAccess.By_Name, generator)
 
 
-def make_dynamic_entries_generator(binaries: list[lief.Binary]) -> Generator:
+class GeneratorFlag(Flag):
+    NONE = 0
+    DYNAMIC_ENTRIES = auto()
+    HEADERS = auto()
+    INSTRUCTIONS = auto()
+    SECTIONS = auto()
+    SYMBOLS = auto()
+    STRINGS = auto()
+    VERSION_REQUIREMENTS = auto()
+    VERSION_DEFINITIONS = auto()
+
+    @classmethod
+    def ALL(cls: type[GeneratorFlag]) -> GeneratorFlag:
+        retval = cls.NONE
+        for member in cls.__members__.values():
+            retval |= member
+        return retval
+
+
+@dataclass
+class MakeGeneratorResponse:
+    """A response from a generator factory.
+
+    Contains everything needed to register the virtual table."""
+
+    generator: Generator
+    table_name: str
+    flag: GeneratorFlag
+    sql: Optional[str] = None
+
+
+def make_dynamic_entries_generator(
+    binaries: list[lief.Binary],
+) -> MakeGeneratorResponse:
     """Create the .dynamic section virtual table."""
 
     def dynamic_entries_generator() -> Iterator[dict[str, Any]]:
@@ -46,13 +80,17 @@ def dynamic_entries_generator() -> Iterator[dict[str, Any]]:
             for entry in binary.dynamic_entries:  # type: ignore
                 yield {"path": binary_name, "tag": entry.tag.name, "value": entry.value}
 
-    return Generator.make_generator(
-        ["path", "tag", "value"],
-        dynamic_entries_generator,
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            ["path", "tag", "value"],
+            dynamic_entries_generator,
+        ),
+        "elf_dynamic_entries",
+        GeneratorFlag.DYNAMIC_ENTRIES,
     )
 
 
-def make_headers_generator(binaries: list[lief.Binary]) -> Generator:
+def make_headers_generator(binaries: list[lief.Binary]) -> MakeGeneratorResponse:
     """Create the ELF headers virtual table,"""
 
     def headers_generator() -> Iterator[dict[str, Any]]:
@@ -65,13 +103,17 @@ def headers_generator() -> Iterator[dict[str, Any]]:
                 "entry": binary.header.entrypoint,
             }
 
-    return Generator.make_generator(
-        ["path", "type", "machine", "version", "entry"],
-        headers_generator,
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            ["path", "type", "machine", "version", "entry"],
+            headers_generator,
+        ),
+        "elf_headers",
+        GeneratorFlag.HEADERS,
     )
 
 
-def make_instructions_generator(binaries: list[lief.Binary]) -> Generator:
+def make_instructions_generator(binaries: list[lief.Binary]) -> MakeGeneratorResponse:
     """Create the instructions virtual table.
 
     This table includes dissasembled instructions from the executable sections"""
@@ -105,9 +147,15 @@ def instructions_generator() -> Iterator[dict[str, Any]]:
                             "operands": op_str,
                         }
 
-    return Generator.make_generator(
-        ["path", "section", "mnemonic", "address", "operands"],
-        instructions_generator,
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            ["path", "section", "mnemonic", "address", "operands"],
+            instructions_generator,
+        ),
+        "raw_elf_instructions",
+        GeneratorFlag.INSTRUCTIONS,
+        """CREATE TEMP TABLE elf_instructions
+        AS SELECT * FROM raw_elf_instructions;""",
     )
 
 
@@ -123,7 +171,7 @@ def arch(binary: lief.Binary) -> int:
     raise RuntimeError(f"Unknown machine type for {binary.name}")
 
 
-def make_sections_generator(binaries: list[lief.Binary]) -> Generator:
+def make_sections_generator(binaries: list[lief.Binary]) -> MakeGeneratorResponse:
     """Create the ELF sections virtual table."""
 
     def sections_generator() -> Iterator[dict[str, Any]]:
@@ -141,9 +189,13 @@ def sections_generator() -> Iterator[dict[str, Any]]:
                     "content": bytes(section.content),
                 }
 
-    return Generator.make_generator(
-        ["path", "name", "offset", "size", "type", "content"],
-        sections_generator,
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            ["path", "name", "offset", "size", "type", "content"],
+            sections_generator,
+        ),
+        "elf_sections",
+        GeneratorFlag.SECTIONS,
     )
 
 
@@ -154,7 +206,7 @@ def coerce_section_name(name: str | None) -> str | None:
     return name
 
 
-def make_strings_generator(binaries: list[lief.Binary]) -> Generator:
+def make_strings_generator(binaries: list[lief.Binary]) -> MakeGeneratorResponse:
     """Create the ELF strings virtual table.
 
     This goes through all string tables in the ELF binary and splits them on null bytes.
@@ -188,9 +240,13 @@ def strings_generator() -> Iterator[dict[str, Any]]:
                         "offset": offset + 1,
                     }
 
-    return Generator.make_generator(
-        ["path", "section", "value", "offset"],
-        strings_generator,
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            ["path", "section", "value", "offset"],
+            strings_generator,
+        ),
+        "elf_strings",
+        GeneratorFlag.STRINGS,
     )
 
 
@@ -207,7 +263,7 @@ def split_with_index(str: str, delimiter: str) -> list[tuple[int, str]]:
     return result
 
 
-def make_symbols_generator(binaries: list[lief.Binary]) -> Generator:
+def make_symbols_generator(binaries: list[lief.Binary]) -> MakeGeneratorResponse:
     """Create the ELF symbols virtual table."""
 
     def symbols_generator() -> Iterator[dict[str, Any]]:
@@ -254,24 +310,32 @@ def symbols_generator() -> Iterator[dict[str, Any]]:
                     "value": symbol.value,
                 }
 
-    return Generator.make_generator(
-        [
-            "path",
-            "name",
-            "demangled_name",
-            "imported",
-            "exported",
-            "section",
-            "size",
-            "version",
-            "type",
-            "value",
-        ],
-        symbols_generator,
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            [
+                "path",
+                "name",
+                "demangled_name",
+                "imported",
+                "exported",
+                "section",
+                "size",
+                "version",
+                "type",
+                "value",
+            ],
+            symbols_generator,
+        ),
+        "raw_elf_symbols",
+        GeneratorFlag.SYMBOLS,
+        """CREATE TEMP TABLE elf_symbols
+        AS SELECT * FROM raw_elf_symbols;
+        CREATE INDEX elf_symbols_path_idx ON elf_symbols (path);
+        CREATE INDEX elf_symbols_name_idx ON elf_symbols (name);""",
     )
 
 
-def make_version_requirements(binaries: list[lief.Binary]) -> Generator:
+def make_version_requirements(binaries: list[lief.Binary]) -> MakeGeneratorResponse:
     """Create the ELF version requirements virtual table.
 
     This should match the values found in .gnu.version_r section.
@@ -292,12 +356,17 @@ def version_requirements_generator() -> Iterator[dict[str, Any]]:
                         "name": aux_requirement.name,
                     }
 
-    return Generator.make_generator(
-        ["path", "file", "name"], version_requirements_generator
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            ["path", "file", "name"],
+            version_requirements_generator,
+        ),
+        "elf_version_requirements",
+        GeneratorFlag.VERSION_REQUIREMENTS,
     )
 
 
-def make_version_definitions(binaries: list[lief.Binary]) -> Generator:
+def make_version_definitions(binaries: list[lief.Binary]) -> MakeGeneratorResponse:
     """Create the ELF version requirements virtual table.
 
     This should match the values found in .gnu.version_d section.
@@ -318,8 +387,13 @@ def version_definitions_generator() -> Iterator[dict[str, Any]]:
                         "flags": flags,
                     }
 
-    return Generator.make_generator(
-        ["path", "name", "flags"], version_definitions_generator
+    return MakeGeneratorResponse(
+        Generator.make_generator(
+            ["path", "name", "flags"],
+            version_definitions_generator,
+        ),
+        "elf_version_definitions",
+        GeneratorFlag.VERSION_DEFINITIONS,
     )
 
 
@@ -343,30 +417,38 @@ def symbols(binary: lief.Binary) -> Sequence[lief.ELF.Symbol]:
 
 
 def register_virtual_tables(
-    connection: apsw.Connection, binaries: list[lief.Binary]
+    connection: apsw.Connection,
+    binaries: list[lief.Binary],
+    flags: GeneratorFlag = GeneratorFlag.ALL(),
 ) -> None:
-    """Register the virtual table modules."""
-    factory_and_names = [
-        (make_dynamic_entries_generator, "elf_dynamic_entries"),
-        (make_headers_generator, "elf_headers"),
-        (make_instructions_generator, "raw_elf_instructions"),
-        (make_sections_generator, "elf_sections"),
-        (make_strings_generator, "elf_strings"),
-        (make_symbols_generator, "raw_elf_symbols"),
-        (make_version_requirements, "elf_version_requirements"),
-        (make_version_definitions, "elf_version_definitions"),
+    """Register the virtual table modules.
+
+    You can make the SQL engine more speedy by only specifying the
+    Generators (virtual tables) that you care about via the flags argument.
+
+    Args:
+        connection: the connection to register the virtual tables on
+        binaries: the list of binaries to analyze
+        flags: the bitwise flags which controls which virtual table to enable"""
+    generator_factories = [
+        make_dynamic_entries_generator,
+        make_headers_generator,
+        make_instructions_generator,
+        make_sections_generator,
+        make_strings_generator,
+        make_symbols_generator,
+        make_version_requirements,
+        make_version_definitions,
     ]
-    for factory, name in factory_and_names:
-        generator = factory(binaries)
-        apsw.ext.make_virtual_module(connection, name, generator)
-    connection.execute(
-        """
-        CREATE TEMP TABLE elf_instructions
-        AS SELECT * FROM raw_elf_instructions;
-
-        CREATE TEMP TABLE elf_symbols
-        AS SELECT * FROM raw_elf_symbols;
-        CREATE INDEX elf_symbols_path_idx ON elf_symbols (path);
-        CREATE INDEX elf_symbols_name_idx ON elf_symbols (name);
-        """
-    )
+    for factory in generator_factories:
+        generator_response = factory(binaries)
+
+        if generator_response.flag not in flags:
+            continue
+
+        apsw.ext.make_virtual_module(
+            connection, generator_response.table_name, generator_response.generator
+        )
+
+        if generator_response.sql:
+            connection.execute(generator_response.sql)