Add support for --recursive (#7)

Add support for loading recursively all shared libraries by a library.
This is useful if you want to inspect symbol resolution for instance.

* add pytest
* added some simple unit tests as scaffolding
* updated the README

README.md

@@ -95,7 +95,10 @@ positional arguments:
   FILE        The ELF file to analyze
 
 options:
-  -h, --help  show this help message and exit
+  -h, --help            show this help message and exit
+  -s SQL, --sql SQL     Potential SQL to execute. Omitting this enters the REPL.
+  --recursive, --no-recursive
+                        Load all shared libraries needed by each file using ldd
 ```
 
 Note: You may provide directories for `FILE`. Avoid giving too many binaries though since they must all be parsed at startup.
@@ -139,6 +142,53 @@ path|num_sections
 /usr/bin/ruby|28
 ```
 
+### Queries
+
+#### List all symbol resolutions
+
+```console
+❯ sqlelf /usr/bin/ruby --sql "SELECT caller.path as 'caller.path',
+       callee.path as 'calee.path',
+       caller.name,
+       caller.demangled_name
+FROM ELF_SYMBOLS caller
+INNER JOIN ELF_SYMBOLS callee
+ON
+caller.name = callee.name AND
+caller.path != callee.path AND
+caller.imported = TRUE AND
+callee.exported = TRUE
+LIMIT 25;"
+┌──────────────────────────────────────────┬──────────────────────────────────────────┬──────────────────────┬──────────────────────┐
+│               caller.path                │                calee.path                │         name         │    demangled_name    │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ ruby_run_node        │ ruby_run_node        │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ ruby_init            │ ruby_init            │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ ruby_options         │ ruby_options         │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ ruby_sysinit         │ ruby_sysinit         │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libc.so.6          │ __stack_chk_fail     │ __stack_chk_fail     │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ ruby_init_stack      │ ruby_init_stack      │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libc.so.6          │ setlocale            │ setlocale            │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libc.so.6          │ __libc_start_main    │ __libc_start_main    │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libc.so.6          │ __libc_start_main    │ __libc_start_main    │
+│ /usr/bin/ruby                            │ /lib/x86_64-linux-gnu/libc.so.6          │ __cxa_finalize       │ __cxa_finalize       │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ initgroups           │ initgroups           │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libm.so.6          │ log10                │ log10                │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ chmod                │ chmod                │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libgmp.so.10       │ __gmpz_mul           │ __gmpz_mul           │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libm.so.6          │ lgamma_r             │ lgamma_r             │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ symlink              │ symlink              │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ mprotect             │ mprotect             │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ pipe2                │ pipe2                │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ seteuid              │ seteuid              │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ chdir                │ chdir                │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ fileno               │ fileno               │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ dup2                 │ dup2                 │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ pthread_cond_destroy │ pthread_cond_destroy │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libc.so.6          │ pthread_cond_destroy │ pthread_cond_destroy │
+│ /lib/x86_64-linux-gnu/libruby-3.1.so.3.1 │ /lib/x86_64-linux-gnu/libm.so.6          │ atan2                │ atan2                │
+└──────────────────────────────────────────┴──────────────────────────────────────────┴──────────────────────┴──────────────────────┘
+```
+
 ## Development
 
 You must have [Nix](https://nixos.org) installed for development.

overlay.nix

@@ -17,6 +17,9 @@ self: super: {
   });
 
   poetryOverrides = self: super: {
+    sh = super.sh.overridePythonAttrs (old: {
+      buildInputs = (old.buildInputs or [ ]) ++ [ super.poetry ];
+    });
     apsw = super.apsw.overridePythonAttrs (old: rec {
       version = "3.43.0.0";
       src = super.pkgs.fetchFromGitHub {

pyproject.toml

@@ -7,7 +7,7 @@ description = "Explore ELF objects through the power of SQL"
 license = "LICENSE"
 
 [tool.poetry.dependencies]
-python = ">=3.10"
+python = ">=3.10,<4.0"
 capstone = "^5.0.1"
 lief = "^0.13.2"
 apsw = "^3.43.0.0"
@@ -16,6 +16,7 @@ apsw = "^3.43.0.0"
 # ERROR: Could not find a version that satisfies the requirement setuptools<61.0.0,>=60.0.0 (from sqlelf) (from versions: none)
 # ERROR: No matching distribution found for setuptools<61.0.0,>=60.0.0
 setuptools = "*"
+sh = "^2.0.6"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.7.0"

sqlelf/cli.py

@@ -1,5 +1,6 @@
 import argparse
 import os
+import os.path
 import sys
 from functools import reduce
 
@@ -8,6 +9,8 @@
 import apsw.shell
 import lief
 
+from sqlelf import ldd
+
 from .elf import dynamic, header, instruction, section, strings, symbol
 
 
@@ -30,6 +33,11 @@ def start(args=sys.argv[1:], stdin=sys.stdin):
     parser.add_argument(
         "-s", "--sql", help="Potential SQL to execute. Omitting this enters the REPL."
     )
+    parser.add_argument(
+        "--recursive",
+        action=argparse.BooleanOptionalAction,
+        help="Load all shared libraries needed by each file using ldd",
+    )
 
     args = parser.parse_args(args)
 
@@ -44,10 +52,22 @@ def start(args=sys.argv[1:], stdin=sys.stdin):
         ),
     )
     # Filter the list of filenames to those that are ELF files only
-    filenames = list(filter(lambda f: lief.is_elf(f), filenames))
+    filenames = list(filter(lambda f: os.path.isfile(f) and lief.is_elf(f), filenames))
+
+    # If none of the inputs are valid files, simply return
+    if len(filenames) == 0:
+        return
 
     binaries: list[lief.Binary] = [lief.parse(filename) for filename in filenames]
 
+    # If the recursive option is specidied, load the shared libraries
+    # the binary would load as well.
+    if args.recursive:
+        shared_libraries = [ldd.libraries(binary).values() for binary in binaries]
+        binaries = binaries + [
+            lief.parse(library) for sub_list in shared_libraries for library in sub_list
+        ]
+
     # forward sqlite logs to logging module
     apsw.bestpractice.apply(apsw.bestpractice.recommended)
 

sqlelf/elf/instruction.py

@@ -7,8 +7,7 @@
 import apsw.ext
 
 # TODO(fzkakaria): https://github.com/capstone-engine/capstone/issues/1993
-# pyright: reportMissingTypeStubs=false
-import capstone
+import capstone  # pyright: ignore
 import lief
 
 

sqlelf/ldd.py

@@ -0,0 +1,23 @@
+import re
+from collections import OrderedDict
+from typing import Dict
+
+import lief
+from sh import Command  # pyright: ignore
+
+
+def libraries(binary: lief.Binary) -> Dict[str, str]:
+    """Use the interpreter in a binary to determine the path of each linked library"""
+    interpreter = Command(binary.interpreter)  # pyright: ignore
+    resolution = interpreter("--list", binary.name)
+    result = OrderedDict()
+    # TODO: Figure out why `--list` and `ldd` produce different outcomes
+    # specifically for the interpreter.
+    # https://gist.github.com/fzakaria/3dc42a039401598d8e0fdbc57f5e7eae
+    for line in resolution.splitlines():  # pyright: ignore
+        m = re.match(r"\s*([^ ]+) => ([^ ]+)", line)
+        if not m:
+            continue
+        soname, lib = m.group(1), m.group(2)
+        result[soname] = lib
+    return result

tests/test_cli.py

@@ -14,6 +14,9 @@ def test_cli_single_file_arguments():
     stdin = StringIO("")
     cli.start(["/bin/ls"], stdin)
 
+def test_cli_single_non_existent_file_arguments():
+    cli.start(["does_not_exist"])
+
 def test_cli_prompt_single_file_arguments():
     stdin = StringIO(".exit 56\n")
     with pytest.raises(SystemExit) as err:

tests/test_ldd.py

@@ -0,0 +1,8 @@
+from sqlelf import ldd
+import lief
+
+def test_simple_binary():
+    binary = lief.parse("/bin/ls")
+    result = ldd.libraries(binary)
+    print(result)
+    assert len(result) > 0