merge will's fixes

lib/cuckoo/common/objects.py

@@ -11,6 +11,7 @@
 import os
 import struct
 import subprocess
+import time
 from pathlib import Path
 from typing import Any, Dict
 
@@ -174,6 +175,15 @@ class File:
     yara_rules = {}
     yara_rules_hash = None
     yara_initialized = False
+    # category -> time.monotonic() of the last failed forced recompile.
+    # get_yara() forces a recompile when a category is missing; without this it
+    # would force a full six-category recompile for every scanned file (~3s each
+    # on a production ruleset). It is a BACKOFF, not a permanent skip: workers
+    # run with max_tasks=0 (no recycling), so permanently disabling a category
+    # after one transient failure -- rules mid-update, storage briefly
+    # unreadable -- would silently return no matches for the rest of the run.
+    yara_uncompilable = {}
+    yara_recompile_backoff = 300
     # static fields which indicate whether the user has been
     # notified about missing dependencies already
     notified_yara = False
@@ -525,6 +535,14 @@ def init_yara(cls, raise_exception: bool = False, force: bool = False):
                     else:
                         # This runs if the inner for loop finishes WITHOUT break (no errors)
                         compiled_rules = compiler.build()
+                        # Cache the compiled Rules, NOT a Scanner. yara_x.Scanner is
+                        # unsendable: PyO3 panics with "Scanner is unsendable, but sent
+                        # to another thread" if one is touched off its constructing
+                        # thread, and writes an unraisable RuntimeError if one is
+                        # DROPPED off it (which happens on fork, and at shutdown for
+                        # daemon threads -- so caching a Scanner per-thread does not
+                        # help either). Rules is sendable; get_yara() builds the
+                        # Scanner on the thread that scans.
                         cls.yara_rules[category] = compiled_rules
                         if category == "memory":
                             index_memory = os.path.join(yara_root, "index_memory.yarc")
@@ -579,6 +597,18 @@ def init_yara(cls, raise_exception: bool = False, force: bool = False):
                     log.debug("\t |-- %s %s", category, entry)
         cls.yara_rules_hash = hasher.hexdigest()
         cls.yara_initialized = True
+        # Any category that compiled this time is healthy again, so drop its
+        # backoff. Dropping only what actually compiled (rather than clearing the
+        # whole record) matters when SEVERAL categories are broken: clearing would
+        # make each one forget the others and force a fresh full recompile on
+        # every alternating call.
+        # Snapshot the keys first: yara_rules is class-level and two threads can
+        # be inside a forced init at once (get_yara's fallback triggers one), so
+        # iterating it directly can raise "dictionary changed size during
+        # iteration". Snapshotting also keeps categories that were injected
+        # outside the built-in list, which iterating `categories` would miss.
+        for compiled_category in tuple(cls.yara_rules):
+            cls.yara_uncompilable.pop(compiled_category, None)
 
     def get_yara(self, category="binaries", externals=None):
         """Get Yara signatures matches.
@@ -614,12 +644,27 @@ def get_yara(self, category="binaries", externals=None):
                 # short-circuits and leaves this category missing. Force ONE full recompile
                 # before giving up so a live category is never silently skipped (returning []
                 # here would look like "no matches" and hide the misconfiguration).
+                last_failure = File.yara_uncompilable.get(category)
+                if last_failure is not None and time.monotonic() - last_failure < File.yara_recompile_backoff:
+                    # Recently forced a recompile for this category and it still
+                    # produced nothing. Back off rather than paying another full
+                    # recompile for every remaining file in the task.
+                    return []
                 File.init_yara(force=True)
                 rules = self.yara_rules.get(category)
                 if not rules:
+                    File.yara_uncompilable[category] = time.monotonic()
+                    log.warning(
+                        "Yara category '%s' produced no rules after a forced recompile; backing off for %ss "
+                        "before retrying it.",
+                        category,
+                        File.yara_recompile_backoff,
+                    )
                     return []
 
             if HAVE_YARA_X:
+                # Built here, on the scanning thread, and deliberately not cached
+                # anywhere that outlives the scan -- see init_yara().
                 yara_results = yara_x.Scanner(rules).scan_file(self.file_path)
                 for match in yara_results.matching_rules:
                     strings = []

tests/test_pebble_engine.py

@@ -95,7 +95,7 @@ def test_reaper_fails_a_task_that_was_never_picked_up():
         stall_grace=1
     )
     future = _MockFuture()
-    
+
     with engine._lock:
         engine._pending[future] = 77
         engine._scheduled_at[future] = time.monotonic() - 3600  # long overdue
@@ -120,7 +120,7 @@ def test_reaper_leaves_a_task_that_is_still_within_its_deadline():
         stall_grace=5
     )
     future = _MockFuture()
-    
+
     with engine._lock:
         engine._pending[future] = 88
         engine._scheduled_at[future] = time.monotonic()  # brand new
@@ -145,7 +145,7 @@ def test_done_handles_base_exception_robustly():
         stall_grace=5
     )
     future = _BaseExceptionRaisingFuture()
-    
+
     with engine._lock:
         engine._pending[future] = 99
         engine._scheduled_at[future] = time.monotonic()

tests/test_yara_x_thread_affinity.py

@@ -0,0 +1,229 @@
+"""yara_x.Scanner is unsendable: PyO3 panics if one is touched from a thread
+other than the one that built it --
+
+    pyo3_runtime.PanicException: assertion `left == right` failed:
+    yara_x::Scanner is unsendable, but sent to another thread
+
+init_yara() may run on any thread (modules/processing/memory.py calls it at
+import time), while get_yara() is reached from others -- notably the threaded
+web app via lib/cuckoo/core/data/tasking.py. Caching a Scanner on the class is
+therefore fatal; the cache must hold the compiled Rules and each thread must
+build its own Scanner.
+"""
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+
+try:
+    import yara_x
+
+    HAVE_YARA_X = True
+except ImportError:
+    HAVE_YARA_X = False
+
+
+pytestmark = pytest.mark.skipif(not HAVE_YARA_X, reason="yara-x not installed")
+
+RULE = 'rule marla { strings: $a = "needle" condition: $a }'
+
+
+@pytest.fixture
+def sample(tmp_path):
+    path = tmp_path / "sample.bin"
+    path.write_bytes(b"....needle....")
+    return str(path)
+
+
+def test_a_scanner_cached_across_threads_would_panic(sample):
+    """Pin the underlying hazard: this is the crash the fix exists to prevent.
+
+    If this ever stops raising, yara-x has changed its threading contract and
+    the rest of this module can be simplified.
+    """
+    scanner = yara_x.Scanner(yara_x.compile(RULE))  # built on THIS thread
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        with pytest.raises(BaseException) as excinfo:
+            pool.submit(scanner.scan_file, sample).result()
+
+    assert type(excinfo.value).__module__ == "pyo3_runtime", excinfo.value
+    assert "unsendable" in str(excinfo.value)
+
+
+def test_init_yara_does_not_cache_a_scanner(monkeypatch, tmp_path):
+    """Exercise the real init_yara() and inspect what it actually cached.
+
+    The previous version of this test asserted against an object it had itself
+    installed, so it passed even against unfixed code.
+    """
+    from lib.cuckoo.common import objects as objects_mod
+    from lib.cuckoo.common.objects import File
+
+    for category in ("binaries", "urls", "memory", "CAPE", "macro", "monitor"):
+        for tree in ("data", "custom"):
+            (tmp_path / tree / "yara" / category).mkdir(parents=True, exist_ok=True)
+    (tmp_path / "data" / "yara" / "binaries" / "marla.yar").write_text(RULE)
+    monkeypatch.setattr(objects_mod, "CUCKOO_ROOT", str(tmp_path))
+    monkeypatch.setattr(File, "yara_rules", {})
+    monkeypatch.setattr(File, "yara_initialized", False)
+
+    File.init_yara(force=True)
+
+    cached = File.yara_rules.get("binaries")
+    assert cached is not None, "init_yara compiled nothing -- test setup is wrong"
+    assert not isinstance(cached, yara_x.Scanner), (
+        "init_yara cached a Scanner; it is unsendable and will panic as soon as "
+        "any other thread scans with it"
+    )
+
+
+def test_get_yara_scans_correctly_from_many_threads(monkeypatch, sample):
+    """The end-to-end guarantee: concurrent get_yara() from N threads, no panic."""
+    from lib.cuckoo.common.objects import File
+
+    monkeypatch.setattr(File, "yara_rules", {"binaries": yara_x.compile(RULE)})
+    monkeypatch.setattr(File, "yara_initialized", True)
+
+    scanner_file = File(sample)
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        results = [f.result() for f in [pool.submit(scanner_file.get_yara, "binaries") for _ in range(64)]]
+
+    assert all([m["name"] for m in r] == ["marla"] for r in results), results
+
+
+def test_missing_category_is_not_recompiled_for_every_file(monkeypatch, sample):
+    """A permanently missing category must cost ONE forced recompile, not one per file.
+
+    Before: get_yara() called init_yara(force=True) on every call for a category
+    that could not be compiled -- a full six-category recompile per scanned file
+    (~3s each on a production ruleset).
+    """
+    from lib.cuckoo.common.objects import File
+
+    calls = []
+    monkeypatch.setattr(File, "yara_rules", {})
+    monkeypatch.setattr(File, "yara_initialized", True)
+    monkeypatch.setattr(File, "yara_uncompilable", {})
+    monkeypatch.setattr(File, "init_yara", classmethod(lambda cls, **kw: calls.append(kw)))
+
+    scanner_file = File(sample)
+    for _ in range(10):
+        assert scanner_file.get_yara("bogus") == []
+
+    assert len(calls) == 1, f"forced a recompile {len(calls)} times for one missing category"
+    assert "bogus" in File.yara_uncompilable
+
+
+def test_forced_reinit_clears_the_uncompilable_record(monkeypatch, tmp_path):
+    """Fixing the rules and reloading must let a failed category be retried."""
+    from lib.cuckoo.common import objects as objects_mod
+    from lib.cuckoo.common.objects import File
+
+    for category in ("binaries", "urls", "memory", "CAPE", "macro", "monitor"):
+        for tree in ("data", "custom"):
+            (tmp_path / tree / "yara" / category).mkdir(parents=True, exist_ok=True)
+    (tmp_path / "data" / "yara" / "binaries" / "marla.yar").write_text(RULE)
+    monkeypatch.setattr(objects_mod, "CUCKOO_ROOT", str(tmp_path))
+    monkeypatch.setattr(File, "yara_rules", {})
+    monkeypatch.setattr(File, "yara_initialized", False)
+    monkeypatch.setattr(File, "yara_uncompilable", {"binaries": 0.0})
+
+    File.init_yara(force=True)
+
+    assert "binaries" not in File.yara_uncompilable, "forced re-init must retry failed categories"
+    assert File.yara_rules.get("binaries") is not None
+
+
+def test_several_broken_categories_do_not_thrash_recompiles(monkeypatch, sample):
+    """Each broken category costs ONE forced recompile -- not one per alternating call.
+
+    Clearing the whole uncompilable set on every forced re-init made each broken
+    category forget the others, so alternating calls recompiled forever.
+    """
+    from lib.cuckoo.common.objects import File
+
+    calls = []
+    monkeypatch.setattr(File, "yara_rules", {})
+    monkeypatch.setattr(File, "yara_initialized", True)
+    monkeypatch.setattr(File, "yara_uncompilable", {})
+    monkeypatch.setattr(File, "init_yara", classmethod(lambda cls, **kw: calls.append(kw)))
+
+    scanner_file = File(sample)
+    for _ in range(10):
+        scanner_file.get_yara("broken_a")
+        scanner_file.get_yara("broken_b")
+
+    assert len(calls) == 2, f"{len(calls)} recompiles for 2 broken categories over 20 calls"
+
+
+def test_cached_scanner_follows_a_forced_recompile(monkeypatch, tmp_path):
+    """After a reload the next scan must use the NEW rules, never the cached old ones."""
+    from lib.cuckoo.common.objects import File
+
+    sample = tmp_path / "s.bin"
+    sample.write_bytes(b"....needle....")
+
+    monkeypatch.setattr(File, "yara_rules", {"binaries": yara_x.compile(RULE)})
+    monkeypatch.setattr(File, "yara_initialized", True)
+    scanner_file = File(str(sample))
+    assert [m["name"] for m in scanner_file.get_yara("binaries")] == ["marla"]
+
+    # Rules reloaded out from under the cached scanner: same text, different rule name.
+    File.yara_rules["binaries"] = yara_x.compile('rule reloaded { strings: $a = "needle" condition: $a }')
+    assert [m["name"] for m in scanner_file.get_yara("binaries")] == ["reloaded"], "scan used the stale cached ruleset"
+
+
+def test_forced_reinit_forgets_only_categories_that_actually_compiled(monkeypatch, tmp_path):
+    """A forced re-init must not wipe failures for categories that are STILL broken.
+
+    Clearing the whole set made each broken category forget the others, so with
+    two broken categories every alternating get_yara() forced a fresh full
+    recompile. Only categories that actually produced rules may be forgotten.
+    """
+    from lib.cuckoo.common import objects as objects_mod
+    from lib.cuckoo.common.objects import File
+
+    for category in ("binaries", "urls", "memory", "CAPE", "macro", "monitor"):
+        for tree in ("data", "custom"):
+            (tmp_path / tree / "yara" / category).mkdir(parents=True, exist_ok=True)
+    (tmp_path / "data" / "yara" / "binaries" / "marla.yar").write_text(RULE)
+    monkeypatch.setattr(objects_mod, "CUCKOO_ROOT", str(tmp_path))
+    monkeypatch.setattr(File, "yara_rules", {})
+    monkeypatch.setattr(File, "yara_initialized", False)
+    # "phantom" is not one of the categories init_yara compiles, so it stands in
+    # for a category that still does not produce rules after a forced recompile.
+    monkeypatch.setattr(File, "yara_uncompilable", {"binaries": 0.0, "phantom": 0.0})
+
+    File.init_yara(force=True)
+
+    assert "binaries" not in File.yara_uncompilable, "compiled category must be retried"
+    assert "phantom" in File.yara_uncompilable, (
+        "a category that STILL does not compile was forgotten; alternating "
+        "get_yara() calls will now force a full recompile every time"
+    )
+
+
+def test_a_transient_compile_failure_is_retried_after_the_backoff(monkeypatch, sample):
+    """A backoff, not a permanent skip.
+
+    Workers run with max_tasks=0 (no recycling), so marking a category dead after
+    one transient failure -- rules mid-update, storage briefly unreadable -- would
+    silently return no matches for the rest of the run.
+    """
+    from lib.cuckoo.common.objects import File
+
+    calls = []
+    monkeypatch.setattr(File, "yara_rules", {})
+    monkeypatch.setattr(File, "yara_initialized", True)
+    monkeypatch.setattr(File, "yara_uncompilable", {})
+    monkeypatch.setattr(File, "yara_recompile_backoff", 300)
+    monkeypatch.setattr(File, "init_yara", classmethod(lambda cls, **kw: calls.append(kw)))
+
+    scanner_file = File(sample)
+    scanner_file.get_yara("flaky")
+    scanner_file.get_yara("flaky")
+    assert len(calls) == 1, "backoff did not suppress the second recompile"
+
+    # once the backoff expires the category must be tried again
+    File.yara_uncompilable["flaky"] -= 301
+    scanner_file.get_yara("flaky")
+    assert len(calls) == 2, "category was permanently skipped instead of retried"