From 5611959693437be63be56c5ff4981d1cc890c227 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Wed, 11 Nov 2026 19:44:07 -0800 Subject: [PATCH] Update [ghstack-poisoned] --- .ci/scripts/test-native.sh | 2 +- .github/workflows/test-native.yml | 1 + backends/native/test/CMakeLists.txt | 6 ++ backends/native/test/runtime/ggml/BUCK | 6 +- backends/native/test/samples/BUCK | 1 + backends/native/test/samples/cases.py | 41 ++++++--- backends/native/test/samples/embedding.py | 93 +++++++++++++++++++++ backends/native/test/samples/linear.py | 14 +--- backends/native/test/samples/ops.py | 4 +- backends/native/test/samples/record.py | 18 ++-- backends/native/test/samples/schemes.py | 9 +- backends/native/test/samples/test_record.py | 39 ++++++++- 12 files changed, 197 insertions(+), 37 deletions(-) create mode 100644 backends/native/test/samples/embedding.py diff --git a/.ci/scripts/test-native.sh b/.ci/scripts/test-native.sh index e1a0479b210..bf7ed2fe192 100644 --- a/.ci/scripts/test-native.sh +++ b/.ci/scripts/test-native.sh @@ -43,7 +43,7 @@ case "${component}" in ) # TODO: run the other sample tests once they pass. targets+=(run_hf_llm_ggml run_whisper_ggml_cpu native_ggml_engine_test native_llm_runner_utils_test run_classification_ggml_cpu native_output_check_test native_ggml_sample_test) - test_regex='^native_(ggml_engine|llm_runner_utils|output_check)_test$|^native_ggml_sample_test_act_quant$' + test_regex='^native_(ggml_engine|llm_runner_utils|output_check)_test$|^native_ggml_sample_test_(act_quant|no_linear)$' ;; vulkan) cmake_args+=( diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index 074c47a1c48..2917203b215 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -18,6 +18,7 @@ on: - backends/cpu/runtime/** - backends/native/** - backends/cadence/utils/FACTO + - backends/test/suite/operators/test_embedding.py - backends/test/suite/operators/test_linear.py - backends/vulkan/CMakeLists.txt - backends/vulkan/cmake/** diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 0874426cc68..62104b4b00d 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -62,6 +62,7 @@ add_custom_command( executorch.backends.native.test.samples.generate --out ${_native_op_samples} COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp DEPENDS flatc ${_native_op_samples_srcs} + ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_embedding.py ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py VERBATIM ) @@ -159,10 +160,15 @@ if(EXECUTORCH_BUILD_NATIVE_GGML) set_tests_properties(native_ggml_sample_test PROPERTIES DISABLED TRUE) # native_ggml_sample_test_act_quant lets ggml quantize the activations of # weight-only quantized linears, judged against golden-int8_act. + # native_ggml_sample_test_no_linear judges the other ops without it. add_test(NAME native_ggml_sample_test_act_quant COMMAND native_ggml_sample_test --packages=${_native_op_samples} --device=cpu --allow_activation_quant_for_weight_only ) + add_test(NAME native_ggml_sample_test_no_linear + COMMAND native_ggml_sample_test --packages=${_native_op_samples} + --device=cpu --gtest_filter=-linear_* + ) target_compile_options( native_ggml_sample_test PRIVATE $<$:/EHsc> diff --git a/backends/native/test/runtime/ggml/BUCK b/backends/native/test/runtime/ggml/BUCK index 94105c0a11f..ba81efda848 100644 --- a/backends/native/test/runtime/ggml/BUCK +++ b/backends/native/test/runtime/ggml/BUCK @@ -97,7 +97,8 @@ _VARIANTS = [ ] # ggml_sample_test_act_quant lets ggml quantize the activations of weight-only -# quantized linears, judged against golden-int8_act. +# quantized linears, judged against golden-int8_act. ggml_sample_test_no_linear +# judges the other ops without it. [ fbcode_target( _kind = runtime.cxx_test, @@ -109,7 +110,7 @@ _VARIANTS = [ env = v.get("env", {}) | v.get("sample_env", {}), remote_execution = v.get("remote_execution"), # TODO: run the others in CI once they pass. - labels = [] if v["suffix"] + name == "_act_quant" else ci.labels(ci.map(ci.skip_test())), + labels = [] if v["suffix"] + name in ("_act_quant", "_no_linear") else ci.labels(ci.map(ci.skip_test())), test_main = "//executorch/backends/native/test/samples:sample_test_main", deps = [":ggml_engine_host" + v["suffix"]], **v.get("compat", {}) @@ -118,6 +119,7 @@ _VARIANTS = [ for name, flags in [ ("", []), ("_act_quant", ["--allow_activation_quant_for_weight_only"]), + ("_no_linear", ["--gtest_filter=-linear_*"]), ] ] diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index 97e54ac7656..4a1054d1e89 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -8,6 +8,7 @@ fbcode_target( name = "record", srcs = [ "cases.py", + "embedding.py", "linear.py", "ops.py", "record.py", diff --git a/backends/native/test/samples/cases.py b/backends/native/test/samples/cases.py index 9a613a14db9..bbca8941762 100644 --- a/backends/native/test/samples/cases.py +++ b/backends/native/test/samples/cases.py @@ -20,6 +20,8 @@ from facto.inputgen.specs.model import Spec Inputs = tuple[torch.Tensor, ...] +# Draws a tensor of the given shape from a generator, as torch.randn does. +Sampler = Callable[..., torch.Tensor] # Caps a suite case's input and output elements over its samples, keeping at # least one sample: a large output alone has enough elements for statistics. @@ -30,7 +32,8 @@ class Case: name: str model: torch.nn.Module - # Positional float32 inputs of each sample; the count may vary by case. + # Positional inputs of each sample, floating ones float32; the count may + # vary by case. inputs: list[Inputs] # `{dim: Dim}` of each positional input. A dynamic case's inputs are the # calls of one sample, which run in order on one executable. @@ -41,20 +44,27 @@ def outputs(result) -> list[torch.Tensor]: return list(result) if isinstance(result, (tuple, list)) else [result] -def draws(count: int, *shapes: tuple[int, ...]) -> list[Inputs]: - """`count` samples of standard-normal inputs of the given shapes, from a fixed - seed, so equal calls return equal samples.""" +def draws( + count: int, *shapes: tuple[int, ...], sampler: Sampler = torch.randn +) -> list[Inputs]: + """`count` samples of inputs of the given shapes from `sampler`, standard + normal by default, and a fixed seed, so equal calls return equal samples.""" generator = torch.Generator().manual_seed(0) return [ - tuple(torch.randn(shape, generator=generator) for shape in shapes) + tuple(sampler(shape, generator=generator) for shape in shapes) for _ in range(count) ] -def chain(name: str, model: torch.nn.Module, *shapes: tuple[int | Dim, ...]) -> Case: - """A dynamic case with standard-normal inputs of `shapes`, from a fixed seed. - Each Dim starts mid-range, which export traces, then visits both bounds and - returns, every other Dim in the opposite order.""" +def chain( + name: str, + model: torch.nn.Module, + *shapes: tuple[int | Dim, ...], + sampler: Sampler = torch.randn, +) -> Case: + """A dynamic case with inputs of `shapes` from `sampler`, standard normal by + default, and a fixed seed. Each Dim starts mid-range, which export traces, + then visits both bounds and returns, every other Dim in the opposite order.""" dims = list(dict.fromkeys(d for s in shapes for d in s if not isinstance(d, int))) walks = {} for k, d in enumerate(dims): @@ -63,7 +73,7 @@ def chain(name: str, model: torch.nn.Module, *shapes: tuple[int | Dim, ...]) -> generator = torch.Generator().manual_seed(0) inputs = [ tuple( - torch.randn( + sampler( [walks[d][step] if d in walks else d for d in s], generator=generator ) for s in shapes @@ -76,6 +86,17 @@ def chain(name: str, model: torch.nn.Module, *shapes: tuple[int | Dim, ...]) -> return Case(name, model, inputs, dynamic_shapes) +class Parallel(torch.nn.Module): + """Modules on one input, each giving its own output.""" + + def __init__(self, *modules: torch.nn.Module) -> None: + super().__init__() + self.branches = torch.nn.ModuleList(modules) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, ...]: + return tuple(m(x) for m in self.branches) + + class _SuiteRunner: """Stands in for the backend test suite's runner, keeping what a test lowers.""" diff --git a/backends/native/test/samples/embedding.py b/backends/native/test/samples/embedding.py new file mode 100644 index 00000000000..f4d854971c9 --- /dev/null +++ b/backends/native/test/samples/embedding.py @@ -0,0 +1,93 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Embedding cases: the backend test suite's, LLM vocabularies and token +shapes, index edge cases, and dynamic shapes.""" + +from functools import partial + +import torch +from executorch.backends.native.test.samples.cases import ( + Case, + chain, + draws, + Parallel, + Sampler, + suite_cases, +) +from executorch.backends.native.test.samples.schemes import GGUF_SUPER_SCALES +from executorch.backends.test.suite.operators import test_embedding +from torch.export import Dim + + +def _index_sampler(vocab: int) -> Sampler: + """A sampler of uniform int64 indices into a `vocab`-row table.""" + return partial(torch.randint, vocab) + + +def cases() -> list[Case]: + suite = suite_cases(test_embedding, samples=4) + torch.manual_seed(0) + tokens, batch = Dim("tokens", min=1, max=64), Dim("batch", min=1, max=4) + return [ + *suite, + Case( + "decode", + torch.nn.Embedding(4096, 256), + draws(8, (1,), sampler=_index_sampler(4096)), + ), + Case( + "prefill", + torch.nn.Embedding(4096, 256), + draws(2, (64,), sampler=_index_sampler(4096)), + ), + Case( + "wide", + torch.nn.Embedding(512, 2048), + draws(4, (4,), sampler=_index_sampler(512)), + ), + Case( + "leading_batch", + torch.nn.Embedding(1000, 256), + draws(4, (2, 7), sampler=_index_sampler(1000)), + ), + Case("scalar_index", torch.nn.Embedding(64, 256), [(torch.tensor(5),)]), + Case( + "edge_rows", + torch.nn.Embedding(512, 256), + [(torch.tensor([0, 511, 0, 511, 7, 7]),)], + ), + Case( + "int32_indices", + torch.nn.Embedding(512, 256), + [(x.int(),) for (x,) in draws(2, (16,), sampler=_index_sampler(512))], + ), + # A zero row 0, and padding_idx in the graph, in the float and torchao + # packages. + Case( + "padding_idx", + torch.nn.Embedding(512, 256, padding_idx=0), + [(torch.tensor([0, 3, 0, 9]),)], + ), + # One output per family of GGUF super-block scales. + Case( + "scale_families", + Parallel(*(torch.nn.Embedding(64, 256) for _ in GGUF_SUPER_SCALES["q4_k"])), + draws(2, (8,), sampler=_index_sampler(64)), + ), + chain( + "dynamic_tokens", + torch.nn.Embedding(4096, 256), + (tokens,), + sampler=_index_sampler(4096), + ), + chain( + "dynamic_batch_tokens", + torch.nn.Embedding(1000, 256), + (batch, tokens), + sampler=_index_sampler(1000), + ), + ] diff --git a/backends/native/test/samples/linear.py b/backends/native/test/samples/linear.py index 6b6a22b1438..0152b6239ef 100644 --- a/backends/native/test/samples/linear.py +++ b/backends/native/test/samples/linear.py @@ -13,6 +13,7 @@ chain, draws, facto_cases, + Parallel, suite_cases, ) from executorch.backends.native.test.samples.schemes import GGUF_SUPER_SCALES @@ -81,17 +82,6 @@ def _facto_case(name: str, x, weight, bias) -> Case: return Case(name, model, [(x,)]) -class _Parallel(torch.nn.Module): - """nn.Linears on one input, each giving its own output.""" - - def __init__(self, *linears: torch.nn.Linear) -> None: - super().__init__() - self.linears = torch.nn.ModuleList(linears) - - def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, ...]: - return tuple(m(x) for m in self.linears) - - class _Pair(torch.nn.Module): """Two modules, each on its own input.""" @@ -202,7 +192,7 @@ def cases() -> list[Case]: # One output per family of GGUF super-block scales. Case( "scale_families", - _Parallel(*(torch.nn.Linear(256, 32) for _ in GGUF_SUPER_SCALES["q4_k"])), + Parallel(*(torch.nn.Linear(256, 32) for _ in GGUF_SUPER_SCALES["q4_k"])), draws(2, (4, 256)), ), # FACTO varies one argument at a time, so only every fifth tuple is kept, diff --git a/backends/native/test/samples/ops.py b/backends/native/test/samples/ops.py index 16332b665ee..78ed6717d39 100644 --- a/backends/native/test/samples/ops.py +++ b/backends/native/test/samples/ops.py @@ -9,7 +9,7 @@ from collections.abc import Callable from dataclasses import dataclass -from executorch.backends.native.test.samples import linear +from executorch.backends.native.test.samples import embedding, linear from executorch.backends.native.test.samples.cases import Case from executorch.backends.native.test.samples.record import DTYPES @@ -21,4 +21,4 @@ class Op: def ops() -> dict[str, Op]: - return {"linear": Op(linear.cases)} + return {"embedding": Op(embedding.cases), "linear": Op(linear.cases)} diff --git a/backends/native/test/samples/record.py b/backends/native/test/samples/record.py index 9465509795e..ed72003af69 100644 --- a/backends/native/test/samples/record.py +++ b/backends/native/test/samples/record.py @@ -10,11 +10,12 @@ and two sample sets over the same inputs: `golden`, the eager outputs, and `precise`, the same math in float64 with quantized weights decoded in float32, whatever the method's dtype, and activations unquantized, floating outputs -stored as float32. Weight-only schemes add `golden-int8_act`, the eager outputs -with the same weights and int8 per-token dynamic activations. A dynamic case -is exported with its dynamic shapes and recorded as one sample whose calls -change them. Nothing in it depends on an engine or a device: consumers judge an -engine by its distance from `precise` relative to eager's. +stored as float32. Weight-only schemes over a case with an nn.Linear add +`golden-int8_act`, the eager outputs with the same weights and int8 per-token +dynamic activations. A dynamic case is exported with its dynamic shapes and +recorded as one sample whose calls change them. Nothing in it depends on an +engine or a device: consumers judge an engine by its distance from `precise` +relative to eager's. """ import copy @@ -61,6 +62,9 @@ def record( ) -> None: """Writes no package, removing any at `path`, when the scheme fits no case.""" programs, golden, precise, golden_act = {}, [], [], [] + act_quant = any( + scheme.adds_int8_act(c.model) for c in cases if scheme.fits(c.model) + ) for case in cases: if not scheme.fits(case.model): logger.info(f"{path.name}: {case.name} does not fit the scheme") @@ -92,7 +96,7 @@ def record( t.float() if t.is_floating_point() else t for t in outputs(ref) ] precise_calls.append(Call(name, list(x), ref)) - if scheme.weight_only: + if act_quant: with int8_activations(method): act_calls.append(Call(name, list(x), outputs(method(*x)))) golden += _samples(golden_calls, chained) @@ -114,7 +118,7 @@ def record( source, ), ) - if scheme.weight_only: + if act_quant: write_samples( str(path), "golden-int8_act", diff --git a/backends/native/test/samples/schemes.py b/backends/native/test/samples/schemes.py index 8d9fe262a43..b9977f6a374 100644 --- a/backends/native/test/samples/schemes.py +++ b/backends/native/test/samples/schemes.py @@ -39,10 +39,15 @@ class Scheme: @property def weight_only(self) -> bool: - """Whether the scheme quantizes only weights; `record` then adds - `golden-int8_act`.""" + """Whether the scheme quantizes only weights.""" return False + def adds_int8_act(self, model: torch.nn.Module) -> bool: + """Whether `record` adds `golden-int8_act` for `model`: the scheme + quantizes only weights and `model` has inputs int8_activations + quantizes.""" + return self.weight_only and bool(_targets(model, _ACTIVATED)) + def fits(self, model: torch.nn.Module) -> bool: return True diff --git a/backends/native/test/samples/test_record.py b/backends/native/test/samples/test_record.py index b89a126822a..1cc2729523d 100644 --- a/backends/native/test/samples/test_record.py +++ b/backends/native/test/samples/test_record.py @@ -7,6 +7,7 @@ import copy import tempfile import unittest +from functools import partial from pathlib import Path from types import ModuleType @@ -18,7 +19,7 @@ draws, suite_cases, ) -from executorch.backends.native.test.samples.record import record +from executorch.backends.native.test.samples.record import DTYPES, record from executorch.backends.native.test.samples.schemes import ( Gguf, GGUF_SUPER_SCALES, @@ -116,6 +117,14 @@ def test_record_adds_the_8da4w_golden_to_weight_only_packages(self) -> None: got.calls[0].outputs, want.calls[0].outputs, rtol=0, atol=0 ) + def test_record_adds_no_8da4w_golden_without_a_linear(self) -> None: + inputs = draws(1, (3,), sampler=partial(torch.randint, 8)) + case = Case("embedding", torch.nn.Embedding(8, 64), inputs) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "embedding.ptn" + record([case], schemes()["int4wo_g32"], path, dtypes=["f32"]) + self.assertEqual(set(read_samples(str(path))), {"golden", "precise"}) + def test_chain_visits_each_dims_bounds_starting_mid_range(self) -> None: a, b = Dim("a", min=1, max=8), Dim("b", min=2, max=4) case = chain("pair", torch.nn.Identity(), (a, 3), (b, a)) @@ -130,6 +139,34 @@ def test_chain_visits_each_dims_bounds_starting_mid_range(self) -> None: ) self.assertEqual(case.dynamic_shapes, ({0: a}, {0: b, 1: a})) + def test_chain_draws_from_its_sampler(self) -> None: + case = chain( + "ids", + torch.nn.Embedding(8, 2), + (Dim("tokens", min=1, max=4),), + sampler=partial(torch.randint, 8), + ) + for (x,) in case.inputs: + self.assertEqual(x.dtype, torch.int64) + self.assertTrue(((x >= 0) & (x < 8)).all()) + + def test_record_keeps_integer_inputs_in_every_dtype(self) -> None: + model = torch.nn.Embedding(8, 4) + inputs = draws(1, (3,), sampler=partial(torch.randint, 8)) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "embedding.ptn" + record([Case("embedding", model, inputs)], Scheme(), path) + samples = read_samples(str(path))["golden"].samples + ((ids,),) = inputs + self.assertEqual(len(samples), len(DTYPES)) + for sample in samples: + (call,) = sample.calls + dtype = DTYPES[call.method.rsplit("_", 1)[1]] + self.assertEqual(call.inputs[0].dtype, torch.int64) + torch.testing.assert_close( + call.outputs[0], model.weight.detach().to(dtype)[ids], rtol=0, atol=0 + ) + def test_record_chains_a_dynamic_case_into_one_sample(self) -> None: case = chain("linear", torch.nn.Linear(32, 4), (Dim("batch", min=1, max=6), 32)) scheme = schemes()["int4wo_g32"]