diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index d7ba414..e7fd5d9 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -21,7 +21,13 @@ jobs: - name: Install modm-data with docs dependencies run: | - pip3 install -r tools/requirements.txt ".[docs]" + pip3 install -r tools/requirements.txt -e ".[docs]" + + - name: Install ARM toolchain and sources for the SVD files + run: | + sudo apt-get update + sudo apt-get install -y gcc-arm-none-eabi + make clone-sources-arm ext/stmicro/header/ ext/stmicro/cubehal/ - name: Clone modm-ext/data.modm.io repository diff --git a/.github/workflows/test-svd.yml b/.github/workflows/test-svd.yml new file mode 100644 index 0000000..b12cf6a --- /dev/null +++ b/.github/workflows/test-svd.yml @@ -0,0 +1,67 @@ +name: Generate SVD Files + +on: + workflow_dispatch: + pull_request: + release: + types: [published] + +jobs: + generate-svd-stmicro: + name: Generate STM32 SVD Files + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + steps: + - name: Check out Repository + uses: actions/checkout@v4 + + - name: Setup Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install Python Dependencies + run: | + pip3 install -r tools/requirements.txt -e . + + - name: Install ARM Toolchain + run: | + sudo apt-get update + sudo apt-get install -y gcc-arm-none-eabi + + - name: Clone STMicro Sources + run: | + make clone-sources-arm ext/stmicro/header/ ext/stmicro/cubehal/ + + - name: Generate SVD Files + run: | + python3 -m modm_data.header2svd.stmicro --all --output svd-archive/ + + - name: Upload SVD Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: svd-logs + path: log/stmicro/svd + retention-days: 1 + + - name: Upload SVD Files + uses: actions/upload-artifact@v4 + with: + name: svd-archive + path: svd-archive/ + retention-days: 7 + + - name: Zip SVD Files + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} + run: | + zip -r svd-archive.zip svd-archive/ + + - name: Upload to Github Release + uses: softprops/action-gh-release@v2 + if: ${{ github.event_name == 'release' && github.event.action == 'published' }} + with: + files: | + svd-archive.zip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3feeb3d..f2af006 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,7 @@ jobs: if: always() uses: chartboost/ruff-action@v1 with: + version: "0.14.10" src: "./src" args: "format --check" @@ -47,5 +48,6 @@ jobs: if: always() uses: chartboost/ruff-action@v1 with: + version: "0.14.10" src: "./src" args: "check" diff --git a/docs/.gitignore b/docs/.gitignore index cd30da5..f2e9bc7 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,6 +1,7 @@ # ignored since these are generated by tools/scripts/synchronize_docs.py src/api/ src/pinout/ +src/svd/ src/index.md src/source/overview.md src/pipeline/overview.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index d08ec2e..ff1e072 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -89,3 +89,4 @@ nav: - Header to SVD: pipeline/header2svd.md - API Reference: api/modm_data.html - Pinouts: pinout/index.html + - SVD Explorer: svd/index.html diff --git a/docs/src/pipeline/header2svd.md b/docs/src/pipeline/header2svd.md index fd7de24..48f32c6 100644 --- a/docs/src/pipeline/header2svd.md +++ b/docs/src/pipeline/header2svd.md @@ -1,15 +1,55 @@ # CMSIS Header to SVD Pipeline +The CMSIS device headers are compiled with `arm-none-eabi-gcc` to extract the +numeric values of all macros and the layout of all peripheral structures. The +bit field macros are then matched to the structure members to reconstruct the +memory map of each device header, see `modm_data.header2svd.stmicro.memory_map`. + +The CMSIS headers are more accurate than the ST SVD files, since they are +compiled and used by the HAL. The ST SVD files are therefore only used to find +discrepancies, which need to be checked with the reference manual. + ## Selective Conversion -The resulting SVD files are found in `ext/stmicro/svd`. -Only takes a few minutes. +The resulting SVD files are found in `ext/stmicro/svd/header_*.svd` and the +reports in `log/stmicro/svd/header_*.txt`. The extracted header data is cached +in `ext/cache/cmsis/header2svd`. + +```bash +# Convert all headers matching the pattern into SVD files +python3 -m modm_data.header2svd.stmicro --header stm32f4 +# Convert all CMSIS headers and compare them with the ST SVD files +python3 -m modm_data.header2svd.stmicro --all --compare +``` + +The CubeHAL source code in `ext/stmicro/cubehal` provides additional information, +see `modm_data.header2svd.stmicro.cubehal`: + +- Register accesses like `SET_BIT(USARTx->CR1, USART_CR1_UE)` pair registers + with bit field macros when the naming heuristics fail. +- The `IS_*_INSTANCE` macros documented in the LL functions remove the bit + fields and registers that are not supported by an instance of a shared + structure type, for example, the break and dead-time register of basic timers. +- The LL function parameters are evaluated as enumerated values of bit fields. + +The interrupts and the descriptions of peripherals, registers and bit fields +are taken from the CMSIS header. Overlapping bit fields with a different layout +are placed into alternate registers, for example, the input capture bit fields +of `TIM_CCMR1` in `CCMR1_ALT`. + +The report lists the bit field macros that could not be assigned to a register, +the registers without bit fields, the overlapping bit fields that were removed, +the alternate registers, the registers paired by CubeHAL, the bit fields not +supported by an instance, the unassigned interrupts and the differences to the +ST SVD file. + +The SVD files are also published on the homepage with the +[SVD Explorer](https://gist.github.com/salkinium/12a18032caa303697c6583937f6fcd16), +which lists them from the `svd-files.json` file: ```bash -# Convert a group of devices into SVD files -python3 -m modm_data.header2svd.stmicro --device stm32f030c6t6 --device stm32f030f4p6 --device stm32f030k6t6 -# Convert all CMSIS headers into SVD files -python3 -m modm_data.header2svd.stmicro --all +# Convert all CMSIS headers into the documentation folder +python3 -m modm_data.header2svd.stmicro --all --output docs/src/svd/ ``` ## Automatic Conversion @@ -20,5 +60,5 @@ To perform the steps automatically, you may also use `make`: # Using make make convert-stmicro-header-svd # Remove all svd files -make clean-stmicro-svd +make clean-stmicro-header-svd ``` diff --git a/pyproject.toml b/pyproject.toml index 873611d..84d97e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ dependencies = [ "anytree==2.12.1", "CppHeaderParser>=2.7.4,<3", + "cxxheaderparser>=1.4.1,<2", "jinja2>=3.1.3,<4", "lxml>=5.2.0,<6", "kuzu>=0.11.3,<0.12", @@ -47,7 +48,7 @@ docs = [ "pdoc>=14,<15", "mkdocs>=1.5,<2", "mkdocs-material>=9.5,<10", - "modm-pinout @ git+https://github.com/modm-io/modm-pinout.git@main", + "modm-pinout @ git+https://github.com/modm-ext/modm-pinout.git@main", ] all = ["modm_data[docs]"] diff --git a/src/modm_data/cubehal/__init__.py b/src/modm_data/cubehal/__init__.py index ae06081..800985e 100644 --- a/src/modm_data/cubehal/__init__.py +++ b/src/modm_data/cubehal/__init__.py @@ -11,5 +11,16 @@ """ from .dmamux_requests import read_request_map, read_bdma_request_map +from .header import read_header +from .registers import RegisterAccess, LLFunction, register_accesses, ll_functions, ll_descriptions -__all__ = ["read_request_map", "read_bdma_request_map"] +__all__ = [ + "read_request_map", + "read_bdma_request_map", + "read_header", + "RegisterAccess", + "LLFunction", + "register_accesses", + "ll_functions", + "ll_descriptions", +] diff --git a/src/modm_data/cubehal/header.py b/src/modm_data/cubehal/header.py new file mode 100644 index 0000000..88e7715 --- /dev/null +++ b/src/modm_data/cubehal/header.py @@ -0,0 +1,283 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +import re +import subprocess +import logging +import shutil +import pickle + +from pathlib import Path +from jinja2 import Environment +import cxxheaderparser.simple +import cxxheaderparser.types + +from ..utils import ext_path + + +_LOGGER = logging.getLogger(__file__) + +_CUBE_PATH = ext_path("stmicro/cubehal") +_HEADER_PATH = ext_path("stmicro/header") +_CMSIS_PATH = ext_path("arm/cmsis/CMSIS/Core/Include") +_CACHE_PATH = ext_path("cache/cmsis/stm32") + +_SUBSTITUTIONS = [ + # (r"/\* +?(Legacy defines|Legacy aliases|Old .*? legacy purpose|Aliases for .*?) +?\*/.*?\n\n", "", re.S|re.M), + (r"/\* +?[Ll]egacy (aliases|defines|registers naming) +?\*/.*?\n\n", "", re.S | re.M), + (r"/\* +?Old .*? legacy purpose +?\*/.*?\n\n", "", re.S | re.M), + (r"/\* +?Aliases for .*? +?\*/.*?\n\n", "", re.S | re.M), + # (r"( 0x[0-9A-F]+)\) ", r"\1U", re.S|re.M), + (r"#define[^<]+?/\*!<.*? [Ll]egacy .*?\*/\n", "", 0), +] + +_HEADER_TEMPLATE = r""" +#include +#include +#include +#include + +{% for name, value in defines | items | sort %} +#define {{name}} {{value}}{% endfor %} + +void __dm_f(char const *symbol, std::integral auto value) { + std::cout << "\"" << symbol << "\": " << value << ",\n"; +} +void __dm_f(char const *symbol, unsigned char value) { + std::cout << "\"" << symbol << "\": " << (uint16_t)value << ",\n"; +} +void __dm_f(char const *symbol, const char *value) { + std::cout << "\"" << symbol << "\": \"" << value << "\",\n"; +} +#define __dm(def) __dm_f(#def, (def)); +int main() +{ +std::cout << "defines = {";{% for name, value in defines | items | sort %} +__dm({{name}}){% endfor %} +std::cout << "}"; +return 0; +} +""" + + +def _get_define_for_device(did, familyDefines): + """ + Returns the STM32 specific define from an identifier + """ + if len(familyDefines) == 1: + return familyDefines[0] + + devName = f"STM32{did.family.upper()}{did.name.upper()}" + + # Map STM32F7x8 -> STM32F7x7 + if did.family == "f7" and devName[8] == "8": + devName = devName[:8] + "7" + + deviceDefines = [d for d in familyDefines if d.startswith(devName)] + # if there is only one define thats the one + if len(deviceDefines) == 1: + return deviceDefines[0] + + # sort with respecting variants + minlen = min(len(d) for d in deviceDefines) + deviceDefines.sort(key=lambda d: (d[:minlen], d[minlen:])) + + # now we match for the size-id (and variant-id if applicable). + devNameMatch = f"{devName}x{did.size.upper()}" + if did.family == "l1": + # Map STM32L1xxQC and STM32L1xxZC -> STM32L162QCxA variants + if did.pin in ["q", "z"] and did.size == "c": + devNameMatch += "A" + else: + devNameMatch += did.variant.upper() + elif did.family == "h7": + devNameMatch = devNameMatch[:-1] + "x" + if did.variant: + devNameMatch += did.variant.upper() + for define in deviceDefines: + if devNameMatch <= define: + return define + + # now we match for the pin-id. + devNameMatch = f"{devName}{did.pin.upper()}x" + for define in deviceDefines: + if devNameMatch <= define: + return define + + return None + + +def _read_cpp_header( + core: str, device: str, includes: list[Path], headers: list[Path], options: str = None +) -> dict[str, str]: + cmd = f"arm-none-eabi-gcc -E -mcpu=cortex-m{core} -D {device} {options or ''}" + for p in includes: + cmd += f" -I {p}" + for h in headers: + cmd += f" {h}" + _LOGGER.debug(cmd) + output = subprocess.run(cmd, shell=True, capture_output=True) + assert not output.returncode, output.stderr.decode("utf-8") + return output.stdout.decode("utf-8") + + +def _read_defines(core: str, device: str, includes: list[Path], headers: list[Path]) -> dict[str, str]: + output = _read_cpp_header(core, device, includes, headers, "-dM") + _LOGGER.debug(output) + defines = {} + for line in output.splitlines(): + name, value = line[8:].split(" ", 1) + defines[name] = value + return defines + + +def _resolve_defines(device, includes, defines, recursive=20): + source = (_CACHE_PATH / f"{device[:7].lower()}xx" / device).with_suffix(".cpp").absolute() + + if not (runner := source.with_suffix(".run")).exists(): + _LOGGER.info(f"Generating '{source.name}'...") + template = Environment().from_string(_HEADER_TEMPLATE) + source.write_text(template.render({"defines": defines})) + + _LOGGER.info(f"Compiling '{source.name}'...") + cmd = f"g++-13 -std=c++20 -Wno-narrowing -fms-extensions -fext-numeric-literals -D {device}" + for inc in includes: + cmd += f" -I {inc}" + cmd += f" -o {runner} {source}" + _LOGGER.debug(cmd) + output = subprocess.run(cmd, shell=True, capture_output=True) + if output.returncode: + assert recursive > 0, "Failed to compile" + output = output.stderr.decode("utf-8") + for missing in re.findall(r"error: ('.+?' was not declared in this scope).+?", output): + _LOGGER.debug(missing) + ignore = re.findall(r"\| +__dm\((.+?)\)", output) + ignore += re.findall(r"warning: \"(.+?)\" redefined", output) + _LOGGER.info(f"Removing {ignore}") + defines = {k: v for k, v in defines.items() if k not in ignore} + return _resolve_defines(device, includes, defines, recursive - 1) + + _LOGGER.info(f"Running '{runner.name}'...") + output = subprocess.run([str(runner)], shell=True, capture_output=True) + localv = {} + exec(output.stdout, globals(), localv) + return localv["defines"] + + +def _copy_headers(outdir, files): + for file in files: + if not (outfile := (outdir / file.name)).exists(): + content = file.read_text(encoding="utf-8-sig", errors="replace") + for pattern, subs, flags in _SUBSTITUTIONS: + content = re.sub(pattern, subs, content, flags=flags) + outfile.write_text(content) + + +def _read_header(device_define): + family = f"{device_define[:7].lower()}xx" + header_inc = _HEADER_PATH / family / "Include" + cube_inc = _CUBE_PATH / family / "Inc" + + (header_cache := (_CACHE_PATH / family)).mkdir(parents=True, exist_ok=True) + includedirs = [_CMSIS_PATH, header_cache] + + # Copy over all header files and clean them up + if not (dst := header_cache / "Legacy").exists() and (src := cube_inc / "Legacy").exists(): + shutil.copytree(src, dst) + for file in dst.glob("*.h"): + file.write_text("") + _copy_headers(header_cache, header_inc.glob("*.h")) + _copy_headers(header_cache, cube_inc.glob("*.h")) + + device_header = header_cache / f"{device_define.lower()}.h" + core_header, core = re.search(r'#include "(core_cm(.+?).h)"', device_header.read_text()).groups() + header_defines = _read_defines(core, device_define, includedirs, [device_header]) + + # ARM CMSIS header files + core_defines = _read_defines(core, device_define, [_CMSIS_PATH], [_CMSIS_PATH / core_header]) + + ll_headers = list(header_cache.glob("*_ll_*.h")) + ll_defines = _read_defines(core, device_define, includedirs, ll_headers) + + cube_headers = list(header_cache.glob("*_hal_*.h")) + all_defines = _read_defines(core, device_define, includedirs, cube_headers) + + value_defines = { + k: v for k, v in all_defines.items() if v and "*" not in v and "(" not in k and not k.startswith("__") + } + value_defines = _resolve_defines(device_define, includedirs, value_defines) + + device_header_content = _read_cpp_header( + core, device_define, includedirs, [device_header], "-P -D __inline= -D __extension__= -D __restrict=" + ) + cxxheader = cxxheaderparser.simple.parse_string(device_header_content) + + peripherals = {} + clanons = { + c.class_decl.typename.segments[0].id: c + for c in cxxheader.namespace.classes + if isinstance(c.class_decl.typename.segments[0], cxxheaderparser.types.AnonymousName) + } + for typedef in cxxheader.namespace.typedefs: + if typedef.name.endswith("_TypeDef"): + fields = [] + tclass = clanons[typedef.type.typename.segments[0].id] + for field in tclass.fields: + if isinstance(field.type, cxxheaderparser.types.Array): + fields.append( + ( + field.name, + field.type.array_of.typename.segments[0].name, + int(field.type.size.tokens[0].value.format()), + ) + ) + else: + fields.append((field.name, field.type.typename.segments[0].name, None)) + peripherals[typedef.name] = fields + + irq_vectors = [(int(i.value.format()), i.name[:-5]) for i in cxxheader.namespace.enums[0].values] + + return { + "device_define": device_define, + "core": core_defines, + "header": header_defines, + "ll": ll_defines, + "cube": all_defines, + "defines": value_defines, + "irqs": irq_vectors, + "peripherals": peripherals, + } + + +def read_header(did): + """ + Finds all register and bit names in the CMSIS header file. + + :returns: a RegisterMap object that allows regex-ing for register names. + """ + family_folder = f"stm32{did.family}xx" + if did.string[5:8] in ["h7r", "h7s"]: + family_folder = "stm32h7rsxx" + elif did.string[5:8] == "wba": + family_folder = "stm32wbaxx" + family_header = f"{family_folder}.h" + if did.string[5:8] == "wb0": + family_folder = "stm32wb0xx" + family_header = "stm32wb0x.h" + elif did.string[5:8] == "wl3": + family_folder = "stm32wl3xx" + family_header = "stm32wl3x.h" + family_header = (_HEADER_PATH / family_folder / "Include" / family_header).read_text( + encoding="utf-8", errors="replace" + ) + match = re.findall(r"if +defined\( *(STM32[A-Z][\w\d]+) *\)", family_header) + assert match, f"No CPP define match found for '{did.string}'!" + device_define = _get_define_for_device(did, match) + assert device_define, f"No device define found for '{did.string}'!" + + if (pkl := _CACHE_PATH / family_folder / f"{device_define}.pkl").exists(): + return pickle.loads(pkl.read_bytes()) + + values = _read_header(device_define) + pkl.write_bytes(pickle.dumps(values, protocol=pickle.HIGHEST_PROTOCOL)) + return values diff --git a/src/modm_data/cubehal/registers.py b/src/modm_data/cubehal/registers.py new file mode 100644 index 0000000..de0e2a4 --- /dev/null +++ b/src/modm_data/cubehal/registers.py @@ -0,0 +1,170 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Register Accesses in CubeHAL + +The CubeHAL and CubeLL source code accesses the registers of the CMSIS header +structures together with the bit field macros, which pairs the register +members with their bit field macros, for example, `SET_BIT(USARTx->CR1, +USART_CR1_UE)`. + +The LL inline functions are additionally documented with the register and bit +field names in the reference manual (`@rmtoll CR1 UE`), the enumerated values +of the function parameters (`@arg @ref LL_USART_PARITY_EVEN`) and the +instances that support the function (`@note IS_UART_HWFLOW_INSTANCE`). +""" + +import re +from pathlib import Path +from functools import cache +from collections import defaultdict +from dataclasses import dataclass, field + +from ..utils import ext_path + +_CUBE_PATH = ext_path("stmicro/cubehal") +_ACCESS = re.compile(r"\b(?:ATOMIC_)?(SET_BIT|CLEAR_BIT|READ_BIT|MODIFY_REG|WRITE_REG|READ_REG|CLEAR_REG)\s*\(") +_FUNCTION = re.compile( + r"/\*\*((?:(?!\*/).)*?)\*/\s*__STATIC_INLINE\s+[\w\s\*]+?\b(LL_\w+)\s*\(([^)]*)\)\s*\{", flags=re.S +) +_IDENTIFIER = re.compile(r"\b[A-Z][A-Z0-9]*_[A-Za-z0-9_]+\b") + + +@dataclass +class RegisterAccess: + kind: str + """The access macro, e.g. `MODIFY_REG`.""" + member: str + """The register member of the structure, e.g. `CR1`.""" + types: set[str] + """The possible structure types of the accessed variable.""" + variable: str + """The name of the accessed variable or instance, e.g. `USARTx` or `RCC`.""" + masks: set[str] + """The identifiers in the mask argument, e.g. `USART_CR1_UE`.""" + + +@dataclass +class LLFunction: + name: str + """The name of the function, e.g. `LL_USART_SetParity`.""" + file: str + """The name of the header file.""" + typedef: str | None + """The structure type of the instance parameter.""" + rmtoll: list[tuple[str, str]] = field(default_factory=list) + """The register and bit field names in the reference manual.""" + values: list[str] = field(default_factory=list) + """The enumerated values of the parameter or return value.""" + instance_macros: set[str] = field(default_factory=set) + """The `IS_*_INSTANCE` macros that check for instance support.""" + accesses: list[RegisterAccess] = field(default_factory=list) + """The register accesses in the function body.""" + + +def folder(family: str) -> Path | None: + """:return: the CubeHAL folder of a CMSIS header family folder, e.g. `stm32wb0xx` -> `stm32wb0x`.""" + for name in (family, family[:-1]): + if (path := _CUBE_PATH / name).exists(): + return path + return None + + +def _arguments(text: str, start: int) -> list[str]: + """:return: the top-level arguments of a macro call starting after the opening parenthesis.""" + depth, arguments, current = 1, [], [] + for char in text[start:]: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if not depth: + break + if char == "," and depth == 1: + arguments.append("".join(current)) + current = [] + else: + current.append(char) + arguments.append("".join(current)) + return arguments + + +def _accesses(text: str, types: dict[str, set[str]]) -> list[RegisterAccess]: + accesses = [] + for match in _ACCESS.finditer(text): + arguments = _arguments(text, match.end()) + target = re.fullmatch( + r"[\s\(]*(?:\(\s*\w+\s*\*\s*\)\s*)?([\w\->\.\[\]\s\+\*]*?)\s*->\s*(\w+)\s*(?:\[.*?\])?[\s\)]*", arguments[0] + ) + if not target: + continue + variable = re.split(r"->|\.", target.group(1))[-1].strip("() *") + masks = set(_IDENTIFIER.findall(arguments[1])) if len(arguments) > 1 else set() + kind = match.group(1) + # Written and read values are not masks + if kind in ("WRITE_REG", "READ_REG", "CLEAR_REG"): + masks = set() + accesses.append(RegisterAccess(kind, target.group(2), set(types.get(variable, ())), variable, masks)) + return accesses + + +def _read(path: Path) -> str: + text = path.read_text(encoding="utf-8", errors="replace") + return re.sub(r"/\*.*?\*/", lambda m: "\n" * m.group(0).count("\n"), text, flags=re.S) + + +@cache +def register_accesses(path: Path) -> list[RegisterAccess]: + """ + :param path: the CubeHAL folder of a family. + :return: all register accesses with a structure member in the HAL and LL source code. + """ + texts = {file: _read(file) for file in sorted(path.glob("*/*.[ch]"))} + # Variables and handle members are declared per module, e.g. UART_HandleTypeDef::Instance in hal_uart.h + modules = defaultdict(lambda: defaultdict(set)) + for file, text in texts.items(): + module = re.sub(r"(_ex)?\.[ch]$", "", file.name) + for typedef, variable in re.findall(r"\b(\w+_TypeDef)\s*\*\s*(?:const\s+)?(\w+)", text): + modules[module][variable].add(typedef) + accesses = [] + for file, text in texts.items(): + accesses += _accesses(text, modules[re.sub(r"(_ex)?\.[ch]$", "", file.name)]) + return accesses + + +@cache +def ll_functions(path: Path) -> list[LLFunction]: + """ + :param path: the CubeHAL folder of a family. + :return: all documented LL inline functions. + """ + functions = [] + for file in sorted((path / "Inc").glob("*_ll_*.h")): + text = file.read_text(encoding="utf-8", errors="replace") + for match in _FUNCTION.finditer(text): + doc, name, parameters = match.groups() + body = text[match.end() : text.find("\n}", match.end())] + typedef = re.search(r"(\w+_TypeDef)\s*\*\s*(\w+)", parameters) + function = LLFunction(name, file.name, typedef.group(1) if typedef else None) + for rmtoll in re.finditer(r"@rmtoll\s+(.*?)(?=@param|@retval|@note|@brief|$)", doc, flags=re.S): + for line in rmtoll.group(1).split("\\n"): + if len(parts := line.replace("*", " ").split()) >= 2: + function.rmtoll.append((parts[0], parts[1])) + function.values = re.findall(r"@arg\s+@ref\s+(LL_\w+)", doc) + function.instance_macros = set(re.findall(r"\b(IS_\w+_INSTANCE)\b", doc)) + types = {typedef.group(2): {typedef.group(1)}} if typedef else {} + function.accesses = _accesses(body, types) + functions.append(function) + return functions + + +@cache +def ll_descriptions(path: Path) -> dict[str, str]: + """:return: the descriptions of all LL macros, e.g. `LL_USART_PARITY_EVEN`.""" + descriptions = {} + for file in sorted((path / "Inc").glob("*_ll_*.h")): + text = file.read_text(encoding="utf-8", errors="replace") + for name, description in re.findall(r"#define\s+(LL_\w+)\s+[^\n]*?/\*!<\s*(.*?)\s*\*/", text): + descriptions.setdefault(name, " ".join(description.split())) + return descriptions diff --git a/src/modm_data/dl/stmicro/document.py b/src/modm_data/dl/stmicro/document.py index 02c6cb5..58dc909 100644 --- a/src/modm_data/dl/stmicro/document.py +++ b/src/modm_data/dl/stmicro/document.py @@ -75,6 +75,7 @@ def __hash__(self) -> int: "SC2155", "SC2157", "SC2156", + "LN2411", ], # Technical docs for STM32 development boards "boards": [ diff --git a/src/modm_data/header2svd/extract.py b/src/modm_data/header2svd/extract.py new file mode 100644 index 0000000..829c59b --- /dev/null +++ b/src/modm_data/header2svd/extract.py @@ -0,0 +1,404 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Header Extraction + +The numeric values of the macros and the layout of the structures in a CMSIS +header are extracted by cross-compiling the header with `arm-none-eabi-gcc`, +so that all expressions are evaluated with the exact semantics of the target +instead of being interpreted in Python or by the host compiler. + +All object-like macros and all `offsetof()` and `sizeof()` expressions of the +structure members are compiled into a single array of 64-bit integers, which +is then copied out of the object file with `arm-none-eabi-objcopy`. Expressions +that fail to compile are removed from the array until the compilation succeeds. +The results are cached, since this is quite expensive. +""" + +import re +import struct +import pickle +import hashlib +import logging +import tempfile +import subprocess +from pathlib import Path +from dataclasses import dataclass, field + +from functools import cache + +import cxxheaderparser.simple +import cxxheaderparser.types as ctypes + +from ..utils import ext_path, cache_path + +LOGGER = logging.getLogger(__name__) + +_CMSIS_PATH = ext_path("arm/cmsis/CMSIS/Core/Include") +_CACHE_PATH = cache_path("cmsis/header2svd") +_VERSION = 2 + + +@dataclass +class Member: + name: str + """Name of the structure member.""" + type: str + """Type name of the member, or `pointer` for pointer members.""" + length: str | None + """The array length expression or `None` if the member is not an array.""" + + +@dataclass +class HeaderData: + header: Path + """The path to the header file.""" + defines: list[str] + """The macros defined on the command line.""" + macros: dict[str, str] = field(default_factory=dict) + """All object-like macros defined by the header and its includes.""" + values: dict[str, int] = field(default_factory=dict) + """The integer values of all macros that can be evaluated.""" + structs: dict[str, list[Member]] = field(default_factory=dict) + """The members of all structure typedefs. Named anonymous nested structures + are called `{typedef}__{member}`.""" + aliases: dict[str, str] = field(default_factory=dict) + """Structure typedef aliases, e.g. `typedef XSPI_TypeDef OCTOSPI_TypeDef;`""" + instances: dict[str, tuple[str, int]] = field(default_factory=dict) + """The typedef and address of all pointer macros, e.g. `((USART_TypeDef *) USART1_BASE)`.""" + sizeof: dict[str, int] = field(default_factory=dict) + """The size of all structures.""" + offset: dict[tuple[str, str], int] = field(default_factory=dict) + """The offset of all structure members by (typedef, member).""" + size: dict[tuple[str, str], int] = field(default_factory=dict) + """The size of all structure members by (typedef, member).""" + local_macros: dict[str, int] = field(default_factory=dict) + """The position of all macro definitions in the header file itself, + which excludes the macros defined in the included headers.""" + interrupts: dict[str, int] = field(default_factory=dict) + """The interrupt numbers by name without the `_IRQn` suffix.""" + instance_macros: dict[str, set[str]] = field(default_factory=dict) + """The identifiers checked by the `IS_*_INSTANCE(INSTANCE)` macros, including nested macros.""" + type_descriptions: dict[str, str] = field(default_factory=dict) + """The descriptions of the structure typedefs.""" + member_descriptions: dict[tuple[str, str], str] = field(default_factory=dict) + """The descriptions of the structure members by (typedef, member).""" + macro_descriptions: dict[str, str] = field(default_factory=dict) + """The descriptions of the macros and interrupts.""" + + +def _cpu(content: str) -> str: + core = re.search(r'#include +"core_cm(\w+?)\.h"', content).group(1) + return {"0plus": "cortex-m0plus"}.get(core, f"cortex-m{core}") + + +def _compiler_options(header: Path, defines: list[str]) -> list[str]: + content = header.read_text(encoding="utf-8", errors="replace") + options = [f"-mcpu={_cpu(content)}"] + options += [f"-D{d}" for d in defines] + options += [f"-I{_CMSIS_PATH}", f"-I{header.parent}"] + return options + + +def _preprocess(header: Path, defines: list[str], options: list[str] = None) -> str: + cmd = ["arm-none-eabi-gcc", "-E", *(options or []), *_compiler_options(header, defines), str(header)] + LOGGER.debug(" ".join(cmd)) + output = subprocess.run(cmd, capture_output=True, text=True) + if output.returncode: + raise ValueError(f"Preprocessing {header.name} failed:\n{output.stderr}") + return output.stdout + + +def _macros(header: Path, defines: list[str]) -> dict[str, str]: + macros = {} + for line in _preprocess(header, defines, ["-dM"]).splitlines(): + # Ignore function-like macros + if match := re.match(r"#define (\w+)(\(.*?\))? ?(.*)", line): + if not match.group(2): + macros[match.group(1)] = match.group(3) + return macros + + +def _anonymous_classes(classes) -> dict: + return { + c.class_decl.typename.segments[0].id: c + for c in classes + if isinstance(c.class_decl.typename.segments[0], ctypes.AnonymousName) + } + + +def _structs(header: Path, defines: list[str]) -> tuple[dict[str, list[Member]], dict[str, str], dict, dict]: + output = _preprocess(header, defines) + # Only parse the content of the device header itself, not the system headers + lines, keep = [], False + for line in output.splitlines(): + if match := re.match(r'# \d+ "(.*?)"', line): + keep = Path(match.group(1)).name == header.name + continue + if keep: + lines.append(line) + source = re.sub(r"__attribute__\s*\(\(.*?\)\)", "", "\n".join(lines)) + parsed = cxxheaderparser.simple.parse_string(source) + + structs, nested = {}, {} + + def members(cls, name): + result = [] + inner = _anonymous_classes(cls.classes) + named = set() + for member in cls.fields: + mtype, length = member.type, None + if isinstance(mtype, ctypes.Array): + length = "".join(token.value for token in mtype.size.tokens) + mtype = mtype.array_of + if isinstance(mtype, (ctypes.Pointer, ctypes.FunctionType)) or not hasattr(mtype, "typename"): + if member.name: + result.append(Member(member.name, "pointer", length)) + continue + segment = mtype.typename.segments[-1] + if isinstance(segment, ctypes.AnonymousName): + named.add(segment.id) + if (inner_cls := inner.get(segment.id)) is None: + continue + if member.name is None: + result.extend(members(inner_cls, name)) + continue + # Named anonymous structure, e.g. struct { ... } MTL_QUEUE[2]; + subtype = f"{name}__{member.name}" + nested[subtype] = (name, member.name, length is not None) + structs[subtype] = members(inner_cls, subtype) + result.append(Member(member.name, subtype, length)) + continue + if member.name: + result.append(Member(member.name, segment.name, length)) + # Anonymous unions and structures without a member name + for cid, inner_cls in inner.items(): + if cid not in named: + result.extend(members(inner_cls, name)) + return result + + classes = _anonymous_classes(parsed.namespace.classes) + aliases = {} + for typedef in parsed.namespace.typedefs: + if (typename := getattr(typedef.type, "typename", None)) is None: + continue + segment = typename.segments[0] + if not isinstance(segment, ctypes.AnonymousName): + if typedef.name != segment.name: + aliases[typedef.name] = segment.name + continue + cls = classes.get(segment.id) + if cls is not None and cls.class_decl.classkey == "struct": + structs[typedef.name] = members(cls, typedef.name) + aliases = {name: target for name, target in aliases.items() if target in structs} + + interrupts = {} + for enum in parsed.namespace.enums: + for value in enum.values: + if value.name.endswith("_IRQn") and value.value is not None: + number = "".join(token.value for token in value.value.tokens) + if re.fullmatch(r"-?\d+", number): + interrupts[value.name[:-5]] = int(number) + return structs, aliases, nested, interrupts + + +def _clean(description: str) -> str: + return " ".join(description.replace("*", " ").split()).strip(" ,.") + + +def _descriptions(content: str, data: HeaderData): + """Parses the descriptions of the structures, members and macros from the comments.""" + for match in re.finditer(r"(?:/\*\*((?:(?!\*/).)*)\*/\s*)?typedef\s+struct\s*\w*\s*\{", content, flags=re.S): + end, depth = match.end(), 1 + while depth and end < len(content): + depth += {"{": 1, "}": -1}.get(content[end], 0) + end += 1 + if not (name := re.match(r"\s*(\w+)\s*;", content[end:])): + continue + typedef = name.group(1) + if match.group(1) and (brief := re.search(r"@brief\s+(.*?)(?:@|$)", match.group(1), flags=re.S)): + data.type_descriptions[typedef] = _clean(brief.group(1)) + body = content[match.end() : end - 1] + for member, description in re.findall(r"(\w+)\s*(?:\[[^\]]*\])?\s*;\s*/\*!<\s*(.*?)\*/", body, flags=re.S): + description = re.sub(r",?\s*(?:Address\s+)?offset.*$", "", _clean(description), flags=re.I | re.S) + if description and not member.upper().startswith("RESERVED"): + data.member_descriptions.setdefault((typedef, member), description.strip(" ,.")) + for name, description in re.findall(r"#define[ \t]+(\w+)[ \t]+[^\n]*?/\*!<(.*?)\*/", content): + if (description := _clean(description)) and not re.fullmatch(r"(0x)?[0-9A-Fa-f]+U?L?", description): + data.macro_descriptions.setdefault(name, description) + for name, description in re.findall(r"\b(\w+_IRQn)\s*=\s*-?\d+\s*,?\s*/\*!<(.*?)\*/", content): + data.macro_descriptions.setdefault(name, _clean(description)) + + +def _instance_macros(content: str) -> dict[str, set[str]]: + """:return: the identifiers of the IS_*_INSTANCE(INSTANCE) macros with nested macros expanded.""" + content = re.sub(r"\\\s*\n", " ", content) + bodies = dict(re.findall(r"#define\s+(IS_\w+_INSTANCE)\s*\(\s*\w+\s*\)(.*)", content)) + + @cache + def expand(name, depth=0): + identifiers = set() + for identifier in re.findall(r"\b[A-Za-z_]\w*\b", bodies.get(name, "")): + if identifier in bodies and depth < 10: + identifiers |= expand(identifier, depth + 1) + elif identifier != "INSTANCE": + identifiers.add(identifier) + return frozenset(identifiers) + + return {name: set(expand(name)) for name in bodies} + + +def evaluate( + header: Path, defines: list[str], expressions: list[tuple], prelude: str = None, paths: list[Path] = None +) -> dict: + """ + Compiles a list of (key, expression) pairs in the context of the header and + returns the integer values of all expressions that compiled successfully. + + :param prelude: the source code to use instead of including the header. + :param paths: additional include paths. + """ + expressions = list(expressions) + options = _compiler_options(header, defines) + [f"-I{path}" for path in (paths or [])] + with tempfile.TemporaryDirectory() as tmpdir: + source, objfile, binfile = (Path(tmpdir) / name for name in ("values.c", "values.o", "values.bin")) + while expressions: + lines = (prelude or f'#include "{header.name}"').splitlines() + lines += [ + "#include ", + '__attribute__((used, section(".dm_values"))) const unsigned long long __dm_values[] = {', + ] + first = len(lines) + 1 + lines += [f"(unsigned long long)({expr})," for _, expr in expressions] + lines += ["};"] + source.write_text("\n".join(lines) + "\n") + cmd = ["arm-none-eabi-gcc", "-c", "-w", "-fmax-errors=0", *options, "-o", str(objfile), str(source)] + output = subprocess.run(cmd, capture_output=True, text=True) + if output.returncode: + # Errors and notes about expressions are reported at their line + failed = { + int(line) - first for line in re.findall(r"values\.c:(\d+):\d+: (?:error|note)", output.stderr) + } + failed = {index for index in failed if 0 <= index < len(expressions)} + if not failed: + raise ValueError(f"Compiling values of {header.name} failed:\n{output.stderr[-2000:]}") + LOGGER.debug(f"Removing {len(failed)} expressions: {[expressions[i][1] for i in sorted(failed)][:10]}") + expressions = [expr for index, expr in enumerate(expressions) if index not in failed] + continue + cmd = ["arm-none-eabi-objcopy", "-O", "binary", "-j", ".dm_values", str(objfile), str(binfile)] + subprocess.run(cmd, check=True) + data = binfile.read_bytes() + values = struct.unpack(f"<{len(data) // 8}Q", data) + return {key: value for (key, _), value in zip(expressions, values)} + return {} + + +def _extract(header: Path, defines: list[str]) -> HeaderData: + data = HeaderData(header, defines) + data.macros = _macros(header, defines) + data.structs, data.aliases, nested, data.interrupts = _structs(header, defines) + + content = header.read_text(encoding="utf-8", errors="replace") + for match in re.finditer(r"^\s*#\s*define\s+(\w+)", content, flags=re.M): + data.local_macros.setdefault(match.group(1), match.start()) + _descriptions(content, data) + data.instance_macros = _instance_macros(content) + + instances = {} + for name, value in data.macros.items(): + if match := re.fullmatch(r"\(\(\s*(\w+)\s*\*\s*\)\s*(.+?)\s*\)", value): + instances[name] = match.group(1) + + expressions = [ + (("value", name), name) + for name, value in data.macros.items() + if value and name not in instances and not name.startswith("__") + ] + expressions += [(("address", name), f"(unsigned long)({name})") for name in instances] + + def access(typedef): + """:return: (root typedef, member access path) of nested structures.""" + if typedef not in nested: + return typedef, "" + parent, member, is_array = nested[typedef] + root, path = access(parent) + return root, f"{path}.{member}{'[0]' if is_array else ''}".lstrip(".") + + for typedef, members in data.structs.items(): + root, path = access(typedef) + if path: + expressions.append((("sizeof", typedef), f"sizeof((({root}*)0)->{path})")) + else: + expressions.append((("sizeof", typedef), f"sizeof({typedef})")) + for member in members: + if path: + offset = f"offsetof({root}, {path}.{member.name}) - offsetof({root}, {path})" + else: + offset = f"offsetof({typedef}, {member.name})" + prefix = f"{path}." if path else "" + expressions.append((("offset", typedef, member.name), offset)) + expressions.append((("size", typedef, member.name), f"sizeof((({root}*)0)->{prefix}{member.name})")) + + values = evaluate(header, defines, expressions) + for key, value in values.items(): + if key[0] == "value": + data.values[key[1]] = value + elif key[0] == "sizeof": + data.sizeof[key[1]] = value + elif key[0] == "offset": + data.offset[key[1:]] = value + elif key[0] == "size": + data.size[key[1:]] = value + for name, typedef in instances.items(): + if (address := values.get(("address", name))) is not None: + data.instances[name] = (typedef, address) + return data + + +def extract_values(header: Path, defines: list[str], sources: dict[str, tuple[str, list[str]]], paths: list[Path]): + """ + Evaluates macros of additional source code in the context of a CMSIS header. + Source code that cannot be compiled is ignored. + + :param sources: the source code and the names of the macros to evaluate by name. + :param paths: the include paths of the source code. + :return: the values of all macros that could be evaluated, which are cached. + """ + hash = hashlib.sha1(header.read_bytes() + str(_VERSION).encode() + " ".join(defines).encode()) + for name, (source, names) in sorted(sources.items()): + hash.update(" ".join([name, source, *sorted(names)]).encode()) + cache = _CACHE_PATH / f"{header.stem}_values_{hash.hexdigest()[:10]}.pkl" + if cache.exists(): + return pickle.loads(cache.read_bytes()) + values = {} + for name, (source, names) in sorted(sources.items()): + expressions = [(n, n) for n in sorted(set(names)) if n not in values] + try: + values.update(evaluate(header, defines, expressions, source, paths)) + except ValueError as error: + LOGGER.warning(f"Ignoring {name}: {str(error)[:2000]}") + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_bytes(pickle.dumps(values, protocol=pickle.HIGHEST_PROTOCOL)) + return values + + +def extract_header(header: Path, defines: list[str] = None) -> HeaderData: + """ + Extracts the macro values and structure layouts of a CMSIS header. + + :param header: path to the CMSIS device header. + :param defines: macros to define on the command line, e.g. the device define. + :return: the extracted header data, which is cached. + """ + defines = list(defines or []) + content = header.read_bytes() + key = hashlib.sha1(content + " ".join(defines).encode() + str(_VERSION).encode()).hexdigest()[:10] + cache = _CACHE_PATH / f"{header.stem}_{key}.pkl" + if cache.exists(): + return pickle.loads(cache.read_bytes()) + LOGGER.info(f"Extracting {header.name} with {defines}...") + data = _extract(header, defines) + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_bytes(pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)) + return data diff --git a/src/modm_data/header2svd/stmicro/__init__.py b/src/modm_data/header2svd/stmicro/__init__.py index beca289..0152178 100644 --- a/src/modm_data/header2svd/stmicro/__init__.py +++ b/src/modm_data/header2svd/stmicro/__init__.py @@ -3,9 +3,17 @@ from .header import Header, getDefineForDevice from .tree import normalize_memory_map +from .memory_map import Report, device_headers, header_defines, memory_map_from_header +from .compare import compare_svd, svd_for_header __all__ = [ "Header", "getDefineForDevice", "normalize_memory_map", + "Report", + "device_headers", + "header_defines", + "memory_map_from_header", + "compare_svd", + "svd_for_header", ] diff --git a/src/modm_data/header2svd/stmicro/__main__.py b/src/modm_data/header2svd/stmicro/__main__.py index 46b7b63..7e41c2b 100644 --- a/src/modm_data/header2svd/stmicro/__main__.py +++ b/src/modm_data/header2svd/stmicro/__main__.py @@ -2,76 +2,112 @@ # SPDX-License-Identifier: MPL-2.0 import re +import json import tqdm +import logging import argparse -import subprocess from pathlib import Path from collections import defaultdict from multiprocessing.pool import ThreadPool -import modm_data -from modm_data.header2svd.stmicro import Header, normalize_memory_map -from modm_data.svd import format_svd, write_svd -from modm_data.owl.stmicro import did_from_string +from modm_data.header2svd.stmicro import device_headers, memory_map_from_header, compare_svd, svd_for_header +from modm_data.svd import format_svd, write_svd, read_svd from modm_data.utils import ext_path -from anytree import RenderTree + + +def _format_report(report, differences) -> str: + assigned = 100 * (1 - len(report.unassigned) / max(report.defines, 1)) + lines = [f"{report.header}: {assigned:.1f}% of {report.defines} bit field macros assigned", ""] + groups = defaultdict(list) + for name in report.unassigned: + groups["_".join(name.split("_")[:2])].append(name.split("_", 2)[-1]) + lines.append(f"Unassigned bit field macros: {len(report.unassigned)}") + lines += [f" {group}: {' '.join(fields)}" for group, fields in sorted(groups.items())] + lines += ["", f"Registers without bit fields: {len(report.empty)}"] + lines += [f" {name}" for name in report.empty] + lines += ["", f"Overlapping bit fields: {len(report.overlapping)}"] + lines += [f" {register}.{removed} overlaps {remaining}" for register, removed, remaining in report.overlapping] + lines += ["", f"Alternate registers of overlapping bit fields: {len(report.alternates)}"] + lines += [f" {register}" for register in report.alternates] + lines += ["", f"Registers paired with bit field macros by CubeHAL: {len(set(report.hinted))}"] + lines += [f" {register}: {prefix}_*" for register, prefix in sorted(set(report.hinted))] + lines += ["", f"Bit fields not supported by the instance: {len(report.restricted)}"] + restricted = defaultdict(list) + for peripheral, register in report.restricted: + restricted[peripheral].append(register) + lines += [f" {peripheral}: {' '.join(registers)}" for peripheral, registers in restricted.items()] + lines += ["", f"Bit fields with enumerated values: {report.enumerations}"] + lines += ["", f"Unassigned interrupts: {len(report.interrupts)}"] + lines += [f" {interrupt}" for interrupt in report.interrupts] + if differences is not None: + lines += ["", f"Differences to ST SVD: {len(differences)}"] + lines += [f" {line}" for line in differences] + return "\n".join(lines) + "\n" + + +def _convert(job): + header, core, compare, output = job + device, report = memory_map_from_header(header, core) + output_path = output / f"header_{device.name}.svd" + write_svd(format_svd(device), str(output_path)) + differences = None + if compare and (svd_path := svd_for_header(header)) is not None: + differences = compare_svd(device, read_svd(svd_path)) + log_path = Path(f"log/stmicro/svd/header_{device.name}.txt") + log_path.write_text(_format_report(report, differences)) + return device.name, report.defines, len(report.unassigned) def main(): parser = argparse.ArgumentParser() - parser.add_argument("--device", type=str, default=[], action="append") - parser.add_argument("--all", type=str, default=[], action="append") + parser.add_argument( + "--header", + type=str, + default=[], + action="append", + help="Regex pattern of CMSIS device header names, e.g. stm32f407xx.", + ) + parser.add_argument("--all", action="store_true", default=False, help="Convert all CMSIS device headers.") + parser.add_argument("--compare", action="store_true", default=False, help="Compare with the ST SVD files.") + parser.add_argument( + "--output", + type=Path, + default=None, + help="Folder for the SVD files and their svd-files.json list, defaults to ext/stmicro/svd.", + ) + parser.add_argument("-v", "--verbose", action="count", default=0) args = parser.parse_args() + logging.basicConfig(level=[logging.WARNING, logging.INFO, logging.DEBUG][min(args.verbose, 2)]) - if args.all: - devices = modm_data.cubemx.devices() - filtered_devices = [d for d in devices if any(re.match(pat, d.string) for pat in args.all)] - - headers = defaultdict(list) - for device in reversed(filtered_devices): - header = Header(device) - headers[header.filename].append(device.string) - header_devices = list(headers.values()) - Path("log/stmicro/svd").mkdir(exist_ok=True, parents=True) - - calls = [] - for devices in header_devices: - call = ( - f"python3 -m modm_data.header2svd.stmicro " - f"--device {' --device '.join(devices)} " - f"> log/stmicro/svd/header_{list(sorted(devices))[0]}.txt 2>&1" - ) - calls.append(call) - # print(call) + output = args.output or ext_path("stmicro/svd") + output.mkdir(exist_ok=True, parents=True) + headers = [h for h in device_headers() if args.all or any(re.match(p, h.stem) for p in args.header)] + jobs = [] + for header in headers: + # Dual-core devices have a memory map for each core + if "CORE_CM4 or CORE_CM7" in header.read_text(encoding="utf-8", errors="replace"): + jobs += [(header, "cm7", args.compare, output), (header, "cm4", args.compare, output)] + else: + jobs.append((header, None, args.compare, output)) + if not jobs: + print("No matching CMSIS headers found!") + return False - with ThreadPool() as pool: - retvals = list(tqdm.tqdm(pool.imap(lambda c: subprocess.run(c, shell=True), calls), total=len(calls))) - for retval, call in zip(retvals, calls): - if retval.returncode != 0: - print(call) - return all(r.returncode == 0 for r in retvals) + Path("log/stmicro/svd").mkdir(exist_ok=True, parents=True) + with ThreadPool() as pool: + results = list(tqdm.tqdm(pool.imap_unordered(_convert, jobs), total=len(jobs), disable=len(jobs) < 5)) - mmaps = defaultdict(list) - headers = {} - # create one or multiple mmaps from device set - for device in args.device: - device = did_from_string(device) - header = Header(device) - print(device.string, header.filename) - mmaptree = header.memory_map_tree # create cache entry - mmaps[header._memory_map_key].append(device) - headers[header._memory_map_key] = header + if args.output: + # The SVD Explorer lists the files from this file on static hosts + files = sorted(path.name for path in output.glob("header_*.svd")) + (output / "svd-files.json").write_text(json.dumps(files, indent=0) + "\n") - # Create one SVD file for each memory map - for key, devices in mmaps.items(): - header = headers[key] - mmaptree = header._cache[key] - mmaptree.compatible = list(sorted(devices, key=lambda d: d.string)) - mmaptree = normalize_memory_map(mmaptree) - print(RenderTree(mmaptree, maxlevel=2)) - svd = format_svd(mmaptree) - output_path = ext_path(f"stmicro/svd/header_{mmaptree.compatible[0].string}.svd") - write_svd(svd, str(output_path)) + defines = sum(r[1] for r in results) + unassigned = sum(r[2] for r in results) + for name, count, missing in sorted(results): + print(f"{name:20} {100 * (1 - missing / max(count, 1)):5.1f}% of {count} bit field macros assigned") + if len(results) > 1: + print(f"{'Total':20} {100 * (1 - unassigned / max(defines, 1)):5.1f}% of {defines} bit field macros assigned") return True diff --git a/src/modm_data/header2svd/stmicro/compare.py b/src/modm_data/header2svd/stmicro/compare.py new file mode 100644 index 0000000..1b92fbb --- /dev/null +++ b/src/modm_data/header2svd/stmicro/compare.py @@ -0,0 +1,141 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# Comparison with ST SVD Files + +The memory map reconstructed from the CMSIS header is compared with the ST +SVD file to find discrepancies. Since the SVD files are not compiled, they are +considered less accurate than the headers, so the result is not a list of +errors in the reconstruction, but a list of differences that need to be +checked, for example, with the reference manual. +""" + +import re +from pathlib import Path +from collections import defaultdict +from ...svd import Device +from ...utils import ext_path + +_SVD_PATH = ext_path("stmicro/svd") + + +def svd_for_header(header: Path) -> Path | None: + """:return: the ST SVD file with the longest name pattern matching the header.""" + stem = header.stem.lower() + best = None + for svd in _SVD_PATH.glob("*/*.svd"): + pattern = re.sub(r"_cm\d+$", "", svd.stem.lower()).replace("x", ".") + if re.match(pattern, stem) and (best is None or len(svd.stem) > len(best.stem)): + best = svd + return best + + +def _normalize_register(peripheral: str, register: str) -> str: + """Removes the peripheral prefix and the alternate suffix of ST SVD register names.""" + register = re.sub(r"_(Output|Input|alternate\d*|Device|Host|ENABLED|DISABLED)$", "", register, flags=re.I) + if "_" in register: + head = register.split("_")[0] + if head.rstrip("0123456789").upper() in peripheral.upper(): + register = register[len(head) + 1 :] + return register + + +def _range(bits: tuple[int, int]) -> str: + return f"{bits[0] + bits[1] - 1}:{bits[0]}" + + +def compare_svd(header: Device, svd: Device) -> list[str]: + """ + :param header: the memory map reconstructed from the CMSIS header. + :param svd: the memory map read from the ST SVD file. + :return: a list of differences. + """ + lines = [] + derived = {p.name: p for p in header.children} + + def registers(peripheral): + while not peripheral.children and getattr(peripheral, "derived_from", None) in derived: + peripheral = derived[peripheral.derived_from] + return peripheral.children + + def normalize(name): + """Ignores the position of the instance number, e.g. GTZC1_TZIC and GTZC_TZIC1""" + secure = bool(re.search(r"(_S$|^SEC_)", name)) + name = re.sub(r"(_S$|^SEC_)", "", name) + return re.sub(r"[\d_]", "", name).upper(), "".join(re.findall(r"\d", name)), secure + + # ST SVDs call the secure instances SEC_* + hperipherals = {re.sub(r"^(.*)_S$", r"SEC_\1", p.name): p for p in header.children} + hnormalized = defaultdict(list) + for p in header.children: + hnormalized[normalize(p.name)].append(p) + haddresses = {p.address: p for p in header.children} + matched = set() + for speripheral in svd.children: + if speripheral.address >= 0xE0000000: + continue + hperipheral = hperipherals.get(speripheral.name) + if hperipheral is None and len(candidates := hnormalized[normalize(speripheral.name)]) == 1: + hperipheral = candidates[0] + if hperipheral is None: + hperipheral = haddresses.get(speripheral.address) + if hperipheral is None: + lines.append(f"{speripheral.name} @ 0x{speripheral.address:08x}: not defined in header") + continue + matched.add(hperipheral.name) + prefix = f"{speripheral.name}" + if hperipheral.name != speripheral.name: + prefix += f" ({hperipheral.name})" + if hperipheral.address != speripheral.address: + lines.append(f"{prefix}: address 0x{hperipheral.address:08x} != 0x{speripheral.address:08x}") + + # Alternate registers are merged, e.g. TIM_CCMR1 and TIM_CCMR1_ALT + halternates = defaultdict(list) + for hregister in registers(hperipheral): + halternates[hregister.offset].append(hregister) + # Alternate registers are merged, e.g. TIM_CCMR1_Output and TIM_CCMR1_Input + sregisters = defaultdict(list) + for sregister in speripheral.children: + sregisters[sregister.offset].append(sregister) + for offset, alternates in sregisters.items(): + snames = {_normalize_register(speripheral.name, r.name) for r in alternates} + sname = "/".join(sorted(snames)) + if not (hregisters := halternates.get(offset)): + lines.append(f"{prefix}.{sname} @ 0x{offset:03x}: not defined in header") + continue + hregister = hregisters[0] + rprefix = f"{prefix}.{hregister.name}" + if hregister.name.replace("[%s]", "") not in snames: + lines.append(f"{rprefix}: named {sname} in SVD") + if hregister.width not in {r.width for r in alternates}: + lines.append(f"{rprefix}: size {hregister.width} != {alternates[0].width} in SVD") + hfields = {(f.position, f.width): f.name for r in reversed(hregisters) for f in r.children} + sfields = defaultdict(set) + for sregister in alternates: + for f in sregister.children: + sfields[(f.position, f.width)].add(f.name) + hnames = {name: bits for bits, name in hfields.items()} + snames = {name: bits for bits, names in sfields.items() for name in names} + for bits, names in sorted(sfields.items()): + for name in sorted(names): + if bits in hfields: + if hfields[bits] not in names: + lines.append(f"{rprefix}.{hfields[bits]}: named {name} in SVD") + break + elif name in hnames and hnames[name] not in sfields: + lines.append(f"{rprefix}.{name}[{_range(hnames[name])}]: located at [{_range(bits)}] in SVD") + else: + lines.append(f"{rprefix}.{name}[{_range(bits)}]: not defined in header") + for bits, name in sorted(hfields.items()): + if bits not in sfields and not (name in snames and snames[name] not in hfields): + lines.append(f"{rprefix}.{name}[{_range(bits)}]: not defined in SVD") + if speripheral.children: + for hregister in registers(hperipheral): + if hregister.offset not in sregisters and not getattr(hregister, "alternate", None): + lines.append(f"{prefix}.{hregister.name} @ 0x{hregister.offset:03x}: not defined in SVD") + + for hperipheral in header.children: + if hperipheral.name not in matched and hperipheral.address < 0xE0000000: + lines.append(f"{hperipheral.name} @ 0x{hperipheral.address:08x}: not defined in SVD") + return lines diff --git a/src/modm_data/header2svd/stmicro/cubehal.py b/src/modm_data/header2svd/stmicro/cubehal.py new file mode 100644 index 0000000..171fd3e --- /dev/null +++ b/src/modm_data/header2svd/stmicro/cubehal.py @@ -0,0 +1,195 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# CubeHAL Information + +The CubeHAL source code complements the CMSIS header with information that +cannot be derived from the header alone: + +- The register accesses pair the register members with their bit field macros, + which is used when the naming heuristics fail, for example, `FW_CR_FPA` for + `FIREWALL_TypeDef::CR`, or when a register reuses the bit field macros of + another register, for example, `OPAMP_OTR_TRIMOFFSETN` for `LPOTR`. +- The LL functions document the instances that support a feature with the + `IS_*_INSTANCE` macros, which restricts the bit fields of the shared + structure types per instance. +- The LL functions document the enumerated values of the bit fields. +""" + +import re +import os +import logging +from pathlib import Path +from collections import defaultdict, Counter + +from ..extract import HeaderData, extract_values +from ...cubehal.registers import register_accesses, ll_functions, ll_descriptions + +LOGGER = logging.getLogger(__name__) + + +def _strip_security(name: str) -> str: + return re.sub(r"_(NS|S)$", "", name) + + +def _normalize(name: str) -> str: + return name.replace("_", "").upper() + + +def _field_macro(identifier: str, bitfields) -> str | None: + """:return: the bit field macro of a mask identifier resolving aliases.""" + macro = re.sub(r"_(Pos|Msk)$", "", identifier) + if macro not in bitfields.fields: + return None + return bitfields.alias.get(macro, macro) + + +def _member_types(data: HeaderData) -> dict[str, set[str]]: + types = defaultdict(set) + for typedef, members in data.structs.items(): + for member in members: + types[member.name].add(typedef) + return types + + +def register_hints(data: HeaderData, bitfields, path: Path) -> dict[tuple[str, str], str]: + """ + Pairs the register members with the prefix of the bit field macros that + are used together in the CubeHAL source code. + + :return: the bit field macro prefix by (typedef, member), e.g. `FW_CR`. + """ + member_types = _member_types(data) + instance_types = {_strip_security(name): data.aliases.get(t, t) for name, (t, _) in data.instances.items()} + macros = defaultdict(set) + for access in register_accesses(path): + types = {data.aliases.get(t, t) for t in access.types} + if access.variable in instance_types: + types.add(instance_types[access.variable]) + types &= member_types.get(access.member, set()) + # Only use unambiguous accesses + if len(types) != 1: + continue + typedef = types.pop() + for identifier in access.masks: + macro = re.sub(r"_(Pos|Msk)$", "", identifier) + if macro in bitfields.local: + macros[(typedef, access.member)].add(macro) + + hints = {} + for (typedef, member), names in macros.items(): + scores = Counter() + for name in names: + parts = name.split("_") + for index in range(2, len(parts)): + scores["_".join(parts[:index])] += 1 + + def score(prefix): + token = prefix.split("_", 1)[1] + return (_normalize(token) == _normalize(member), scores[prefix], -len(prefix)) + + if scores: + hints[(typedef, member)] = max(scores, key=score) + return hints + + +def instance_restrictions(data: HeaderData, bitfields, path: Path): + """ + Derives the instances that support a bit field from the `IS_*_INSTANCE` + macros documented in the LL functions. A bit field that is used by at least + one function without such a note is supported by all instances. + + :return: (instances by (typedef, member, macro or field name), 32-bit + counter instances by (typedef, member, macro or field name)) + """ + instances = {name: {_strip_security(i) for i in identifiers} for name, identifiers in data.instance_macros.items()} + gates = defaultdict(list) + counters = defaultdict(set) + for function in ll_functions(path): + if not function.typedef or function.typedef not in data.structs: + continue + # The remap feature is not supported by all instances with the remap registers, e.g. TIM15_TISEL + macros = {m for m in function.instance_macros if m in instances and m != "IS_TIM_REMAP_INSTANCE"} + counter = {m for m in macros if "32B_COUNTER" in m} + macros -= counter + gate = set().union(*(instances[m] for m in macros)) if macros else None + counter = set().union(*(instances[m] for m in counter)) if counter else None + for access in function.accesses: + keys = {_field_macro(i, bitfields) for i in access.masks} - {None} + if not access.masks: + # Registers written as a whole are documented with the field names + keys = {f for r, f in function.rmtoll if re.sub(r"^\w+_", "", r) == access.member} + for key in keys: + gates[(function.typedef, access.member, key)].append(gate) + if counter is not None: + counters[(function.typedef, access.member, key)] |= counter + restrictions = {} + for key, usages in gates.items(): + if None not in usages: + restrictions[key] = set().union(*usages) + return restrictions, dict(counters) + + +def _enumeration_name(names: list[str]) -> dict[str, str]: + """:return: the shortest unique names without the common prefix, e.g. LL_ADC_RESOLUTION_12B -> RESOLUTION_12B.""" + prefix = os.path.commonprefix([name + "_" for name in names]) + prefix = prefix[: prefix.rfind("_") + 1] + result = {} + for name in names: + suffix = name[len(prefix) :] + if not suffix or not suffix[0].isalpha(): + head = prefix[:-1].rsplit("_", 1)[-1] + suffix = f"{head}_{suffix}" if suffix else head + result[name] = suffix + return result + + +def enumerations(header: Path, data: HeaderData, bitfields, path: Path) -> dict[str, list[tuple[str, int, str]]]: + """ + Evaluates the enumerated values of LL functions that access a single bit field. + + :return: the (name, value, description) of the enumerated values by bit field macro. + """ + candidates = [] + for function in ll_functions(path): + accesses = [a for a in function.accesses if a.kind in ("MODIFY_REG", "READ_BIT")] + if len(function.values) < 2 or len(accesses) != 1: + continue + macros = {_field_macro(i, bitfields) for i in accesses[0].masks} - {None} + if len(macros) != 1 or len(bitfields.fields[(macro := macros.pop())]) != 1: + continue + candidates.append((macro, function)) + if not candidates: + return {} + + # Only the preprocessor directives of the LL headers are compiled, since the + # inline functions do not always compile outside of their intended context + names = defaultdict(set) + for _, function in candidates: + names[function.file].update(function.values) + sources = {} + for file, values in names.items(): + text = (path / "Inc" / file).read_text(encoding="utf-8", errors="replace") + text = re.sub(r"\\\s*\n", " ", re.sub(r"/\*.*?\*/", "", text, flags=re.S)) + source = "\n".join(line for line in text.splitlines() if line.lstrip().startswith("#")) + sources[file] = (source, sorted(values & set(re.findall(r"#\s*define\s+(LL_\w+)", source)))) + values = extract_values(header, data.defines, sources, [path / "Inc"]) + descriptions = ll_descriptions(path) + + result = {} + for macro, function in candidates: + position, width, _ = bitfields.fields[macro][0] + mask = ((1 << width) - 1) << position + evaluated = {name: values[name] for name in function.values if name in values} + # LL values sometimes encode additional information outside of the bit field + if len(evaluated) < 2 or any(value & ~mask for value in evaluated.values()): + continue + enumeration = result.setdefault(macro, []) + seen = {v for _, v, _ in enumeration} + shortnames = _enumeration_name(list(evaluated)) + for name, value in evaluated.items(): + if (value := value >> position) not in seen: + seen.add(value) + enumeration.append((shortnames[name], value, descriptions.get(name, ""))) + return result diff --git a/src/modm_data/header2svd/stmicro/header.py b/src/modm_data/header2svd/stmicro/header.py index e138371..dd4b6bc 100644 --- a/src/modm_data/header2svd/stmicro/header.py +++ b/src/modm_data/header2svd/stmicro/header.py @@ -3,34 +3,14 @@ import re import logging -import subprocess from collections import defaultdict from ..header import Header as CmsisHeader -from ...utils import ext_path, cache_path -import modm_data.svd as svd +from ...utils import ext_path LOGGER = logging.getLogger(__file__) -HEADER_TEMPLATE = r""" -#include -#include <{{header}}> - -template -void __modm_dump_f(char const *symbol, T value) { - std::cout << "\"" << symbol << "\": " << uint64_t(value) << "," << std::endl; -} -#define __modm_dump(def) __modm_dump_f(#def, (def)) -int main() { - std::cout << "cpp_defines = {";{% for define in defines %} -#ifdef {{define}} - __modm_dump({{define}}); -#endif{% endfor %} - std::cout << "}"; - return 0; -} -""" def getDefineForDevice(device_id, familyDefines): @@ -66,17 +46,7 @@ def getDefineForDevice(device_id, familyDefines): class Header(CmsisHeader): _HEADER_PATH = ext_path("stmicro/header") - _CACHE_PATH = cache_path("cmsis/stm32") - _CACHE_HEADER = defaultdict(dict) _CACHE_FAMILY = defaultdict(dict) - _BUILTINS = { - "const uint32_t": 4, - "const uint16_t": 2, - "const uint8_t": 1, - "uint32_t": 4, - "uint16_t": 2, - "uint8_t": 1, - } def __init__(self, did, family_header_file, define): self.did = did @@ -105,33 +75,6 @@ def __init__(self, did, family_header_file, define): } super().__init__(self.cmsis_folder / self.header_file, substitutions) - def get_defines(self): - key = "defines" + self.did.get("core", "") - if key not in self._cache: - self._cache[key] = self._get_defines() - return self._cache[key] - - @property - def _memory_map_key(self): - return "memmap" + self.did.get("core", "") - - @property - def memory_map_tree(self): - if self._memory_map_key not in self._cache: - self._cache[self._memory_map_key] = self._get_memmap() - return self._cache[self._memory_map_key] - - @property - def get_memory_sizes(self): - if "memsizes" not in Header._CACHE_HEADER[self.header_file]: - sizes = { - m.group(1): int(m.group(2), 0) - for d in self.header.defines - if (m := re.match(r"(\w+)_SIZE +\((0x.+?)UL\)", d)) - } - Header._CACHE_HEADER[self.header_file]["memsizes"] = sizes - return Header._CACHE_HEADER[self.header_file]["memsizes"] - @property def interrupt_table(self): if "vectors" not in self._cache: @@ -151,243 +94,3 @@ def _get_family_defines(self): defines.append(define.group(1)) Header._CACHE_FAMILY[self.family_folder]["family_defines"] = defines return Header._CACHE_FAMILY[self.family_folder]["family_defines"] - - def _get_filtered_defines(self): - defines = {} - # get all the non-empty defines - for define in self.header.defines: - if comment := re.search(r"/\* *(.*?) *\*/", define): - if "legacy" in comment.group(1): - continue - define = re.sub(r"/\*.*?\*/", "", define).strip() - name, *parts = define.split(" ") - if not len(parts): - continue - if any(i in name for i in ["("]): - continue - if any(name.endswith(i) for i in ["_IRQn", "_IRQHandler", "_SUPPORT", "_TypeDef"]): - continue - if any(name.startswith(i) for i in ["IS_"]): - continue - if name in [ - "FLASH_SIZE", - "FLASH_BANK_SIZE", - "COMP1_COMMON", - "COMP12_COMMON_BASE", - "OPAMP12_COMMON_BASE", - "BL_ID", - ]: - continue - defines[name] = "".join(parts[1:]).strip() - return defines - - def _get_defines(self): - from jinja2 import Environment - - # create the destination directory - destination = (Header._CACHE_PATH / self.family_folder / self.header_file).with_suffix(".cpp").absolute() - core = self.did.get("core", "") - executable = destination.with_suffix("." + core if core else "") - defines = self._get_filtered_defines() - if not executable.exists(): - # generate the cpp file from the template - LOGGER.info(f"Generating {destination.name} ...") - substitutions = {"header": self.header_file, "defines": sorted(defines)} - content = Environment().from_string(HEADER_TEMPLATE).render(substitutions) - # write the cpp file into the cache - destination.parent.mkdir(exist_ok=True, parents=True) - destination.write_text(content) - header_defines = [self.define] - if core: - header_defines.append(f"CORE_C{core.upper()}=1") - # compile file into an executable - includes = [str(Header.CMSIS_PATH.absolute()), str(self.cmsis_folder.absolute())] - gcc_command = [ - "g++", - "-Wno-narrowing", - "-fms-extensions", - " ".join(f"-I{incl}" for incl in includes), - " ".join(f"-D{define}" for define in header_defines), - f"-o {executable}", - str(destination), - ] - LOGGER.info(f"Compiling {destination.name} ...") - retval = subprocess.run(" ".join(gcc_command), shell=True) - if retval.returncode: - LOGGER.error(f"Header compilation failed! {retval}") - return None - # execute the file - LOGGER.info(f"Running {executable.name} ...") - retval = subprocess.run([str(executable)], stdout=subprocess.PIPE) - if retval.returncode: - LOGGER.error(f"Header execution failed! {retval}") - return None - # parse the printed values - localv = {} - exec(retval.stdout, globals(), localv) - undefined = [d for d in defines if d not in localv["cpp_defines"]] - if len(undefined): - LOGGER.warning(f"Undefined macros: {undefined}") - return localv["cpp_defines"] - - def _get_memmap(self): - # get the values of the definitions in this file - defines = self.get_defines() - - # get the mapping of peripheral to its type - peripheral_map = {} - seen_defines = [] - for name, value in self._get_filtered_defines().items(): - if "*)" in value: - values = value.split("*)") - typedef = values[0].strip()[2:].strip() - peripheral_map[name] = (typedef, defines[name]) - LOGGER.debug(f"Found peripheral ({typedef} *) {name} @ 0x{defines[name]:x}") - - # build the array containing the peripheral types - raw_types = { - typedef: [ - ( - v["type"], - v["name"], - int(v["array_size"], 16 if v["array_size"].startswith("0x") else 10) if v["array"] else 0, - ) - for v in values["properties"]["public"] - ] - for typedef, values in self.header.classes.items() - } - - # function to recursively flatten the types - def _flatten_type(typedef, result, prefix=""): - for t, n, s in raw_types[typedef]: - if t in Header._BUILTINS: - size = Header._BUILTINS[t] - name = None if n.upper().startswith("RESERVED") else n - if s == 0: - result.append((size, (prefix + name) if name else name)) - else: - if not name: - result.append((size * s, name)) - else: - result.extend([(size, f"{prefix}{name}.{ii}") for ii in range(s)]) - elif t in raw_types.keys(): - if s == 0: - _flatten_type(t, result, prefix) - else: - for ii in range(s): - _flatten_type(t, result, prefix + f"{n}.{ii}.") - else: - LOGGER.error(f"Unknown type: {t} ({n} {s})") - exit(1) - - # flatten all types - flat_types = defaultdict(list) - for typedef in raw_types: - _flatten_type(typedef, flat_types[typedef]) - - # match the macro definitions to the type structures - matched_types = defaultdict(list) - for typedef, pregs in flat_types.items(): - # print(typedef, pregs) - peri = "_".join([t for t in typedef.split("_") if t.isupper()]) - position = 0 - for reg in pregs: - if reg[1] is None: - position += reg[0] - continue - sreg = [r for r in reg[1].split(".") if r.isupper() or r.isdigit()] - prefix = ["{}_{}_".format(peri, "".join([r for r in sreg if r.isupper()]))] - if len(sreg) > 1: - if sreg[0].isdigit(): - parts = sreg[1].split("R") - parts[-2] += sreg[0] - prefix.append("{}_{}_".format(peri, "R".join(parts))) - elif sreg[1].isdigit(): - sreg[1] = str(int(sreg[1]) + 1) - prefix.append("{}_{}_".format(peri, "".join([r for r in sreg if r.isupper()]) + sreg[1])) - prefix.append("{}_{}x_".format(peri, "".join([r for r in sreg if r.isupper()]))) - # A bunch of aliases - if "FSMC_BTCR_" in prefix: - prefix.extend(["FSMC_BCRx_", "FSMC_BTRx_"]) - if "ADC_TR_" in prefix: - prefix.append("ADC_TR1_") - if "DBGMCU_APB1FZ_" in prefix: - prefix.append("DBGMCU_APB1_FZ_") - if "DBGMCU_APB2FZ_" in prefix: - prefix.append("DBGMCU_APB2_FZ_") - # if "FLASH_KEYR_" in prefix: prefix.extend(["FLASH_KEY1_", "FLASH_KEY2_"]) - # if "FLASH_OPTKEYR_" in prefix: prefix.extend(["FLASH_OPTKEY1_", "FLASH_OPTKEY2_"]) - if "GPIO_AFR1_" in prefix: - prefix.append("GPIO_AFRL_") - if "GPIO_AFR2_" in prefix: - prefix.append("GPIO_AFRH_") - if "SAI_Block" in typedef: - prefix = [p.replace("SAI_", "SAI_x") for p in prefix] - - regmap = {} - for p in prefix: - keys = [d for d in defines.keys() if d.startswith(p)] - seen_defines.extend(keys) - regmap.update({k.replace(p, ""): defines[k] for k in keys}) - if not len(regmap): - LOGGER.info(f"Empty: {typedef:30} {peri}->{prefix} ({reg[1]} >> {sreg})") - - # convert macro names to positional arguments - fields = sorted(list(set([r[:-4] for r in regmap if r.endswith("_Pos")]))) - registers = {} - for field in fields: - regs = {k: v for k, v in regmap.items() if k == field or k.startswith(field + "_")} - val = regs.pop(field, None) - pos = regs.pop(field + "_Pos", None) - msk = regs.pop(field + "_Msk", None) - if val is None: - LOGGER.warning(f"{field} not found: {regs}") - continue - if pos is None: - LOGGER.warning(f"{field}_Pos not found: {regs}") - continue - if msk is None: - LOGGER.warning(f"{field}_Msk not found: {regs}") - continue - - rem = {k.replace(field + "_", ""): v for k, v in regs.items()} - mask = msk >> pos - width = 0 - while mask: - width += 1 - mask >>= 1 - registers[pos] = (field, width, msk, val, rem) - - # print(registers) - # Store in map - matched_types[typedef].append((position, reg[0], reg[1], registers)) - position += reg[0] - - # print the remaining - remaining_defines = [d for d in defines if d not in seen_defines and not d.endswith("_BASE")] - for typedef in matched_types: - peri = "_".join([t for t in typedef.split("_") if t.isupper()]) + "_" - rem = [d for d in remaining_defines if d.startswith(peri)] - if len(rem): - LOGGER.warning(f"Unassigned defines for ({typedef} *) {peri}: {len(rem)}") - for d in rem: - LOGGER.info(f"{d}: {defines[d]}") - - # for typedef, registers in matched_types.items(): - # print(typedef) - # for reg in registers: - # print(f" {reg[0]:03x}: {reg[2]}") - - device = svd.Device(self.did.string) - for name, (typedef, address) in peripheral_map.items(): - svd.Peripheral(name, typedef, defines[name], parent=device) - - for name, registers in matched_types.items(): - peripheral = svd.PeripheralType(name, parent=device) - for offset, width, name, bitfields in registers: - register = svd.Register(name, offset, width, parent=peripheral) - for pos, (name, width, mask, value, _) in bitfields.items(): - svd.BitField(name, pos, width, parent=register) - - return device - # return (peripheral_map, matched_types) diff --git a/src/modm_data/header2svd/stmicro/memory_map.py b/src/modm_data/header2svd/stmicro/memory_map.py new file mode 100644 index 0000000..58e9621 --- /dev/null +++ b/src/modm_data/header2svd/stmicro/memory_map.py @@ -0,0 +1,834 @@ +# Copyright 2025, Niklas Hauser +# SPDX-License-Identifier: MPL-2.0 + +""" +# STM32 Memory Map from CMSIS Header + +The memory map is reconstructed from the CMSIS device header, which is +compiled and therefore more accurate than the ST SVD files: + +1. The peripheral instances and their addresses are the pointer macros, e.g. + `#define USART1 ((USART_TypeDef *) USART1_BASE)`. +2. The registers are the members of the `*_TypeDef` structure with their + offset and size computed by the compiler. +3. The bit fields are the `{HEAD}_{REGISTER}_{FIELD}_Pos` and `_Msk` macros. + +The difficult part is matching the bit field macros to the structure members, +since the naming is inconsistent between the peripherals and families: + +- The head is the peripheral type (`USART_CR1_UE`), the instance + (`ADC4_SMPR_SMP1`, `OPAMP1_CSR_OPAMP1EN`) or the parent type for + sub-instances (`DFSDM_CHCFGR1_CHEN` for `DFSDM1_Channel0`). +- Array members use explicit indices (`CAN_F0R1`, `SYSCFG_EXTICR1`), indices + inside the name (`CAN_TI0R` for `sTxMailBox[0].TIR`) or placeholders + (`DMA_SxCR`, `SAI_xCR1`, `TSC_IOGXCR`, `GFXMMU_LUTxL`). +- Some macros differ from the member name (`DBGMCU_APB1_FZ` for `APB1FZ`, + `DCMI_RIS` for `RISR`, `FLASH_CR` for `NSCR` and `SECCR`). + +For each member, a list of candidate tokens is generated in order of +preference, and the first one with bit field macros is used together with the +register name derived from it. Instance-specific heads take precedence over +the type heads. + +The USB OTG device and host blocks have no instance macros, so they are added +from the `USB_OTG_*_BASE` offset macros. +""" + +import os +import re +import bisect +import logging +from pathlib import Path +from collections import defaultdict +from dataclasses import dataclass, field + +from . import cubehal +from ..extract import HeaderData, extract_header +from ...svd import Device, Peripheral, Register, BitField, EnumeratedValue +from ...cubehal.registers import folder as cubehal_folder +from ...utils import ext_path + +LOGGER = logging.getLogger(__name__) + +_HEADER_PATH = ext_path("stmicro/header") +_BLOCK_SIZE = 0x400 +# Arrays without bit fields and more elements are described as one register with dimension +_MAX_ARRAY_REGISTERS = 64 + +# (sub-instance, typedef, base macro, count macros, size macro) +_USB_OTG_BLOCKS = ( + ("Device", "USB_OTG_DeviceTypeDef", "USB_OTG_DEVICE_BASE", None, None), + ( + "INEndpoint", + "USB_OTG_INEndpointTypeDef", + "USB_OTG_IN_ENDPOINT_BASE", + ("MAX_IN_ENDPOINTS", "EP_NBR"), + "USB_OTG_EP_REG_SIZE", + ), + ( + "OUTEndpoint", + "USB_OTG_OUTEndpointTypeDef", + "USB_OTG_OUT_ENDPOINT_BASE", + ("MAX_OUT_ENDPOINTS", "EP_NBR"), + "USB_OTG_EP_REG_SIZE", + ), + ("Host", "USB_OTG_HostTypeDef", "USB_OTG_HOST_BASE", None, None), + ( + "HostChannel", + "USB_OTG_HostChannelTypeDef", + "USB_OTG_HOST_CHANNEL_BASE", + ("HOST_MAX_CHANNEL_NBR", "CH_NBR"), + "USB_OTG_HOST_CHANNEL_SIZE", + ), +) +_USB_OTG_REGISTERS = (("HPRT", "USB_OTG_HOST_PORT_BASE"), ("PCGCCTL", "USB_OTG_PCGCCTL_BASE")) + + +@dataclass +class Report: + """Discrepancies found in the header during the reconstruction.""" + + header: str + defines: int = 0 + """Number of bit field macros defined in the device header.""" + unassigned: list[str] = field(default_factory=list) + """Bit field macros that are not assigned to any register.""" + empty: list[str] = field(default_factory=list) + """Registers without any bit field macros.""" + overlapping: list[tuple[str, str, str]] = field(default_factory=list) + """(register, removed bit field, remaining bit field) with overlapping bits.""" + renamed: list[tuple[str, str]] = field(default_factory=list) + """(peripheral, register) that were renamed due to name collisions.""" + hinted: list[tuple[str, str]] = field(default_factory=list) + """(register, bit field macro prefix) that were paired by the CubeHAL source code.""" + restricted: list[tuple[str, str]] = field(default_factory=list) + """(peripheral, register.field) that are not supported by the instance.""" + alternates: list[str] = field(default_factory=list) + """Registers that were split into alternate registers due to overlapping bit fields.""" + interrupts: list[str] = field(default_factory=list) + """Interrupts that could not be assigned to a peripheral.""" + enumerations: int = 0 + """Number of bit fields with enumerated values.""" + + +@dataclass +class _Register: + name: str + offset: int + size: int + fields: dict[str, tuple[int, int]] + sub: str | None = None + dim: int = 0 + type: str = "" + member: str = "" + macros: dict[str, str] = field(default_factory=dict) + """The bit field macro of each bit field.""" + + +def device_headers() -> list[Path]: + """:return: all STM32 device headers in the CMSIS header repository.""" + headers = [] + for header in sorted(_HEADER_PATH.glob("stm32*xx/Include/stm32*.h")): + content = header.read_text(encoding="utf-8", errors="replace") + if "_TypeDef" in content and re.search(r'#include +"core_cm', content): + headers.append(header) + return headers + + +def header_defines(header: Path, core: str = None) -> list[str]: + """:return: the device define from the family header and the core define for dual-core devices.""" + define = None + for family in header.parent.glob("stm32*.h"): + content = family.read_text(encoding="utf-8", errors="replace") + if match := re.search(rf'#include +"{re.escape(header.name)}"', content): + # The define is checked right before the include + lines = content[: match.start()].splitlines()[-3:] + if defines := re.findall(r"defined *\( *(STM32\w+) *\)", "\n".join(lines)): + define = defines[-1] + break + defines = [define or (header.stem[:9].upper() + header.stem[9:])] + if "CORE_CM4 or CORE_CM7" in header.read_text(encoding="utf-8", errors="replace"): + defines.append(f"CORE_{(core or 'cm7').upper()}") + return defines + + +def _strip_security(name: str) -> str: + return re.sub(r"_(NS|S)$", "", name) + + +class _BitFields: + """Bit field macros with a fast prefix search.""" + + def __init__(self, data: HeaderData): + self.fields = {} + self.order = {} + self.alias = {} + for name, position in data.values.items(): + if not name.endswith("_Pos") or (mask := data.values.get(f"{name[:-4]}_Msk")) is None: + continue + name = name[:-4] + self.fields[name] = _runs((mask & 0xFFFFFFFF) >> position, position) + self.order[name] = data.local_macros.get(f"{name}_Pos", len(data.local_macros)) + # Aliases of other bit field macros, e.g. OCTOSPI_CR_EN_Pos XSPI_CR_EN_Pos + value = data.macros.get(f"{name}_Pos", "").strip("() \t") + if value.endswith("_Pos") and data.values.get(f"{value[:-4]}_Msk") is not None: + self.alias[name] = value[:-4] + self.local = {name for name in self.fields if f"{name}_Pos" in data.local_macros} + self.names = sorted(self.fields) + self.assigned = set() + + def _with_prefix(self, prefix: str) -> list[str]: + index = bisect.bisect_left(self.names, prefix) + result = [] + while index < len(self.names) and self.names[index].startswith(prefix): + result.append(self.names[index]) + index += 1 + return result + + def has(self, head: str, token: str) -> bool: + return f"{head}_{token}" in self.fields or bool(self._with_prefix(f"{head}_{token}_")) + + def take(self, head: str, token: str) -> dict[str, str]: + """:return: the bit fields of a register by name, which are marked as assigned.""" + prefix = f"{head}_{token}_" + names = self._with_prefix(prefix) + if f"{head}_{token}" in self.fields: + names.append(f"{head}_{token}") + self.assigned.update(names) + return {name[len(prefix) :] if name.startswith(prefix) else token: name for name in names} + + def take_prefix(self, prefix: str) -> dict[str, str]: + """:return: the bit fields of a register with the macro prefix, e.g. `FW_CR`.""" + head, token = prefix.split("_", 1) + return self.take(head, token) + + def unassigned(self) -> list[str]: + assigned = set(self.assigned) + for name, target in self.alias.items(): + if name in assigned or target in assigned: + assigned.update((name, target)) + return sorted(name for name in self.local if name not in assigned) + + +def _runs(mask: int, position: int) -> list[tuple[int, int, int]]: + """:return: the contiguous bit runs of a mask as (position, width, index of first bit).""" + runs, index = [], 0 + while mask: + while not mask & 1: + mask >>= 1 + position += 1 + width = 0 + while mask & 1: + mask >>= 1 + width += 1 + runs.append((position, width, index)) + position += width + index += width + return runs + + +def _type_heads(typedef: str, instance: str = None) -> list[str]: + name = re.sub(r"_?TypeDef$", "", typedef) + parts = name.split("_") + heads = ["_".join(parts[:i]) for i in range(len(parts), 0, -1)] + heads += [head.upper() for head in heads if head.upper() != head] + if instance: + instance = _strip_security(instance) + heads.append(instance) + heads.append(re.sub(r"\d+$", "", instance)) + heads.append(re.sub(r"\d+$", "", instance.split("_")[0])) + return list(dict.fromkeys(heads)) + + +def _instance_heads(instance: str) -> list[str]: + """Heads of the instance itself, which are more specific than the type, e.g. ADC4_SMPR or OPAMP1_CSR.""" + if not instance: + return [] + instance = _strip_security(instance) + return list(dict.fromkeys([instance, re.sub(r"\d+$", "", instance)])) + + +def _underscores(token: str) -> list[str]: + """Variants with an underscore at a digit boundary, e.g. APB1FZ -> APB1_FZ.""" + return [ + token[:p] + "_" + token[p:] + for p in range(1, len(token)) + if token[p - 1].isdigit() != token[p].isdigit() and "_" not in token[p - 1 : p + 1] + ] + + +def _indexed(member: str, index: int) -> list[str]: + tokens = [f"{member[:-1]}{index}R"] if member.endswith("R") else [] + tokens.append(f"{member}{index}") + tokens += [member[:p] + str(index) + member[p:] for p in range(1, len(member))] + return tokens + + +def _tokens(member: str, index: int = None, subtype: str = None, array_base: int = 0) -> list[tuple[str, str]]: + """:return: ordered candidates of (register token, kind) for a structure member.""" + m = member + tokens = [(m, "exact")] + indices = () + if index is not None: + # Pairs of registers with low and high part, e.g. GFXMMU_LUTxL and GFXMMU_LUTxH + tokens.append((f"{m}x{'LH'[index % 2]}", "lut")) + # FSMC and FMC interleave the chip-select and timing registers + if m in ("BTCR", "BWTR"): + reg = "BCR" if m == "BTCR" and index % 2 == 0 else "BTR" if m == "BTCR" else "BWTR" + return [(f"{reg}{index // 2 + 1}", "indexed"), (f"{reg}x", "fsmc")] + indices = (index, index + 1) if array_base == 0 else (index + 1,) + for idx in indices: + tokens += [(token, "indexed") for token in _indexed(m, idx)] + if index < 2: + tokens.append((f"{m}{'LH'[index]}", "indexed")) + tokens += [(m[:p] + "x" + m[p:], "x") for p in range(len(m) + 1)] + tokens += [(m[:p] + "X" + m[p:], "X") for p in range(len(m) + 1)] + if subtype and (letter := subtype.split("_")[-1][:1].upper()): + tokens.append((f"{letter}x{m}", "x")) + # Register suffix omitted, e.g. DMA_SxNDT for NDTR + if m.endswith("R"): + tokens.append((f"{letter}x{m[:-1]}", "xR")) + tokens.append((re.sub(r"\d+$", "", m), "reduced")) + tokens.append((re.sub(r"\d", "x", m), "reduced")) + if "_" in m: + tokens.append((m.split("_")[0], "reduced")) + # Member names with placeholder, e.g. HRTIM_TIMCR for TIMxCR + if "x" in m: + tokens.append((m.replace("x", ""), "reduced")) + # Digits inside the name, e.g. ETH_MACL3L4CR for MACL3L4C0R and USB_CHEP for CHEP0R + tokens += [ + (m[:p] + m[p + 1 :], "reduced") for p in range(1, len(m) - 1) if m[p].isdigit() and not m[p + 1].isdigit() + ] + tokens.append((re.sub(r"\d+R$", "", m), "reduced")) + # Register suffix omitted, e.g. DCMI_RIS for RISR + if m.endswith("R"): + tokens.append((m[:-1], "reduced")) + # Generic definitions for secure and non-secure registers, e.g. FLASH_CR for NSCR and SECCR + if re.match(r"(NS|SEC)[A-Z]", m): + tokens.append((re.sub(r"^(NS|SEC)", "", m), "reduced")) + # Underscores at different positions, e.g. DBGMCU_APB1_FZ and SYSCFG_ITLINE0_SR + squashed = m.replace("_", "") + for idx in indices: + tokens += [(u, "indexed") for token in _indexed(squashed, idx) for u in _underscores(token)] + tokens += [(u, "reduced") for u in _underscores(squashed)] + return list(dict.fromkeys((token, kind) for token, kind in tokens if token)) + + +def _abbreviation(members: list[str], sub: str) -> str: + """:return: the common member prefix that abbreviates the sub-instance, e.g. FLT for Filter or CH for Channel.""" + members = [m for m in members if not m.upper().startswith("RESERVED")] + if not sub or len(members) < 2: + return "" + prefix = os.path.commonprefix(members) + consonants = "".join(c for c in sub.upper() if c not in "AEIOU") + if len(prefix) < 2 or prefix[0] != sub[0].upper() or not re.match(".*?".join(prefix), consonants): + return "" + return prefix + + +def _name(kind: str, token: str, member: str, index: int, subindex: str, abbreviation: str) -> str: + idx = subindex if subindex is not None else index + if kind == "fsmc": + return token.replace("x", str(index // 2 + 1), 1) + if kind == "lut": + return f"{token[:-2]}{index // 2}{token[-1]}" + if kind == "indexed": + return token + if kind == "X" or (kind == "exact" and "X" in token and index is not None and subindex is None): + if idx is None: + return token + return token.replace("X", str(idx + 1) if subindex is None else str(idx), 1) + if kind == "x" and idx is not None: + return token.replace("x", str(idx), 1) + if kind == "xR" and idx is not None: + return token.replace("x", str(idx), 1) + "R" + # The exact or reduced tokens use the member name + if idx is None: + return member + if abbreviation and subindex is not None and member.startswith(abbreviation) and member != abbreviation: + return f"{abbreviation}{idx}{member[len(abbreviation) :]}" + if "x" in member: + return member.replace("x", str(idx), 1) + # The member name already contains the index, e.g. FSMC_Bank4->PCR4 + if subindex is not None and member.endswith(str(idx)): + return member + if kind == "reduced" and index is not None and subindex is None and token == re.sub(r"\d+$", "", member): + return token + return f"{member}{idx}" + + +class _Matcher: + def __init__(self, data: HeaderData, bitfields: _BitFields, hints: dict = None, report: Report = None): + self.bitfields = bitfields + self.hints = hints or {} + self.report = report + self.instances = {_strip_security(name) for name in data.instances} + self.members = {typedef: [m.name for m in members] for typedef, members in data.structs.items()} + + def _heads(self, typedef, instance, parent, sub): + heads = _instance_heads(sub or instance) + _type_heads(typedef, instance) + return list(dict.fromkeys(heads + (_type_heads(parent) if parent else []))) + + def _array_base(self, heads, member): + for head in heads: + if any(self.bitfields.has(head, token) for token in _indexed(member, 0)): + return 0 + return 1 + + def _match(self, typedef, member, index, subindex, instance, parent, sub): + abbreviation = "" + if sub and subindex is not None and not typedef.startswith("USB_OTG"): + name = re.sub(r"(\d+|[A-Z])$", "", _strip_security(sub).split("_")[-1]) + abbreviation = _abbreviation(self.members.get(typedef, []), name) + subtype = typedef.replace("_TypeDef", "") if subindex is not None or parent else None + heads = self._heads(typedef, instance, parent, sub) + base = self._array_base(heads, member) if index is not None else 0 + tokens = _tokens(member, index, subtype if subtype and "_" in subtype else None, base) + specific = _instance_heads(sub or instance) + others = [head for head in heads if head not in specific] + # Definitions specific to the instance take precedence, e.g. ADC4_SMPR over ADC_SMPR1 + for candidates in (specific, others): + for token, kind in tokens: + for head in candidates: + if self.bitfields.has(head, token): + return _name(kind, token, member, index, subindex, abbreviation), self.bitfields.take( + head, token + ) + return _name("exact", member, member, index, subindex, abbreviation), {} + + def match(self, typedef, member, index=None, subindex=None, instance=None, parent=None, sub=None): + """:return: the register name and its bit field macros by field name.""" + name, fields = self._match(typedef, member, index, subindex, instance, parent, sub) + if not fields and index: + # Arrays with bit fields only defined for the first element, e.g. CAN_F0R1 for all filter banks + name0, fields0 = self._match(typedef, member, 0, subindex, instance, parent, sub) + if fields0: + if "0" in name0: + name = re.sub(r"(? 1 and min(addresses) // _BLOCK_SIZE == max(addresses) // _BLOCK_SIZE: + virtual[parent] = min(addresses) + for name, _ in members: + parents[name] = parent + return parents, virtual + + +def _bit_fields(register: _Register, fields: dict[str, str], bitfields: _BitFields, report: Report) -> list[dict]: + """ + Splits non-contiguous bit fields and removes overlapping bit fields. + + :return: the bit fields of alternate registers by bit field name, e.g. the + input capture bit fields of TIM_CCMR1 that overlap the output compare bit fields. + """ + candidates = [] + for name, macro in fields.items(): + for position, width, index in bitfields.fields[macro]: + split = len(bitfields.fields[macro]) > 1 + fname = f"{name}_{index}" if index else name + candidates.append((fname, position, width, macro, split)) + + def bits(position, width): + return set(range(position, position + width)) + + # Remove values and bits of other bit fields, e.g. AFIO_EXTICR1_EXTI0_PB in AFIO_EXTICR1_EXTI0 + result = [] + for candidate in candidates: + cbits = bits(*candidate[1:3]) + parent = next( + ( + c + for c in candidates + if c is not candidate and candidate[0].startswith(c[0] + "_") and cbits <= bits(*c[1:3]) and not c[4] + ), + None, + ) + if parent is not None: + report.overlapping.append((register.name, candidate[0], parent[0])) + continue + result.append(candidate) + candidates = result + # Remove masks of multiple bit fields, e.g. EXTI_IMR1_IM for all EXTI_IMR1_IMx + result = [] + for candidate in candidates: + cbits = bits(*candidate[1:3]) + base = re.sub(r"(_ALL|\d+)$", "", candidate[0]) + others = [c for c in candidates if c is not candidate and bits(*c[1:3]) < cbits] + covered = set().union(*(bits(*c[1:3]) for c in others)) if others else set() + # A mask of many bit fields or of bit fields with the same name, e.g. EXTI_IMR_IM for EXTI_IMR_MRx + similar = all(c[0].startswith(base) for c in others) + if len(others) > 1 and covered == cbits and (similar or len(others) >= 8): + report.overlapping.append((register.name, candidate[0], others[0][0])) + continue + result.append(candidate) + # Prefer non-alias and non-split bit fields in the order of definition + result.sort(key=lambda c: (c[3] in bitfields.alias, c[4], bitfields.order.get(c[3], 0), c[1])) + used, alternate = {}, [] + for fname, position, width, macro, split in result: + overlap = next((used[b] for b in bits(position, width) if b in used), None) + if overlap is not None and fname not in register.fields: + # A narrow field at the edge of a much wider field, e.g. TIM_CNT_UIFCPY in TIM_CNT_CNT + opos, owidth = register.fields[overlap] + cbits, obits = bits(position, width), bits(opos, owidth) + if cbits < obits and owidth > 2 * width and (position == opos or position + width == opos + owidth): + trimmed = sorted(obits - cbits) + register.fields[overlap] = (trimmed[0], len(trimmed)) + report.overlapping.append((register.name, overlap, fname)) + overlap = None + if overlap is not None or fname in register.fields: + # Bit fields with a different layout are an alternate function of the register + inside = any( + bits(position, width) <= bits(*register.fields[n]) + for n in {used[b] for b in bits(position, width) if b in used} + ) + # Bit fields inside another bit field are values, e.g. I2C_OAR2_OA2MASK01 in I2C_OAR2_OA2MSK + if overlap is not None and not split and macro not in bitfields.alias and not inside: + if register.fields[overlap] != (position, width): + alternate.append((fname, position, width, macro)) + continue + report.overlapping.append((register.name, fname, overlap or fname)) + continue + register.fields[fname] = (position, width) + register.macros[fname] = macro + used.update((b, fname) for b in bits(position, width)) + + # Build alternate registers from the overlapping bit fields + layers = [] + while alternate: + layer, used, remaining = {}, {}, [] + for fname, position, width, macro in alternate: + if any(b in used for b in bits(position, width)) or fname in layer: + remaining.append((fname, position, width, macro)) + continue + layer[fname] = (position, width, macro) + used.update((b, fname) for b in bits(position, width)) + # Bit fields of the register that do not overlap and do not belong to an overlapping bit field + conflicting = {re.sub(r"_\d+$", "", n) for n, (p, w) in register.fields.items() if bits(p, w) & set(used)} + for fname, (position, width) in register.fields.items(): + if not bits(position, width) & set(used) and re.sub(r"_\d+$", "", fname) not in conflicting: + layer.setdefault(fname, (position, width, register.macros[fname])) + layers.append(layer) + alternate = remaining + return layers + + +def _interrupts(data: HeaderData, names: set[str], report: Report) -> dict[str, list[tuple[str, int, str]]]: + """:return: the (name, number, description) of the interrupts by peripheral name.""" + interrupts = defaultdict(list) + for irq, number in sorted(data.interrupts.items(), key=lambda i: i[1]): + if number < 0: + continue + parts = irq.split("_") + tokens = {"_".join(parts[i:j]) for i in range(len(parts)) for j in range(i + 1, len(parts) + 1)} + matches = {n for n in names if n in tokens or any(n.endswith(f"_{token}") for token in tokens if "_" in token)} + if not matches: + # Numbered interrupt lines, e.g. EXTI0 or EXTI9_5 for EXTI + matches = {n for n in names if n in {re.sub(r"\d+$", "", token) for token in tokens}} + if not matches: + # Shared interrupts of numbered instances, e.g. ADC for ADC1 and ADC2 + matches = {n for n in names if re.fullmatch(r"[A-Z]+\d+", n) and re.sub(r"\d+$", "", n) in tokens} + # Do not assign to both the sub-instance and its parent, e.g. DFSDM1_FLT0 to DFSDM1 + matches = {n for n in matches if not any(m != n and m.startswith(n) for m in matches)} + description = data.macro_descriptions.get(f"{irq}_IRQn", "") + for match in matches: + interrupts[match].append((irq, number, description)) + if not matches: + report.interrupts.append(irq) + return interrupts + + +def _channels(data: HeaderData) -> dict[int, set[str]]: + """:return: the timer instances by capture/compare channel, e.g. IS_TIM_CC3_INSTANCE.""" + channels = {} + for macro, identifiers in data.instance_macros.items(): + if match := re.fullmatch(r"IS_TIM_CC(\d)_INSTANCE", macro): + channels[int(match.group(1))] = {_strip_security(i) for i in identifiers} + return channels + + +def _restrict(pname: str, registers: list[_Register], restrictions, counters, channels, report: Report): + """Removes the bit fields and registers that are not supported by the instance.""" + instance = _strip_security(pname) + for register in list(registers): + if not register.fields: + continue + for fname, macro in list(register.macros.items()): + keys = [(register.type, register.member, macro), (register.type, register.member, fname)] + # The capture/compare channel bit fields of timers, e.g. TIM_CCER_CC3E or TIM_CCR3 + channel = None + if register.type == "TIM_TypeDef": + if match := re.match(r"(?:(?:CC|OC|IC)(\d)(?!\d)|OIS(\d)(?!\d))", fname): + channel = int(match.group(1) or match.group(2)) + elif match := re.fullmatch(r"CCR(\d)", register.member): + channel = int(match.group(1)) + if channel in channels and instance not in channels[channel]: + keys.append(None) + if None in keys or any(key in restrictions and instance not in restrictions[key] for key in keys): + report.restricted.append((pname, f"{register.name}.{fname}")) + del register.fields[fname] + del register.macros[fname] + continue + position, width = register.fields[fname] + # Instances without 32-bit counter only have 16-bit fields + if width > 16 and any(key in counters and instance not in counters[key] for key in keys): + report.restricted.append((pname, f"{register.name}.{fname}[{position + 15}:{position}]")) + register.fields[fname] = (position, 16) + if not register.fields: + registers.remove(register) + + +def memory_map(data: HeaderData, name: str = None, cubehal_path: Path = None) -> tuple[Device, Report]: + """ + Reconstructs the memory map of a CMSIS header. + + :param data: the extracted header data. + :param name: the name of the device, defaults to the header name. + :param cubehal_path: the CubeHAL folder of the family for additional information. + :return: the memory map as SVD device tree and a report of discrepancies. + """ + report = Report(data.header.name) + bitfields = _BitFields(data) + hints, restrictions, counters, enumerations = {}, {}, {}, {} + if cubehal_path is not None: + hints = cubehal.register_hints(data, bitfields, cubehal_path) + restrictions, counters = cubehal.instance_restrictions(data, bitfields, cubehal_path) + enumerations = cubehal.enumerations(data.header, data, bitfields, cubehal_path) + matcher = _Matcher(data, bitfields, hints, report) + + instances, seen = [], defaultdict(list) + for iname, (typedef, address) in sorted(data.instances.items(), key=lambda i: (len(i[0]), i[0])): + typedef = data.aliases.get(typedef, typedef) + if not typedef.endswith("TypeDef") or typedef not in data.structs: + continue + # Sub-instances of the same type at the same address are aliases, e.g. ETH_MAC for ETH + base = _strip_security(iname) + if any(base.startswith(_strip_security(other) + "_") for other in seen[(typedef, address)]): + LOGGER.debug(f"Ignoring {iname} as alias of {seen[(typedef, address)]}") + continue + seen[(typedef, address)].append(iname) + instances.append((iname, typedef, address)) + usb_blocks, usb_registers = _usb_otg_blocks(data, instances) + instances += [(iname, typedef, address) for iname, typedef, address, _ in usb_blocks] + parents, virtual = _group_instances(instances, usb_blocks) + + peripherals = {} + for iname, typedef, address in instances: + if iname not in parents: + peripherals[iname] = (address, typedef, []) + for pname, address in virtual.items(): + peripherals[pname] = (address, "", []) + + # Sub-instances with letter index, e.g. HRTIM1_TIMA to HRTIM1_TIMF, but not HRTIM1_COMMON + siblings = defaultdict(set) + for iname, parent in parents.items(): + sub = _strip_security(iname)[len(_strip_security(parent)) + 1 :] + siblings[(parent, sub[:-1])].add(sub) + + for iname, typedef, address in sorted(instances, key=lambda i: i[2]): + pname = parents.get(iname, iname) + paddress, ptype, registers = peripherals[pname] + sub, subindex = None, None + if iname in parents: + sub = _strip_security(iname)[len(_strip_security(pname)) + 1 :] + if match := re.search(r"\d+$", sub): + subindex = match.group(0) + elif re.search(r"[A-Z]$", sub) and len(siblings[(pname, sub[:-1])]) > 1: + subindex = sub[-1] + arrays = defaultdict(list) + for rtype, member, index, offset, size in _flatten(data, typedef, address - paddress): + parent = (ptype or None) if iname in parents else None + rname, fields = matcher.match(rtype, member, index, subindex, pname, parent, iname) + register = _Register(rname, offset, size, {}, sub, type=rtype, member=member) + layers = _bit_fields(register, fields, bitfields, report) + registers.append(register) + if layers: + report.alternates.append(f"{pname}.{rname}") + for number, layer in enumerate(layers, start=1): + alternate = _Register( + f"{rname}_ALT{number if len(layers) > 1 else ''}", offset, size, {}, sub, type=rtype, member=member + ) + alternate.fields = {fname: (p, w) for fname, (p, w, _) in layer.items()} + alternate.macros = {fname: m for fname, (_, _, m) in layer.items()} + registers.append(alternate) + if index is not None: + arrays[(rtype, member)].append(register) + # Large arrays without bit fields are usually memories, e.g. PKA_RAM + for (rtype, member), elements in arrays.items(): + if len(elements) > _MAX_ARRAY_REGISTERS and not any(r.fields for r in elements): + for register in elements: + registers.remove(register) + rname = f"{sub}_{member}" if subindex is None and sub else member + registers.append(_Register(rname, elements[0].offset, elements[0].size, {}, sub, len(elements), rtype)) + for pname, member, offset in usb_registers: + rname, fields = matcher.match("USB_OTG_TypeDef", member, instance=pname) + register = _Register(rname, offset, 4, {}, member=member) + _bit_fields(register, fields, bitfields, report) + peripherals[pname][2].append(register) + + for address, ptype, registers in peripherals.values(): + # Sub-instances take precedence over the same nested structures of the parent, e.g. HRTIM1_TIMA, + # otherwise the parent registers take precedence, e.g. EXTI_C2IMR1 over EXTI_D2 + subs = {(register.offset, register.type) for register in registers if register.sub is not None} + parent = { + register.offset + for register in registers + if register.sub is None and (register.offset, register.type) not in subs + } + registers[:] = [ + r + for r in registers + if (r.sub is None and (r.offset, r.type) not in subs) or (r.sub is not None and r.offset not in parent) + ] + + channels = _channels(data) + for pname, (address, ptype, registers) in peripherals.items(): + _restrict(pname, registers, restrictions, counters, channels, report) + interrupts = _interrupts(data, {re.sub(r"_NS$", "", pname) for pname in peripherals}, report) + + device = Device(name or data.header.stem, compatible=data.defines[:1]) + signatures, addresses = {}, {} + for pname, (address, ptype, registers) in sorted(peripherals.items(), key=lambda p: (p[1][0], p[0])): + # Registers of sub-instances with colliding names are prefixed with the sub-instance name + names = defaultdict(set) + for register in registers: + names[register.name].add(register.sub) + colliding = {sub for subs in names.values() if len(subs) > 1 for sub in subs if sub is not None} + for register in registers: + if register.sub in colliding: + report.renamed.append((pname, register.name)) + register.name = f"{register.sub}_{register.name}" + registers.sort(key=lambda r: (r.offset, r.name)) + for register in registers: + if not register.fields: + report.empty.append(f"{pname}.{register.name}") + + # Instances with identical registers are derived from the first instance + signature = (ptype, tuple((r.name, r.offset, r.size, tuple(sorted(r.fields.items()))) for r in registers)) + derived = signatures.setdefault(signature, pname) if ptype and registers else pname + # The non-secure instances use the plain name + peripheral = Peripheral(re.sub(r"_NS$", "", pname), ptype, address, parent=device) + peripheral.description = data.type_descriptions.get(ptype, "") + peripheral.interrupts = interrupts.get(peripheral.name, []) + if (alternate := addresses.setdefault(address, peripheral.name)) != peripheral.name: + peripheral.alternate = alternate + if derived != pname: + peripheral.derived_from = re.sub(r"_NS$", "", derived) + continue + offsets = {} + for register in registers: + treg = Register(register.name, register.offset, register.size, parent=peripheral) + treg.description = data.member_descriptions.get((register.type, register.member), "") + if register.dim: + treg.dim = register.dim + # Registers at the same offset are alternates, e.g. unions + if (alternate := offsets.setdefault(register.offset, register.name)) != register.name: + treg.alternate = alternate + for fname, (position, width) in sorted(register.fields.items(), key=lambda f: f[1]): + tfield = BitField(fname, position, width, parent=treg) + macro = register.macros.get(fname, "") + tfield.description = data.macro_descriptions.get(macro, "") + values = enumerations.get(bitfields.alias.get(macro, macro), []) + if values and all(value < (1 << width) for _, value, _ in values): + report.enumerations += 1 + for vname, value, description in values: + EnumeratedValue(vname, value, description=description, parent=tfield) + + report.defines = len(bitfields.local) + report.unassigned = bitfields.unassigned() + return device, report + + +def memory_map_from_header(header: Path, core: str = None) -> tuple[Device, Report]: + """ + Extracts the header data and reconstructs the memory map of a CMSIS header. + + :param header: path to the CMSIS device header. + :param core: the core of dual-core devices, `cm4` or `cm7`. + :return: the memory map as SVD device tree and a report of discrepancies. + """ + data = extract_header(header, header_defines(header, core)) + name = header.stem + (f"_{core}" if core else "") + return memory_map(data, name, cubehal_folder(header.parent.parent.name)) diff --git a/src/modm_data/kg/stmicro/model.py b/src/modm_data/kg/stmicro/model.py index 020a642..28cd74c 100644 --- a/src/modm_data/kg/stmicro/model.py +++ b/src/modm_data/kg/stmicro/model.py @@ -38,7 +38,7 @@ def kg_from_cubemx(db, data): """) # Add the peripherals and their type - for pbase, name, version, ptype, features, stype in sorted(data["modules"]): + for pbase, name, version, ptype, features in sorted(data["modules"]): instance = "NULL" if pbase != name: try: diff --git a/src/modm_data/svd/__init__.py b/src/modm_data/svd/__init__.py index 8129788..7e5419e 100644 --- a/src/modm_data/svd/__init__.py +++ b/src/modm_data/svd/__init__.py @@ -1,7 +1,7 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -from .model import Device, PeripheralType, Peripheral, Register, BitField, compare_device_trees +from .model import Device, PeripheralType, Peripheral, Register, BitField, EnumeratedValue, compare_device_trees from .write import format_svd, write_svd from .read import read_svd @@ -12,6 +12,7 @@ "Peripheral", "Register", "BitField", + "EnumeratedValue", "compare_device_trees", "format_svd", "write_svd", diff --git a/src/modm_data/svd/model.py b/src/modm_data/svd/model.py index 92c8cab..cb08163 100644 --- a/src/modm_data/svd/model.py +++ b/src/modm_data/svd/model.py @@ -110,9 +110,18 @@ def __eq__(self, other) -> bool: return NotImplemented -# class EnumeratedValue(Node): -# def __init__(self, name, value): -# super().__init__(name, value=value) +class EnumeratedValue(Node): + def __init__(self, name, value, **kw): + super().__init__(name, value=value, **kw) + + def __hash__(self) -> int: + return hash(f"{self.name} {self.value}") + + def __eq__(self, other) -> bool: + if isinstance(other, self.__class__): + return self.name == other.name and self.value == other.value + else: + return NotImplemented def _compare_trees(left, right): diff --git a/src/modm_data/svd/read.py b/src/modm_data/svd/read.py index fbebbdc..3cdd781 100644 --- a/src/modm_data/svd/read.py +++ b/src/modm_data/svd/read.py @@ -1,24 +1,56 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 +import re +from lxml import etree from .model import Device, Peripheral, Register, BitField +def _int(node, tag, default=None): + if (text := node.findtext(tag)) is None: + return default + return int(text.strip().replace("#", "0b"), 0) + + +def _bit_range(node) -> tuple[int, int]: + if (offset := _int(node, "bitOffset")) is not None: + return offset, _int(node, "bitWidth", 1) + if (lsb := _int(node, "lsb")) is not None: + return lsb, _int(node, "msb") - lsb + 1 + msb, lsb = map(int, re.findall(r"\d+", node.findtext("bitRange"))) + return lsb, msb - lsb + 1 + + def read_svd(path) -> Device: - from cmsis_svd.parser import SVDParser - - parser = SVDParser.for_xml_file(path) - pdev = parser.get_device() - device = Device(pdev.name, compatible=pdev.description.split(",")) - - for peripheral in pdev.peripherals: - ptype = peripheral.get_derived_from() - if ptype is not None: - ptype = ptype.name - nper = Peripheral(peripheral.name, ptype, peripheral.base_address, parent=device) - for register in peripheral.registers: - nreg = Register(register.name, register.address_offset, register.size // 8, parent=nper) - for field in register.fields: - BitField(field.name, field.bit_offset, field.bit_width, parent=nreg) + """ + Reads the peripherals, registers and bit fields of a CMSIS-SVD file. + Derived peripherals are resolved, register arrays are not expanded. + + :param path: path to the SVD file. + :return: the SVD device tree. + """ + root = etree.parse(str(path)).getroot() + device = Device(root.findtext("name"), compatible=(root.findtext("description") or "").split(",")) + + registers = {} + peripherals = [] + for node in root.iter("peripheral"): + name = node.findtext("name") + registers[name] = node.find("registers") + peripherals.append((name, node.get("derivedFrom"), _int(node, "baseAddress"))) + + for name, derived_from, address in peripherals: + peripheral = Peripheral(name, derived_from, address, parent=device) + if derived_from: + peripheral.derived_from = derived_from + if (nodes := registers[name]) is None and derived_from: + nodes = registers.get(derived_from) + if nodes is None: + continue + for rnode in nodes.iter("register"): + size = _int(rnode, "size", 32) // 8 + register = Register(rnode.findtext("name"), _int(rnode, "addressOffset"), size, parent=peripheral) + for fnode in rnode.iter("field"): + BitField(fnode.findtext("name"), *_bit_range(fnode), parent=register) return device diff --git a/src/modm_data/svd/write.py b/src/modm_data/svd/write.py index f2c2c38..9fd0b00 100644 --- a/src/modm_data/svd/write.py +++ b/src/modm_data/svd/write.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MPL-2.0 from lxml import etree -from .model import Device, Peripheral, Register, BitField +from .model import Device, Peripheral, Register, BitField, EnumeratedValue def _add_element(node, tag, text=None): @@ -16,7 +16,7 @@ def _add_element(node, tag, text=None): def _format_device(xmlnode, treenode): _add_element(xmlnode, "name", str(treenode.name).upper().replace("X", "x")) _add_element(xmlnode, "version", "1.0") - descr = ",".join(d.string.upper() for d in (treenode.compatible or [])) + descr = ",".join((d if isinstance(d, str) else d.string).upper() for d in (treenode.compatible or [])) _add_element(xmlnode, "description", descr) _add_element(xmlnode, "addressUnitBits", "8") _add_element(xmlnode, "width", "32") @@ -28,8 +28,20 @@ def _format_device(xmlnode, treenode): def _format_peripheral(xmlnode, treenode): peripheral = _add_element(xmlnode, "peripheral") + if derived_from := getattr(treenode, "derived_from", None): + peripheral.set("derivedFrom", derived_from) _add_element(peripheral, "name", treenode.name) + if description := getattr(treenode, "description", None): + _add_element(peripheral, "description", description) + if alternate := getattr(treenode, "alternate", None): + _add_element(peripheral, "alternatePeripheral", alternate) _add_element(peripheral, "baseAddress", hex(treenode.address)) + for name, value, description in getattr(treenode, "interrupts", None) or []: + interrupt = _add_element(peripheral, "interrupt") + _add_element(interrupt, "name", name) + if description: + _add_element(interrupt, "description", description) + _add_element(interrupt, "value", value) if treenode.children: return _add_element(peripheral, "registers") else: @@ -38,7 +50,16 @@ def _format_peripheral(xmlnode, treenode): def _format_register(xmlnode, treenode): register = _add_element(xmlnode, "register") - _add_element(register, "name", treenode.name) + if dim := getattr(treenode, "dim", None): + _add_element(register, "dim", dim) + _add_element(register, "dimIncrement", hex(treenode.width)) + _add_element(register, "name", f"{treenode.name}[%s]") + else: + _add_element(register, "name", treenode.name) + if description := getattr(treenode, "description", None): + _add_element(register, "description", description) + if alternate := getattr(treenode, "alternate", None): + _add_element(register, "alternateRegister", alternate) _add_element(register, "addressOffset", hex(treenode.offset)) _add_element(register, "size", hex(treenode.width * 8)) if treenode.children: @@ -50,11 +71,24 @@ def _format_register(xmlnode, treenode): def _format_bit_field(xmlnode, treenode): field = _add_element(xmlnode, "field") _add_element(field, "name", treenode.name) + if description := getattr(treenode, "description", None): + _add_element(field, "description", description) _add_element(field, "bitOffset", treenode.position) _add_element(field, "bitWidth", treenode.width) + if treenode.children: + return _add_element(field, "enumeratedValues") return field +def _format_enumerated_value(xmlnode, treenode): + value = _add_element(xmlnode, "enumeratedValue") + _add_element(value, "name", treenode.name) + if description := getattr(treenode, "description", None): + _add_element(value, "description", description) + _add_element(value, "value", treenode.value) + return value + + def _format_svd(xmlnode, treenode): current = xmlnode if isinstance(treenode, Device): @@ -67,10 +101,10 @@ def _format_svd(xmlnode, treenode): current = _format_register(current, treenode) elif isinstance(treenode, BitField): - _format_bit_field(current, treenode) + current = _format_bit_field(current, treenode) - # elif isinstance(treenode, EnumeratedValue): - # _format_enumerated_value(current, treenode) + elif isinstance(treenode, EnumeratedValue): + _format_enumerated_value(current, treenode) for child in treenode.children: _format_svd(current, child) diff --git a/tools/make/common.mk b/tools/make/common.mk index 31962fd..1957bed 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -59,6 +59,9 @@ build-homepage: @python3 tools/scripts/synchronize_docs.py @pdoc --mermaid -o docs/src/api -t docs/pdoc modm_data @modm_pinout --all -o docs/src/pinout/ + @python3 -m modm_data.header2svd.stmicro --all --output docs/src/svd/ + @curl -fsSL --create-dirs -o docs/src/svd/index.html \ + https://gist.githubusercontent.com/salkinium/12a18032caa303697c6583937f6fcd16/raw/index.html @(cd docs && mkdocs build) diff --git a/tools/make/stmicro.mk b/tools/make/stmicro.mk index 7128433..83e1245 100644 --- a/tools/make/stmicro.mk +++ b/tools/make/stmicro.mk @@ -198,24 +198,19 @@ clean-stmicro-html-svd: # ========================= Converting Header to SVD ========================== .PHONY: convert-stmicro-header-svd-% -## Convert a C header file from the CMSIS header archive into a memory map in -## the SVD archive. The log will be placed in log/stmicro/svd/header_%.txt. -convert-stmicro-header-svd-%: log/stmicro/svd/ ext/stmicro/header/ ext/arm/cmsis/ - @-python3 -m modm_data.header2svd.stmicro $(@:convert-stmicro-header-svd-%=%) > \ - $(@:convert-stmicro-header-svd-%=log/stmicro/svd/header_%.txt) 2>&1 +## Convert the CMSIS device headers matching the pattern into SVD files and +## compare them with the ST SVD files. The report will be placed in +## log/stmicro/svd/header_%.txt. +convert-stmicro-header-svd-%: log/stmicro/svd/ ext/stmicro/header/ ext/stmicro/cubehal/ ext/stmicro/svd/ ext/arm/cmsis/ + @-python3 -m modm_data.header2svd.stmicro --compare --header $(@:convert-stmicro-header-svd-%=%) -# We are ignoring L5 U5 WB WL due to ARMv8-M S/NS aliasing and issues in headers .PHONY: convert-stmicro-header-svd -## Convert all STMicro CMSIS header files into SVD files. -convert-stmicro-header-svd: log/stmicro/svd/ ext/stmicro/header/ ext/arm/cmsis/ +## Convert all STMicro CMSIS device headers into SVD files. +convert-stmicro-header-svd: log/stmicro/svd/ ext/stmicro/header/ ext/stmicro/cubehal/ ext/stmicro/svd/ ext/arm/cmsis/ @echo "Converting all CMSIS Headers to SVD." - @-python3 -m modm_data.header2svd.stmicro \ - --all stm32f0 --all stm32f1 --all stm32f2 \ - --all stm32f3 --all stm32f4 --all stm32f7 \ - --all stm32g0 --all stm32g4 --all stm32h7 \ - --all stm32l0 --all stm32l1 --all stm32l4 + @-python3 -m modm_data.header2svd.stmicro --all --compare .PHONY: clean-stmicro-header-svd -## Remove all STMicro SVD files in the archive. +## Remove all STMicro SVD files converted from the CMSIS headers. clean-stmicro-header-svd: - @rm -f $(wildcard ext/stmicro/svd-archive/header_*.svd) + @rm -f $(wildcard ext/stmicro/svd/header_*.svd)