RFC: replace setuptools with meson-python for more stable and predictable builds and installations

.flake8

@@ -9,3 +9,5 @@ max-line-length = 88
 select = C,E,F,W,B,B950
 ignore = E501,W503,E203
 builtins = IPython
+exclude = 
+    build

.gitattributes

@@ -0,0 +1,13 @@
+hatchet/tests export-ignore
+screenshot.png export-ignore
+pytest.ini export-ignore
+logo.png export-ignore
+logo-hex.png export-ignore
+install.sh export-ignore
+CHANGELOG.md export-ignore
+.pre-commit-config.yaml export-ignore
+.flake8 export-ignore
+.coveragerc export-ignore
+.codecov.yml export-ignore
+docs export-ignore
+.github export-ignore

.github/workflows/unit-tests.yaml

@@ -27,15 +27,10 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install Python3 dependencies
+    - name: Install Hatchet
       run: |
         python -m pip install --upgrade pip pytest
-        pip install -r requirements.txt
-        # Optional Dependency for HDF Checkpointing
-        pip install tables
-        python setup.py install
-        python setup.py build_ext --inplace
-        python -m pip list
+        python -m pip install .[hdf_writer]
 
     - name: Update Black
       if: ${{ matrix.python-version == 3.9 }}
@@ -91,7 +86,8 @@ jobs:
 
     - name: Test Caliper and Timemory Support with pytest
       run: |
-        PYTHONPATH=. $(which pytest) --cov=./ --cov-report=xml
+        # PYTHONPATH=. $(which pytest) --cov=./ --cov-report=xml
+        $(which pytest) --cov=./ --cov-report=xml
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4

.gitignore

@@ -14,3 +14,4 @@ hatchet/vis/static/*_bundle*
 
 dist/
 llnl_hatchet.egg-info/
+**/__pycache__/

hatchet/cython_modules/meson.build

@@ -0,0 +1,17 @@
+# Specify the names of the Cython extension modules, excluding file extensions
+cython_module_names = [
+    'graphframe_modules',
+    'reader_modules'
+]
+
+# Loop over the Cython modules and build/install them to
+# <INSTALL_PREFIX>/hatchet/cython_modules/libs
+foreach mod_name : cython_module_names
+    py.extension_module(
+        mod_name,
+        mod_name + '.pyx',
+        dependencies: py_dep,
+        subdir: 'hatchet' / 'cython_modules' / 'libs',
+        install: true
+    )
+endforeach

hatchet/external/meson.build

@@ -0,0 +1,20 @@
+# Specify the pure Python files for this directory
+hatchet_external_python_sources = [
+    '__init__.py',
+    'console.py'
+]
+
+# Install the specified pure Python files into 
+# <INSTALL_PREFIX>/hatchet/external
+py.install_sources(
+    hatchet_external_python_sources,
+    pure: false,
+    subdir: 'hatchet' / 'external'
+)
+
+# Install roundtrip as-is into
+# <INSTALL_PREFIX>/hatchet/external
+install_subdir(
+    'roundtrip',
+    install_dir: py_top_install_dir / 'hatchet' / 'external'
+)

hatchet/meson.build

@@ -0,0 +1,26 @@
+# Specify the pure Python files for this directory
+hatchet_python_sources = [
+    '__init__.py',
+    'frame.py',
+    'graph.py',
+    'graphframe.py',
+    'node.py',
+    'version.py'
+]
+
+# Install the specified pure Python files into 
+# <INSTALL_PREFIX>/hatchet
+py.install_sources(
+    hatchet_python_sources,
+    pure: false,
+    subdir: 'hatchet'
+)
+
+# Go through all the subdirectories except for 'test'
+subdir('cython_modules')
+subdir('external')
+subdir('query')
+subdir('readers')
+subdir('util')
+subdir('vis')
+subdir('writers')

hatchet/query/meson.build

@@ -0,0 +1,19 @@
+# Specify the pure Python files for this directory
+hatchet_query_python_sources = [
+    '__init__.py',
+    'compat.py',
+    'compound.py',
+    'engine.py',
+    'errors.py',
+    'object_dialect.py',
+    'query.py',
+    'string_dialect.py'
+]
+
+# Install the specified pure Python files into
+# <INSTALL_PREFIX>/hatchet/query
+py.install_sources(
+    hatchet_query_python_sources,
+    pure: false,
+    subdir: 'hatchet' / 'query'
+)

hatchet/readers/meson.build

@@ -0,0 +1,25 @@
+# Specify the pure Python files for this directory
+hatchet_readers_python_sources = [
+    '__init__.py',
+    'caliper_native_reader.py',
+    'caliper_reader.py',
+    'cprofile_reader.py',
+    'dataframe_reader.py',
+    'gprof_dot_reader.py',
+    'hdf5_reader.py',
+    'hpctoolkit_reader.py',
+    'json_reader.py',
+    'literal_reader.py',
+    'pyinstrument_reader.py',
+    'spotdb_reader.py',
+    'tau_reader.py',
+    'timemory_reader.py'
+]
+
+# Install the specified pure Python files into 
+# <INSTALL_PREFIX>/hatchet/readers
+py.install_sources(
+    hatchet_readers_python_sources,
+    pure: false,
+    subdir: 'hatchet' / 'readers'
+)

hatchet/tests/query_compat.py

@@ -1,4 +1,4 @@
-# Copyright 2017-2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2017-2023 Lawrence Livermore National Security, LLC and other
 # Hatchet Project Developers. See the top-level LICENSE file for details.
 #
 # SPDX-License-Identifier: MIT

hatchet/util/meson.build

@@ -0,0 +1,21 @@
+# Specify the pure Python files for this directory
+# Note: don't include 'print_version.py' because
+#       that file is not needed in binary distributions
+hatchet_util_python_sources = [
+    '__init__.py',
+    'colormaps.py',
+    'config.py',
+    'deprecated.py',
+    'dot.py',
+    'executable.py',
+    'profiler.py',
+    'timer.py'
+]
+
+# Install the specified pure Python files into 
+# <INSTALL_PREFIX>/hatchet/util
+py.install_sources(
+    hatchet_util_python_sources,
+    pure: false,
+    subdir: 'hatchet' / 'util'
+)

hatchet/util/print_version.py

@@ -0,0 +1,13 @@
+# Copyright 2017-2023 Lawrence Livermore National Security, LLC and other
+# Hatchet Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: MIT
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from version import __version__  # noqa: E402
+
+print(__version__)

hatchet/vis/meson.build

@@ -0,0 +1,47 @@
+# Specify the pure Python files for this directory
+hatchet_vis_python_sources = [
+    '__init__.py',
+    'loader.py',
+    'static_fixer.py'
+]
+
+# Install the specified pure Python files into 
+# <INSTALL_PREFIX>/hatchet/vis
+py.install_sources(
+    hatchet_vis_python_sources,
+    pure: false,
+    subdir: 'hatchet' / 'vis'
+)
+
+# Install directory for this source directory
+hatchet_vis_install_dir = py_top_install_dir / 'hatchet' / 'vis'
+
+# Install all the subdirectories for visualization code as-is
+# into hatchet_vis_install_dir
+install_subdir(
+    'external',
+    install_dir: hatchet_vis_install_dir
+)
+install_subdir(
+    'scripts',
+    install_dir: hatchet_vis_install_dir
+)
+install_subdir(
+    'static',
+    install_dir: hatchet_vis_install_dir
+)
+install_subdir(
+    'styles',
+    install_dir: hatchet_vis_install_dir
+)
+install_subdir(
+    'templates',
+    install_dir: hatchet_vis_install_dir
+)
+
+# Install the following non-Python files as-is
+# into hatchet_vis_install_dir
+install_data(
+    'package.json', 'README.md', 'requirements.txt', 'webpack.config.js',
+    install_dir: hatchet_vis_install_dir
+)

hatchet/writers/meson.build

@@ -0,0 +1,14 @@
+# Specify the pure Python files for this directory
+hatchet_writers_python_sources = [
+    '__init__.py',
+    'dataframe_writer.py',
+    'hdf5_writer.py'
+]
+
+# Install the specified pure Python files into 
+# <INSTALL_PREFIX>/hatchet/writers
+py.install_sources(
+    hatchet_writers_python_sources,
+    pure: false,
+    subdir: 'hatchet' / 'writers'
+)

install_editable.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+python3 -m pip install meson[ninja] meson-python Cython
+
+python3 -m pip install --no-build-isolation -e .

meson.build

@@ -0,0 +1,23 @@
+# Setup the Meson Project
+project('llnl-hatchet', 'cython',
+    version: '2024.1.1'
+)
+
+# Get the Meson Python object, a dependency object to Python for extension modules,
+# and the path to the top-level install directory
+py = import('python').find_installation(pure: false)
+py_dep = py.dependency()
+py_top_install_dir = py.get_install_dir()
+
+# Verify that the version from version.py matches the Meson project version
+version_run = run_command(py.path(), meson.current_source_dir() / 'hatchet' / 'util' / 'print_version.py')
+if version_run.returncode() != 0
+    error('The __version__ variable in Hatchet cannot be determined')
+endif
+version_from_py = version_run.stdout().strip()
+if not version_from_py.version_compare(meson.project_version())
+    error('The __version__ variable in Hatchet does not match the Meson project version')
+endif
+
+# Enter the 'hatchet' subdirectory
+subdir('hatchet')

pyproject.toml

@@ -1,33 +1,65 @@
 [build-system]
-requires = ["setuptools", "wheel", "Cython"]
-build-backend = "setuptools.build_meta"
+requires = ["meson-python", "meson[ninja]", "Cython"]
+build-backend = "mesonpy"
 
-[tool.poetry]
+[project]
 name = "llnl-hatchet"
-version = "2024.1.1"
-description = "A Python library for analyzing hierarchical performance data."
+description = "A Python library for analyzing hierarchical performance data"
+dynamic = ["version"]
+readme = "./README.md"
+license = { file="LICENSE" }
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "License :: OSI Approved :: MIT License",
+]
 authors = [
-    "Abhinav Bhatele <bhatele@cs.umd.edu>",
-    "Stephanie Brink <brink2@llnl.gov>",
-    "Todd Gamblin <tgamblin@llnl.gov>",
+  { name = "Abhinav Bhatele", email = "bhatele@cs.umd.edu" },
+  { name = "Stephanie Brink", email = "brink2@llnl.gov" },
+  { name = "Todd Gamblin", email = "tgamblin@llnl.gov" }
+]
+maintainers = [
+  { name = "Olga Pearce", email = "pearce8@llnl.gov" },
+  { name = "Ian Lumsden", email = "ilumsden@vols.utk.edu" },
+  { name = "Connor Scully-Allison", email = "cscullyallison@sci.utah.edu" },
+  { name = "Dewi Yokelson", email = "dewiy@cs.uoregon.edu" },
+  { name = "Michael McKinsey", email = "michaelmckinsey@tamu.edu" }
+]
+requires-python = ">= 3.7"
+dependencies = [
+  "pydot",
+  "PyYAML",
+  "matplotlib",
+  "numpy",
+  "pandas",
+  "textX >= 3.0.0",
+  "multiprocess",
+  "caliper-reader",
+]
+
+[project.urls]
+source_code = "https://github.com/llnl/hatchet"
+documentation = "https://llnl-hatchet.readthedocs.io/en/latest/"
+
+[project.optional-dependencies]
+hdf_writer = [
+  "tables"
 ]
-license = "MIT"
 
 [tool.ruff]
 line-length = 88
 target-version = 'py37'
 include = ['\.pyi?$']
 exclude = [
-    ".eggs",
-    ".git",
-    ".hg",
-    ".mypy_cache",
-    ".tox",
-    ".venv",
-    "_build",
-    "buck-out",
-    "build",
-    "dist",
+  ".eggs",
+  ".git",
+  ".hg",
+  ".mypy_cache",
+  ".tox",
+  ".venv",
+  "_build",
+  "buck-out",
+  "build",
+  "dist",
 ]
 
 [tool.black]