From 45adc810576c471381e02fefcc2ad2d94c11aadb Mon Sep 17 00:00:00 2001 From: Jesse <16379819+JesseVent@users.noreply.github.com> Date: Mon, 8 Jun 2026 17:42:58 +0930 Subject: [PATCH 1/7] feat: add supa_privacy anonymisation TLE extension --- Makefile | 3 +- supa_privacy/README.md | 285 +++++++++++++++++++ supa_privacy/supa_privacy--1.0.0.sql | 342 +++++++++++++++++++++++ supa_privacy/supa_privacy.control | 5 + supa_privacy/test_validation.sql | 394 +++++++++++++++++++++++++++ 5 files changed, 1028 insertions(+), 1 deletion(-) create mode 100644 supa_privacy/README.md create mode 100644 supa_privacy/supa_privacy--1.0.0.sql create mode 100644 supa_privacy/supa_privacy.control create mode 100644 supa_privacy/test_validation.sql diff --git a/Makefile b/Makefile index cd6b7a7..0d3d9c3 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ reset: .PHONY: tle.install tle.install: \ tle.install.pg_idkit \ - tle.install.is_even + tle.install.is_even \ + tle.install.supa_privacy @echo "\n\nDone!" tle.install.%: diff --git a/supa_privacy/README.md b/supa_privacy/README.md new file mode 100644 index 0000000..e96594f --- /dev/null +++ b/supa_privacy/README.md @@ -0,0 +1,285 @@ +# supa_privacy - PostgreSQL Data Anonymisation TLE Extension + +`supa_privacy` is a relocatable, pure SQL PostgreSQL **Trusted Language Extension (TLE)** designed for database-side data de-identification, compliance, and sandbox security. It provides standard masking, hashing, and perturbation functions and is fully optimized for **[database.dev](https://database.dev)** and **Supabase**. + +It is particularly useful for: +- Preparing production database schemas for staging or development environments. +- Sharing data with external analytics teams while respecting privacy regulations (GDPR, HIPAA, CCPA). +- Implementing Dynamic Data Masking (DDM) for low-privilege database roles. + +--- + +## 🚀 Installation + +### 1. Enable TLE Support +`supa_privacy` runs as a Trusted Language Extension and requires the `pg_tle` extension to be enabled in your PostgreSQL database: + +```sql +CREATE EXTENSION IF NOT EXISTS "pg_tle"; +``` + +### 2. Installing from database.dev + +Using the [dbdev CLI](https://supabase.github.io/dbdev): + +```bash +dbdev add -o ./migrations -s extensions -v 1.0.0 package -n "jvent@supa_privacy" +``` + +This will generate a migration file in your `./migrations` folder containing the SQL required to load the extension. After applying the migration, enable the extension: + +```sql +CREATE EXTENSION "jvent@supa_privacy" VERSION '1.0.0' SCHEMA supa_privacy; +``` + +--- + +## 🛠️ API Reference + +All functions are located under the schema where the extension is installed (e.g. `supa_privacy`). + +### 1. Email Masking +Masks the username part of an email address while keeping the domain. +* **Function:** `supa_privacy.mask_email(email text) RETURNS text` +* **Behavior:** + - `jesse@vent.com` ➡️ `j***e@vent.com` + - `ab@vent.com` ➡️ `a*@vent.com` + - `a@vent.com` ➡️ `*@vent.com` +* **Example:** + ```sql + SELECT supa_privacy.mask_email('jesse@vent.com'); -- Returns: j***e@vent.com + ``` + +### 2. Phone Masking +Masks digit characters while preserving formatting characters (such as spaces, hyphens, plus signs, and brackets). +* **Format-Preserving Masking:** `supa_privacy.mask_phone_flexible(phone text, keep_digits int DEFAULT 4, mask_char char DEFAULT '*') RETURNS text` +* **Default Wrapper:** `supa_privacy.mask_phone(phone text) RETURNS text` (wrapper calling `mask_phone_flexible` keeping 4 digits). +* **Behavior:** + - `+1 (202) 555-0143` ➡️ `+* (***) ***-0143` + - `+61412345678` ➡️ `+*******5678` +* **Example:** + ```sql + SELECT supa_privacy.mask_phone_flexible('+1 (202) 555-0143', 4); -- Returns: +* (***) ***-0143 + ``` + +### 3. Text Masking & Redaction +Fully or partially obscures text strings. +* **Full Masking:** `supa_privacy.mask_text(val text, mask_char char DEFAULT '*') RETURNS text` + - `secret` ➡️ `******` +* **Partial Masking:** `supa_privacy.partial_mask(val text, prefix_keep int, suffix_keep int, mask_char char DEFAULT '*') RETURNS text` + - `supa_privacy.partial_mask('1234-5678-9012', 4, 4, '*')` ➡️ `1234******9012` +* **Examples:** + ```sql + SELECT supa_privacy.mask_text('secret'); -- Returns: ****** + SELECT supa_privacy.partial_mask('1234-5678-9012', 4, 4, '*'); -- Returns: 1234******9012 + ``` + +### 4. Cryptographic Salted Hashing (Deterministic) +Generates a deterministic SHA-256 hash using a secret salt. +* **Function:** `supa_privacy.salted_hash(val text, salt text) RETURNS text` +* **Example:** + ```sql + SELECT supa_privacy.salted_hash('user_id_123', 'secret_salt_key'); + -- Returns hex-encoded SHA-256 hash: 39de1a88b56f... + ``` + +### 5. Numeric Perturbation (Noise Injection) +Adds bounded random variance (noise) to numeric fields to protect details while preserving statistical averages. +* **Volatile Perturbation:** `supa_privacy.perturb_numeric(val numeric, max_deviation numeric DEFAULT 0.07) RETURNS numeric` + - Noise changes on every query execution. +* **Deterministic Perturbation:** `supa_privacy.perturb_numeric_deterministic(val numeric, seed_key text, max_deviation numeric DEFAULT 0.07) RETURNS numeric` + - Noise is seeded by `seed_key` (typically the row's primary key) so it remains identical across multiple query executions (critical for analytical query stability). +* **Examples:** + ```sql + -- Volatile noise + SELECT supa_privacy.perturb_numeric(100.0, 0.10); -- Returns a value between 90.0 and 110.0 + -- Deterministic noise (always returns same value for 'user_1') + SELECT supa_privacy.perturb_numeric_deterministic(100.0, 'user_1', 0.10); + ``` + +### 6. Date Generalization & Shifting +Obfuscates absolute dates via truncation or chronological offsets. +* **Date Generalization (Bucketing):** `supa_privacy.generalize_date(val date, bucket text DEFAULT 'month') RETURNS date` + - Truncates dates to `'year'`, `'quarter'`, `'month'`, or `'week'`. +* **Deterministic Date Shifting:** `supa_privacy.shift_date_deterministic(val date, seed_key text, max_days int DEFAULT 30) RETURNS date` + - Shifts date back/forward by a deterministic number of days based on a seed key. Ideal for HIPAA compliance (preserves chronological order of events within a user profile while masking the absolute dates). +* **Examples:** + ```sql + SELECT supa_privacy.generalize_date('2026-06-08'::date, 'year'); -- Returns: 2026-01-01 + SELECT supa_privacy.shift_date_deterministic('2026-06-08'::date, 'user_123_salt', 15); -- Shifts by +/- 15 days deterministically + ``` + +### 7. Numeric Generalization (Bucketing) +Groups numbers into custom bucket intervals (useful for ages, salaries). +* **Function:** `supa_privacy.generalize_numeric(val numeric, bucket_size numeric) RETURNS numeric` +* **Example:** + ```sql + SELECT supa_privacy.generalize_numeric(27, 5); -- Returns: 25 (rounds to nearest multiple of 5) + ``` + +### 8. Dynamic Masked View Generator (Extensible) +Scans a physical table and automatically generates a secure VIEW that replaces sensitive fields with masking expressions while leaving other columns untouched. +* **Function:** `supa_privacy.create_masked_view(source_table regclass, view_name text, rules jsonb) RETURNS void` +* **Rules JSONB Format:** + A JSON object mapping column names to rule configurations. Supported rule configurations: + - `{"type": "email"}` + - `{"type": "phone", "keep_digits": 4}` (uses formatting-preserving masking) + - `{"type": "hash", "salt": "my_salt"}` + - `{"type": "perturb", "variance": 0.05, "seed_column": "id"}` (deterministic noise) + - `{"type": "shift_date", "days": 15, "seed_column": "id"}` (deterministic date shifting) + - `{"type": "generalize_numeric", "bucket": 10}` + - `{"type": "generalize_date", "bucket": "year"}` + - `{"type": "redact", "value": "NULL"}` (replaces value with constant or NULL) + - `{"type": "custom", "expression": "upper(reverse({col}))"}` (interpolates `{col}` with column name) + +--- + +## 📖 End-to-End Example + +Here is a full walkthrough showing how to de-identify a customer table. + +### 1. Create a Source Table and Insert Mock Data +```sql +CREATE TABLE public.customers ( + id SERIAL PRIMARY KEY, + full_name TEXT NOT NULL, + email TEXT UNIQUE NOT NULL, + phone TEXT, + age INT, + yearly_salary NUMERIC(10, 2), + signup_date DATE NOT NULL +); + +INSERT INTO public.customers (full_name, email, phone, age, yearly_salary, signup_date) VALUES +('John Doe', 'john.doe@gmail.com', '+1 (202) 555-0143', 27, 85000.00, '2025-03-12'), +('Jane Smith', 'jane_smith@corp.com', '+61 412 000 222', 43, 142000.00, '2024-11-20'), +('Bob Johnson', 'bjohnson@yahoo.com', '0491 570 156', 58, 62500.50, '2026-01-05'); +``` + +### 2. Generate a Masked View +Define anonymisation rules in JSON and invoke `create_masked_view`: + +```sql +SELECT supa_privacy.create_masked_view( + 'public.customers'::regclass, + 'public.v_customers_deidentified', + '{ + "full_name": {"type": "hash", "salt": "AppSaltKey123"}, + "email": {"type": "email"}, + "phone": {"type": "phone", "keep_digits": 4}, + "age": {"type": "generalize_numeric", "bucket": 10}, + "yearly_salary": {"type": "perturb", "variance": 0.07, "seed_column": "id"}, + "signup_date": {"type": "shift_date", "days": 15, "seed_column": "id"} + }'::jsonb +); +``` + +### 3. Query the Masked View +Query the generated view: + +```sql +SELECT * FROM public.v_customers_deidentified; +``` + +**Result:** +| id | full_name | email | phone | age | yearly_salary | signup_date | +|---|---|---|---|---|---|---| +| 1 | `e97a3a9489f...` | `j***e@gmail.com` | `+* (***) ***-0143` | 30 | `87140.23` (Stable) | `2025-03-14` (Shifted) | +| 2 | `a902b4d812d...` | `j***h@corp.com` | `+** *** *** 0222` | 40 | `138402.12` (Stable) | `2024-11-12` (Shifted) | +| 3 | `d8293bcde11...` | `b***n@yahoo.com` | `**** *** 0156` | 60 | `64102.50` (Stable) | `2026-01-08` (Shifted) | + +--- + +## 🔐 RLS & Access Control Integration Patterns + +For production environments (like **Supabase**), you should combine `supa_privacy`'s Column-level Dynamic Data Masking (DDM) with PostgreSQL **Row-Level Security (RLS)** for comprehensive access control. + +### 1. Dual-Layer Masking + RLS Pattern +**Best Practice:** Always apply RLS to both the base table and the generated view. This ensures that users see only the rows they are allowed to see, with sensitive columns masked. + +```sql +-- 1. Create source table with RLS +CREATE TABLE public.customers ( + id SERIAL PRIMARY KEY, + full_name TEXT NOT NULL, + email TEXT UNIQUE NOT NULL, + phone TEXT, + department_id INT, -- RLS filter key + yearly_salary NUMERIC(10, 2) +); + +ALTER TABLE public.customers ENABLE ROW LEVEL SECURITY; + +-- 2. Create RLS policy for the base table +CREATE POLICY cust_department_policy ON public.customers + FOR SELECT + USING (department_id = current_setting('app.current_department_id')::int); + +-- 3. Generate the masked view (DDM layer) +SELECT supa_privacy.create_masked_view( + 'public.customers'::regclass, + 'public.v_customers_masked', + '{ + "full_name": {"type": "hash", "salt": "SecureAppSaltKey"}, + "email": {"type": "email"}, + "phone": {"type": "phone"}, + "yearly_salary": {"type": "perturb", "variance": 0.07} + }'::jsonb +); + +-- 4. Enable RLS on the masked view (CRITICAL!) +ALTER TABLE public.v_customers_masked ENABLE ROW LEVEL SECURITY; + +CREATE POLICY mask_department_policy ON public.v_customers_masked + FOR SELECT + USING (department_id = current_setting('app.current_department_id')::int); + +-- 5. Grant permissions to database role +GRANT SELECT ON public.v_customers_masked TO app_user; +``` + +### 2. Alternative: Role-Based Transparent Masking +If you want users to query the base table directly but have masking applied conditionally based on the user's role or session variables: + +```sql +-- Wrapper function to conditionally mask based on session variables +CREATE OR REPLACE FUNCTION public.get_secured_customers() +RETURNS TABLE( + id INT, + full_name TEXT, + email TEXT, + phone TEXT, + department_id INT, + yearly_salary NUMERIC(10, 2) +) AS $$ +BEGIN + RETURN QUERY + SELECT + c.id, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.mask_text(c.full_name, '*') + ELSE c.full_name + END, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.mask_email(c.email) + ELSE c.email + END, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.mask_phone(c.phone) + ELSE c.phone + END, + c.department_id, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.perturb_numeric(c.yearly_salary, 0.07) + ELSE c.yearly_salary + END + FROM public.customers c + WHERE c.department_id = current_setting('app.current_department_id', true)::int; +END; +$$ LANGUAGE plpgsql; +``` + +--- + +## 🛡️ License +MIT License. Feel free to use and distribute. diff --git a/supa_privacy/supa_privacy--1.0.0.sql b/supa_privacy/supa_privacy--1.0.0.sql new file mode 100644 index 0000000..3f91e21 --- /dev/null +++ b/supa_privacy/supa_privacy--1.0.0.sql @@ -0,0 +1,342 @@ +-- supa_privacy extension database schema definition + +-- --------------------------------------------------------------------- +-- 1. EMAIL MASKING +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.mask_email(email text) +RETURNS text AS $$ +DECLARE + parts text[]; + username text; + domain text; + len int; +BEGIN + IF email IS NULL OR email = '' THEN + RETURN email; + END IF; + + -- Split email by '@' + parts := string_to_array(email, '@'); + IF array_length(parts, 1) != 2 THEN + -- Fallback if format is not standard email + RETURN regexp_replace(email, '.', '*', 'g'); + END IF; + + username := parts[1]; + domain := parts[2]; + len := length(username); + + IF len <= 1 THEN + RETURN '*' || '@' || domain; + ELSIF len = 2 THEN + RETURN left(username, 1) || '*' || '@' || domain; + ELSE + -- e.g. jesse@vent.com -> j***e@vent.com + RETURN left(username, 1) || '***' || right(username, 1) || '@' || domain; + END IF; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + + +-- --------------------------------------------------------------------- +-- 2. PHONE MASKING (Formatting-Preserving) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.mask_phone_flexible( + phone text, + keep_digits int DEFAULT 4, + mask_char char DEFAULT '*' +) +RETURNS text AS $$ +DECLARE + result text := ''; + char_val text; + digit_count int := 0; + total_digits int := 0; + digits_to_mask int; +BEGIN + IF phone IS NULL OR phone = '' THEN + RETURN phone; + END IF; + + -- Count total digits in the string + total_digits := length(regexp_replace(phone, '\D', '', 'g')); + digits_to_mask := total_digits - keep_digits; + + -- Iterate and mask digits selectively + FOR i IN 1..length(phone) LOOP + char_val := substr(phone, i, 1); + IF char_val ~ '[0-9]' THEN + digit_count := digit_count + 1; + IF digit_count <= digits_to_mask THEN + result := result || mask_char; + ELSE + result := result || char_val; + END IF; + ELSE + -- Keep formatting spaces, hyphens, plus signs, brackets + result := result || char_val; + END IF; + END LOOP; + + RETURN result; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION @extschema@.mask_phone(phone text) +RETURNS text AS $$ +BEGIN + -- Backward-compatible wrapper that defaults to format-preserving masking + RETURN @extschema@.mask_phone_flexible(phone, 4, '*'); +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + + +-- --------------------------------------------------------------------- +-- 3. TEXT MASKING & REDACTION +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.mask_text(val text, mask_char char DEFAULT '*') +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN repeat(mask_char, length(val)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION @extschema@.partial_mask(val text, prefix_keep int, suffix_keep int, mask_char char DEFAULT '*') +RETURNS text AS $$ +DECLARE + len int; +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + len := length(val); + IF len <= (prefix_keep + suffix_keep) THEN + RETURN repeat(mask_char, len); + END IF; + RETURN left(val, prefix_keep) || repeat(mask_char, len - prefix_keep - suffix_keep) || right(val, suffix_keep); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 4. CRYPTOGRAPHIC HASHING (Salted & Deterministic) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.salted_hash(val text, salt text) +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + -- Uses built-in sha256 to hash concatenation of value and salt, and encodes to hex + RETURN encode(sha256(convert_to(val || salt, 'UTF8')), 'hex'); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 5. NUMERIC PERTURBATION (Volatile & Deterministic Noise) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.perturb_numeric(val numeric, max_deviation numeric DEFAULT 0.07) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + -- deviation is a random number between -max_deviation and +max_deviation (volatile per run) + RETURN val * (1.0 + (random() * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE OR REPLACE FUNCTION @extschema@.perturb_numeric_deterministic( + val numeric, + seed_key text, + max_deviation numeric DEFAULT 0.07 +) +RETURNS numeric AS $$ +DECLARE + raw_hash bigint; + normalized_rand numeric; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + + -- Convert SHA-256 hash fragment to a big integer deterministically + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + + -- Normalize big integer to a 0.0 .. 1.0 range (divide by 2^60 - 1) + normalized_rand := abs(raw_hash)::numeric / 1152921504606846975.0; + + -- Map normalized value to the deviation bounds + RETURN val * (1.0 + (normalized_rand * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 6. DATE GENERALIZATION & SHIFTING +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.generalize_date(val date, bucket text DEFAULT 'month') +RETURNS date AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN date_trunc(bucket, val)::date; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION @extschema@.shift_date_deterministic( + val date, + seed_key text, + max_days int DEFAULT 30 +) +RETURNS date AS $$ +DECLARE + raw_hash bigint; + shift_days int; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + + -- Convert SHA-256 hash fragment to a big integer deterministically + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + + -- Map to a shift in days between [-max_days, max_days] + shift_days := (abs(raw_hash) % (max_days * 2 + 1)) - max_days; + + RETURN val + shift_days; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 7. NUMERIC GENERALIZATION (Bucketing) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.generalize_numeric(val numeric, bucket_size numeric) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL OR bucket_size IS NULL OR bucket_size <= 0 THEN + RETURN val; + END IF; + -- Rounds value to the nearest multiple of bucket_size + RETURN round(val / bucket_size) * bucket_size; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 8. DYNAMIC MASKED VIEW GENERATOR (Enhanced & Extensible) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.create_masked_view( + source_table regclass, + view_name text, + rules jsonb +) +RETURNS void AS $$ +DECLARE + column_record record; + col_name text; + col_type text; + rule jsonb; + rule_type text; + select_expr text; + select_list text := ''; + sql_stmt text; + full_table_name text; + seed_col text; +BEGIN + -- Resolve full schema-qualified table name + full_table_name := source_table::text; + + -- Loop through all columns of the source table + FOR column_record IN + SELECT attname, format_type(atttypid, atttypmod) AS type_desc + FROM pg_attribute + WHERE attrelid = source_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + col_name := column_record.attname; + col_type := column_record.type_desc; + + -- Check if there is a rule defined for this column + rule := rules -> col_name; + + IF rule IS NOT NULL THEN + rule_type := rule ->> 'type'; + + CASE rule_type + WHEN 'email' THEN + select_expr := '@extschema@.mask_email(' || quote_ident(col_name) || '::text)'; + + WHEN 'phone' THEN + IF (rule ->> 'keep_digits') IS NOT NULL THEN + select_expr := '@extschema@.mask_phone_flexible(' || quote_ident(col_name) || '::text, ' || (rule ->> 'keep_digits') || ')'; + ELSE + select_expr := '@extschema@.mask_phone(' || quote_ident(col_name) || '::text)'; + END IF; + + WHEN 'hash' THEN + select_expr := '@extschema@.salted_hash(' || quote_ident(col_name) || '::text, ' || quote_literal(coalesce(rule ->> 'salt', '')) || ')'; + + WHEN 'perturb' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := '@extschema@.perturb_numeric_deterministic(' || quote_ident(col_name) || '::numeric, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + ELSE + select_expr := '@extschema@.perturb_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + END IF; + + WHEN 'shift_date' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := '@extschema@.shift_date_deterministic(' || quote_ident(col_name) || '::date, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'days', '30') || ')'; + ELSE + select_expr := '@extschema@.shift_date_deterministic(' || quote_ident(col_name) || '::date, ' || quote_literal('default_seed') || ', ' || coalesce(rule ->> 'days', '30') || ')'; + END IF; + + WHEN 'generalize_numeric' THEN + select_expr := '@extschema@.generalize_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'bucket', '10') || ')'; + + WHEN 'generalize_date' THEN + select_expr := '@extschema@.generalize_date(' || quote_ident(col_name) || '::date, ' || quote_literal(coalesce(rule ->> 'bucket', 'month')) || ')'; + + WHEN 'redact' THEN + IF (rule ->> 'value') IS NULL OR (rule ->> 'value') = 'NULL' THEN + select_expr := 'NULL'; + ELSE + select_expr := quote_literal(rule ->> 'value'); + END IF; + + WHEN 'custom' THEN + -- Replace placeholder {col} with the actual quoted column identifier + select_expr := replace(rule ->> 'expression', '{col}', quote_ident(col_name)); + + ELSE + -- Unknown rule type: default to as-is + select_expr := quote_ident(col_name); + END CASE; + + -- Ensure expression is cast back to the original column type + select_expr := '(' || select_expr || ')::' || col_type; + ELSE + -- No rule defined: select as-is + select_expr := quote_ident(col_name); + END IF; + + -- Append to select list + IF select_list != '' THEN + select_list := select_list || ', '; + END IF; + select_list := select_list || select_expr || ' AS ' || quote_ident(col_name); + END LOOP; + + -- Build and execute CREATE VIEW statement + sql_stmt := 'CREATE OR REPLACE VIEW ' || quote_ident(view_name) || ' AS SELECT ' || select_list || ' FROM ' || full_table_name; + EXECUTE sql_stmt; +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/supa_privacy/supa_privacy.control b/supa_privacy/supa_privacy.control new file mode 100644 index 0000000..cd8547a --- /dev/null +++ b/supa_privacy/supa_privacy.control @@ -0,0 +1,5 @@ +# supa_privacy extension for PostgreSQL +comment = 'Formatting-preserving anonymisation and data masking' +default_version = '1.0.0' +relocatable = true +superuser = false diff --git a/supa_privacy/test_validation.sql b/supa_privacy/test_validation.sql new file mode 100644 index 0000000..5c0774c --- /dev/null +++ b/supa_privacy/test_validation.sql @@ -0,0 +1,394 @@ +-- ===================================================================== +-- SUPA_PRIVACY EXTENSION VALIDATION TEST SCRIPT +-- Run this script in any PostgreSQL database (version 13+) to verify +-- the correctness of the anonymisation functions and view generation. +-- ===================================================================== + +BEGIN; + +-- --------------------------------------------------------- +-- 1. Loading extension schema and functions... +-- --------------------------------------------------------- + +CREATE SCHEMA IF NOT EXISTS supa_privacy; + +-- Copy of functions for standalone execution verification + +CREATE OR REPLACE FUNCTION supa_privacy.mask_email(email text) +RETURNS text AS $$ +DECLARE + parts text[]; + username text; + domain text; + len int; +BEGIN + IF email IS NULL OR email = '' THEN + RETURN email; + END IF; + parts := string_to_array(email, '@'); + IF array_length(parts, 1) != 2 THEN + RETURN regexp_replace(email, '.', '*', 'g'); + END IF; + username := parts[1]; + domain := parts[2]; + len := length(username); + IF len <= 1 THEN + RETURN '*' || '@' || domain; + ELSIF len = 2 THEN + RETURN left(username, 1) || '*' || '@' || domain; + ELSE + RETURN left(username, 1) || '***' || right(username, 1) || '@' || domain; + END IF; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_privacy.mask_phone_flexible( + phone text, + keep_digits int DEFAULT 4, + mask_char char DEFAULT '*' +) +RETURNS text AS $$ +DECLARE + result text := ''; + char_val text; + digit_count int := 0; + total_digits int := 0; + digits_to_mask int; +BEGIN + IF phone IS NULL OR phone = '' THEN + RETURN phone; + END IF; + total_digits := length(regexp_replace(phone, '\D', '', 'g')); + digits_to_mask := total_digits - keep_digits; + FOR i IN 1..length(phone) LOOP + char_val := substr(phone, i, 1); + IF char_val ~ '[0-9]' THEN + digit_count := digit_count + 1; + IF digit_count <= digits_to_mask THEN + result := result || mask_char; + ELSE + result := result || char_val; + END IF; + ELSE + result := result || char_val; + END IF; + END LOOP; + RETURN result; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_privacy.mask_phone(phone text) +RETURNS text AS $$ +BEGIN + RETURN supa_privacy.mask_phone_flexible(phone, 4, '*'); +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_privacy.mask_text(val text, mask_char char DEFAULT '*') +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN repeat(mask_char, length(val)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.partial_mask(val text, prefix_keep int, suffix_keep int, mask_char char DEFAULT '*') +RETURNS text AS $$ +DECLARE + len int; +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + len := length(val); + IF len <= (prefix_keep + suffix_keep) THEN + RETURN repeat(mask_char, len); + END IF; + RETURN left(val, prefix_keep) || repeat(mask_char, len - prefix_keep - suffix_keep) || right(val, suffix_keep); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.salted_hash(val text, salt text) +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN encode(sha256(convert_to(val || salt, 'UTF8')), 'hex'); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.perturb_numeric(val numeric, max_deviation numeric DEFAULT 0.07) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN val * (1.0 + (random() * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE OR REPLACE FUNCTION supa_privacy.perturb_numeric_deterministic( + val numeric, + seed_key text, + max_deviation numeric DEFAULT 0.07 +) +RETURNS numeric AS $$ +DECLARE + raw_hash bigint; + normalized_rand numeric; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + normalized_rand := abs(raw_hash)::numeric / 1152921504606846975.0; + RETURN val * (1.0 + (normalized_rand * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.generalize_date(val date, bucket text DEFAULT 'month') +RETURNS date AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN date_trunc(bucket, val)::date; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.shift_date_deterministic( + val date, + seed_key text, + max_days int DEFAULT 30 +) +RETURNS date AS $$ +DECLARE + raw_hash bigint; + shift_days int; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + shift_days := (abs(raw_hash) % (max_days * 2 + 1)) - max_days; + RETURN val + shift_days; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.generalize_numeric(val numeric, bucket_size numeric) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL OR bucket_size IS NULL OR bucket_size <= 0 THEN + RETURN val; + END IF; + RETURN round(val / bucket_size) * bucket_size; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.create_masked_view( + source_table regclass, + view_name text, + rules jsonb +) +RETURNS void AS $$ +DECLARE + column_record record; + col_name text; + col_type text; + rule jsonb; + rule_type text; + select_expr text; + select_list text := ''; + sql_stmt text; + full_table_name text; + seed_col text; +BEGIN + full_table_name := source_table::text; + FOR column_record IN + SELECT attname, format_type(atttypid, atttypmod) AS type_desc + FROM pg_attribute + WHERE attrelid = source_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + col_name := column_record.attname; + col_type := column_record.type_desc; + rule := rules -> col_name; + IF rule IS NOT NULL THEN + rule_type := rule ->> 'type'; + CASE rule_type + WHEN 'email' THEN + select_expr := 'supa_privacy.mask_email(' || quote_ident(col_name) || '::text)'; + WHEN 'phone' THEN + IF (rule ->> 'keep_digits') IS NOT NULL THEN + select_expr := 'supa_privacy.mask_phone_flexible(' || quote_ident(col_name) || '::text, ' || (rule ->> 'keep_digits') || ')'; + ELSE + select_expr := 'supa_privacy.mask_phone(' || quote_ident(col_name) || '::text)'; + END IF; + WHEN 'hash' THEN + select_expr := 'supa_privacy.salted_hash(' || quote_ident(col_name) || '::text, ' || quote_literal(coalesce(rule ->> 'salt', '')) || ')'; + WHEN 'perturb' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := 'supa_privacy.perturb_numeric_deterministic(' || quote_ident(col_name) || '::numeric, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + ELSE + select_expr := 'supa_privacy.perturb_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + END IF; + WHEN 'shift_date' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := 'supa_privacy.shift_date_deterministic(' || quote_ident(col_name) || '::date, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'days', '30') || ')'; + ELSE + select_expr := 'supa_privacy.shift_date_deterministic(' || quote_ident(col_name) || '::date, ''default_seed'', ' || coalesce(rule ->> 'days', '30') || ')'; + END IF; + WHEN 'generalize_numeric' THEN + select_expr := 'supa_privacy.generalize_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'bucket', '10') || ')'; + WHEN 'generalize_date' THEN + select_expr := 'supa_privacy.generalize_date(' || quote_ident(col_name) || '::date, ' || quote_literal(coalesce(rule ->> 'bucket', 'month')) || ')'; + WHEN 'redact' THEN + IF (rule ->> 'value') IS NULL OR (rule ->> 'value') = 'NULL' THEN + select_expr := 'NULL'; + ELSE + select_expr := quote_literal(rule ->> 'value'); + END IF; + WHEN 'custom' THEN + select_expr := replace(rule ->> 'expression', '{col}', quote_ident(col_name)); + ELSE + select_expr := quote_ident(col_name); + END CASE; + select_expr := '(' || select_expr || ')::' || col_type; + ELSE + select_expr := quote_ident(col_name); + END IF; + IF select_list != '' THEN + select_list := select_list || ', '; + END IF; + select_list := select_list || select_expr || ' AS ' || quote_ident(col_name); + END LOOP; + sql_stmt := 'CREATE OR REPLACE VIEW ' || quote_ident(view_name) || ' AS SELECT ' || select_list || ' FROM ' || full_table_name; + EXECUTE sql_stmt; +END; +$$ LANGUAGE plpgsql VOLATILE; + +-- --------------------------------------------------------- +-- 2. Running Function Unit Tests... +-- --------------------------------------------------------- + +DO $$ +BEGIN + -- test email masking + ASSERT supa_privacy.mask_email('jesse@vent.com') = 'j***e@vent.com', 'mask_email failed for normal username'; + ASSERT supa_privacy.mask_email('ab@vent.com') = 'a*@vent.com', 'mask_email failed for length 2 username'; + ASSERT supa_privacy.mask_email('a@vent.com') = '*@vent.com', 'mask_email failed for length 1 username'; + ASSERT supa_privacy.mask_email('') = '', 'mask_email failed for empty string'; + ASSERT supa_privacy.mask_email(NULL) IS NULL, 'mask_email failed for NULL'; + + -- test standard and flexible phone masking + ASSERT supa_privacy.mask_phone('+61412345678') = '+*******5678', 'mask_phone failed for +61412345678'; + ASSERT supa_privacy.mask_phone_flexible('+1 (202) 555-0143', 4) = '+* (***) ***-0143', 'mask_phone_flexible failed for formatted phone'; + ASSERT supa_privacy.mask_phone_flexible('12345', 2, '#') = '###45', 'mask_phone_flexible failed for custom mask char'; + + -- test text masking + ASSERT supa_privacy.mask_text('secret') = '******', 'mask_text failed'; + ASSERT supa_privacy.partial_mask('1234-5678-9012', 4, 4, '*') = '1234******9012', 'partial_mask failed'; + + -- test salted hash (should be deterministic) + ASSERT supa_privacy.salted_hash('test_value', 'my_salt') = supa_privacy.salted_hash('test_value', 'my_salt'), 'salted_hash is not deterministic'; + ASSERT supa_privacy.salted_hash('test_value', 'my_salt') != supa_privacy.salted_hash('test_value', 'different_salt'), 'salted_hash does not respect salt'; + ASSERT length(supa_privacy.salted_hash('test_value', 'my_salt')) = 64, 'salted_hash output is not 64-char sha256 hex string'; + + -- test numeric perturbation (volatile vs deterministic) + ASSERT supa_privacy.perturb_numeric(100.0, 0.0) = 100.0, 'perturb_numeric variance 0.0 deviation failed'; + + -- verify deterministic perturbation is stable + ASSERT supa_privacy.perturb_numeric_deterministic(100.0, 'seed1', 0.07) = supa_privacy.perturb_numeric_deterministic(100.0, 'seed1', 0.07), 'deterministic perturbation is not stable'; + ASSERT supa_privacy.perturb_numeric_deterministic(100.0, 'seed1', 0.07) != supa_privacy.perturb_numeric_deterministic(100.0, 'seed2', 0.07), 'deterministic perturbation did not respect seed'; + + -- test date generalization and shifting + ASSERT supa_privacy.generalize_date('2026-06-08'::date, 'year') = '2026-01-01'::date, 'generalize_date year failed'; + ASSERT supa_privacy.generalize_date('2026-06-08'::date, 'month') = '2026-06-01'::date, 'generalize_date month failed'; + + -- verify deterministic date shifting + ASSERT supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30) = supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30), 'deterministic date shift is not stable'; + ASSERT supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30) != supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed2', 30), 'deterministic date shift did not respect seed'; + ASSERT abs(supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30) - '2026-06-08'::date) <= 30, 'deterministic date shift out of bounds'; + + -- test numeric generalization + ASSERT supa_privacy.generalize_numeric(27, 5) = 25, 'generalize_numeric 27/5 failed'; + ASSERT supa_privacy.generalize_numeric(84200, 10000) = 80000, 'generalize_numeric salary failed'; + + RAISE NOTICE '✅ All individual unit tests passed successfully!'; +END; +$$; + +-- --------------------------------------------------------- +-- 3. Testing Masked View Generator... +-- --------------------------------------------------------- + +-- Create temporary mock table +CREATE TEMP TABLE test_users ( + id int PRIMARY KEY, + name text, + email text, + phone text, + salary numeric, + birth_date date, + secret_code text +); + +INSERT INTO test_users VALUES +(1, 'Alice Smith', 'alice@gmail.com', '+1 (202) 555-0143', 95000.50, '1990-04-15', 'top_secret_code_123'), +(2, 'Bob Jones', 'bob@corp.com', '+61 412 000 222', 125000.00, '1985-09-22', 'top_secret_code_456'); + +-- Generate masked view with advanced and custom rules +SELECT supa_privacy.create_masked_view( + 'test_users'::regclass, + 'v_test_users_masked', + '{ + "name": {"type": "hash", "salt": "testsalt"}, + "email": {"type": "email"}, + "phone": {"type": "phone", "keep_digits": 4}, + "salary": {"type": "perturb", "variance": 0.05, "seed_column": "id"}, + "birth_date": {"type": "shift_date", "days": 15, "seed_column": "id"}, + "secret_code": {"type": "custom", "expression": "upper(reverse({col}))"} + }'::jsonb +); + +-- Check that view exists and columns are masked correctly +DO $$ +DECLARE + r1 record; + r2 record; +BEGIN + SELECT * INTO r1 FROM v_test_users_masked WHERE id = 1; + SELECT * INTO r2 FROM v_test_users_masked WHERE id = 1; -- query view again to check deterministic stability + + -- Verify types and values + ASSERT r1.id = 1; + ASSERT r1.email = 'a***e@gmail.com', 'View email masking failed'; + ASSERT r1.phone = '+* (***) ***-0143', 'View phone formatting preservation failed'; + ASSERT abs(r1.birth_date - '1990-04-15'::date) <= 15, 'View birth date shifting failed'; + ASSERT length(r1.name) = 64, 'View name hashing failed'; + ASSERT r1.salary >= 90250.0 AND r1.salary <= 99751.0, 'View salary perturbation failed'; + ASSERT r1.secret_code = '321_EDOC_TERCES_POT', 'View custom expression execution failed'; + + -- Verify deterministic stability (r1 values MUST equal r2 values exactly) + ASSERT r1.salary = r2.salary, 'Deterministic perturbation is not stable in view'; + ASSERT r1.birth_date = r2.birth_date, 'Deterministic date shifting is not stable in view'; + + RAISE NOTICE '✅ Masked View Generator advanced test passed successfully!'; +END; +$$; + +-- Cleanup view and tables +DROP VIEW IF EXISTS v_test_users_masked; +DROP TABLE IF EXISTS test_users; +DROP SCHEMA supa_privacy CASCADE; + +-- --------------------------------------------------------- +-- ✅ All tests passed successfully! +-- --------------------------------------------------------- + +ROLLBACK; -- rollback to keep the test clean From 0d8302990982c3d872f4e6dc54fce9756ea9b9c8 Mon Sep 17 00:00:00 2001 From: Jesse <16379819+JesseVent@users.noreply.github.com> Date: Mon, 8 Jun 2026 17:45:43 +0930 Subject: [PATCH 2/7] Update README.md --- supa_privacy/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/supa_privacy/README.md b/supa_privacy/README.md index e96594f..72adb4c 100644 --- a/supa_privacy/README.md +++ b/supa_privacy/README.md @@ -1,4 +1,6 @@ -# supa_privacy - PostgreSQL Data Anonymisation TLE Extension +# supa_privacy - It's not just private, it's supa private. + +## PostgreSQL Data Anonymisation TLE Extension `supa_privacy` is a relocatable, pure SQL PostgreSQL **Trusted Language Extension (TLE)** designed for database-side data de-identification, compliance, and sandbox security. It provides standard masking, hashing, and perturbation functions and is fully optimized for **[database.dev](https://database.dev)** and **Supabase**. From b42cae202e462f226bb4a2501dd12b6856a4c53f Mon Sep 17 00:00:00 2001 From: Jesse <16379819+JesseVent@users.noreply.github.com> Date: Mon, 8 Jun 2026 17:53:05 +0930 Subject: [PATCH 3/7] fix: update Makefile to support newer dbdev CLI syntax and add supa_privacy --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0d3d9c3..0be3525 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,12 @@ tle.install: \ tle.install.%: @echo "\n\nInstalling $*" - dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres --path $(REPO_DIR)/$* + dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres path --directory $(REPO_DIR)/$* PGPASSWORD=postgres psql -U postgres -d postgres -h localhost -p 54322 -c "CREATE EXTENSION $*;" tle.update.%: @echo "\n\Updating $*" - dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres --path $(REPO_DIR)/$* + dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres path --directory $(REPO_DIR)/$* dbdev.publish.%: @echo "\n\nPublishing $* \n" From 9b13b70df9bd66b7963b64f09d1667d47ca4b2f8 Mon Sep 17 00:00:00 2001 From: Jesse <16379819+JesseVent@users.noreply.github.com> Date: Mon, 8 Jun 2026 18:11:37 +0930 Subject: [PATCH 4/7] feat: add supa_profile database profiling TLE extension --- Makefile | 3 +- supa_profile/README.md | 216 +++++++++++++ supa_profile/supa_profile--1.0.0.sql | 286 ++++++++++++++++++ supa_profile/supa_profile.control | 5 + supa_profile/test_validation.sql | 436 +++++++++++++++++++++++++++ 5 files changed, 945 insertions(+), 1 deletion(-) create mode 100644 supa_profile/README.md create mode 100644 supa_profile/supa_profile--1.0.0.sql create mode 100644 supa_profile/supa_profile.control create mode 100644 supa_profile/test_validation.sql diff --git a/Makefile b/Makefile index 0be3525..52e74bb 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,8 @@ reset: tle.install: \ tle.install.pg_idkit \ tle.install.is_even \ - tle.install.supa_privacy + tle.install.supa_privacy \ + tle.install.supa_profile @echo "\n\nDone!" tle.install.%: diff --git a/supa_profile/README.md b/supa_profile/README.md new file mode 100644 index 0000000..bd4b4a0 --- /dev/null +++ b/supa_profile/README.md @@ -0,0 +1,216 @@ +# supa_profile - PostgreSQL Database Table & Column Profiler + +`supa_profile` is a relocatable, pure SQL PostgreSQL **Trusted Language Extension (TLE)** designed for native table and column-level statistical profiling. It introspects schema metadata, dynamically constructs profiling queries, and compiles a comprehensive statistical report (including counts, nullability, distinctness, min/max, string length distributions, averages, medians, percentiles, and frequent value distributions) returned directly as a single structured JSONB document. + +The JSONB output format is fully compatible with the **`ProfilingResult`** TypeScript interface used by frontend profiling dashboards. + +--- + +## 🚀 Installation + +### 1. Enable TLE Support +`supa_profile` runs as a Trusted Language Extension and requires the `pg_tle` extension to be enabled: + +```sql +CREATE EXTENSION IF NOT EXISTS "pg_tle"; +``` + +### 2. Installing from database.dev + +Using the [dbdev CLI](https://supabase.github.io/dbdev): + +```bash +dbdev add -o ./migrations -s extensions -v 1.0.0 package -n "jvent@supa_profile" +``` + +This generates a migration script to load the TLE. Apply the migration, then enable the extension: + +```sql +CREATE EXTENSION "jvent@supa_profile" VERSION '1.0.0' SCHEMA supa_profile; +``` + +--- + +## 🛠️ API Reference + +### 1. Profile Table +Introspects and profiles a physical table, view, or materialized view. +* **Function:** `supa_profile.profile_table(target_table regclass, options jsonb DEFAULT '{}') RETURNS jsonb` +* **Options Parameter (`options`):** + A JSON object supporting the following parameters: + - `scan_field_values` (boolean, default: `true`): Scan column values to compute top value frequency distributions. + - `min_cell_count` (int, default: `5`): Minimum occurrence count of a value to include it in the value distribution list. + - `max_distinct_values` (int, default: `100`): Maximum distinct values returned per column in the value distribution list. + - `rows_per_table` (int, default: `0` = unlimited): Maximum rows to scan. If greater than zero, it wraps the source in a sampled subquery (`LIMIT N`). + - `calculate_numeric_stats` (boolean, default: `true`): Compute average, median, 90th percentile, and 99th percentile for numeric fields. + +* **Example:** + ```sql + SELECT supa_profile.profile_table( + 'public.users'::regclass, + '{"rows_per_table": 10000, "min_cell_count": 10}'::jsonb + ); + ``` + +--- + +## 📊 Output Schema (JSONB) + +The function returns a JSONB object containing: +- `tableStats`: Table name, row count, column count. +- `fields`: Array of column-level statistics. +- `values`: Value frequency distribution array. +- `profiledAt`: Timestamp in UTC. +- `durationMs`: Total duration of the profiling process. +- `queryMode`: Current query mode (`NORMAL` or `FAST` for sampled scans). + +### Field Properties: +| Property | Type | Description | +|---|---|---| +| `columnName` | `text` | Column identifier | +| `dataType` | `text` | Column type format | +| `nullable` | `boolean` | True if nulls are allowed | +| `nonNullCount` | `bigint` | Total count of non-null cells | +| `nullCount` | `bigint` | Total count of null cells | +| `distinctCount` | `bigint` | Total unique value count | +| `minValue` | `text` | Minimum value (omitted for JSON/bytes/arrays) | +| `maxValue` | `text` | Maximum value (omitted for JSON/bytes/arrays) | +| `avgValue` | `numeric` | Average value (numeric columns only) | +| `median` | `numeric` | Median value / P50 (numeric columns only) | +| `p90` | `numeric` | 90th percentile value (numeric columns only) | +| `p99` | `numeric` | 99th percentile value (numeric columns only) | +| `avgLength` | `numeric` | Average string length of text representation | +| `maxLength` | `bigint` | Maximum string length of text representation | +| `mode` | `text` | Most frequent value (omitted if `scan_field_values` is false) | +| `pattern` | `text` | Inferred regex pattern for type verification | + +--- + +## 📖 Example Walkthrough + +### 1. Create Sample Table +```sql +CREATE TABLE public.customers ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + age INT, + country VARCHAR(3) DEFAULT 'USA', + signup_date DATE +); + +INSERT INTO public.customers (name, age, country, signup_date) VALUES +('Alice', 25, 'USA', '2025-01-10'), +('Bob', 30, 'USA', '2025-02-15'), +('Charlie', 45, 'CAN', '2024-11-20'), +('Diana', 25, 'USA', '2025-03-01'), +('Ethan', NULL, 'GBR', NULL); +``` + +### 2. Profile the Table +```sql +SELECT jsonb_pretty(supa_profile.profile_table('public.customers'::regclass, '{"min_cell_count": 1}'::jsonb)); +``` + +### 3. Example JSONB Output +```json +{ + "fields": [ + { + "mode": "", + "pattern": "-?\\d+", + "dataType": "integer", + "nullable": false, + "nullCount": 0, + "avgLength": 1.0, + "columnName": "id", + "distinctCount": 5, + "maxLength": 1, + "nonNullCount": 5, + "tableName": "public.customers", + "avgValue": 3.0, + "median": 3.0, + "p90": 5.0, + "p99": 5.0, + "minValue": "1", + "maxValue": "5" + }, + { + "mode": "", + "pattern": ".*", + "dataType": "text", + "nullable": false, + "nullCount": 0, + "avgLength": 4.6, + "columnName": "name", + "distinctCount": 5, + "maxLength": 7, + "nonNullCount": 5, + "tableName": "public.customers", + "minValue": "Alice", + "maxValue": "Ethan" + }, + { + "mode": "25", + "pattern": "-?\\d+", + "dataType": "integer", + "nullable": true, + "nullCount": 1, + "avgLength": 1.6, + "columnName": "age", + "distinctCount": 3, + "maxLength": 2, + "nonNullCount": 4, + "tableName": "public.customers", + "avgValue": 31.25, + "median": 30.0, + "p90": 45.0, + "p99": 45.0, + "minValue": "25", + "maxValue": "45" + }, + { + "mode": "USA", + "pattern": ".*", + "dataType": "character varying(3)", + "nullable": true, + "nullCount": 0, + "avgLength": 3.0, + "columnName": "country", + "distinctCount": 3, + "maxLength": 3, + "nonNullCount": 5, + "tableName": "public.customers", + "minValue": "CAN", + "maxValue": "USA" + } + ], + "values": [ + { + "value": "25", + "percent": 0.4000, + "frequency": 2, + "columnName": "age" + }, + { + "value": "USA", + "percent": 0.6000, + "frequency": 3, + "columnName": "country" + } + ], + "queryMode": "NORMAL", + "durationMs": 4.32, + "profiledAt": "2026-06-08T18:00:00Z", + "tableStats": { + "rowCount": 5, + "selected": false, + "tableName": "public.customers", + "columnCount": 5 + } +} +``` + +--- + +## 🛡️ License +MIT License. Feel free to use and distribute. diff --git a/supa_profile/supa_profile--1.0.0.sql b/supa_profile/supa_profile--1.0.0.sql new file mode 100644 index 0000000..e99fc8e --- /dev/null +++ b/supa_profile/supa_profile--1.0.0.sql @@ -0,0 +1,286 @@ +-- supa_profile extension database schema definition + +-- --------------------------------------------------------------------- +-- 1. PATTERN INFERENCE HELPER +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.infer_pattern(data_type text) +RETURNS text AS $$ +DECLARE + t text := upper(data_type); +BEGIN + IF t LIKE '%DATE%' THEN RETURN 'YYYY-MM-DD'; END IF; + IF t LIKE '%TIME%' THEN RETURN 'HH:MM:SS'; END IF; + IF t LIKE '%INT%' THEN RETURN '-?\d+'; END IF; + IF t LIKE '%FLOAT%' OR t LIKE '%NUMERIC%' OR t LIKE '%DECIMAL%' OR t LIKE '%DOUBLE%' OR t LIKE '%REAL%' THEN + RETURN '-?\d+\.?\d*'; + END IF; + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + + +-- --------------------------------------------------------------------- +-- 2. TABLE PROFILER FUNCTION +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.profile_table( + target_table regclass, + options jsonb DEFAULT '{}' +) +RETURNS jsonb AS $$ +DECLARE + -- Options + scan_field_values boolean; + min_cell_count int; + max_distinct_values int; + rows_per_table int; + calculate_numeric_stats boolean; + + -- Execution metadata + start_time timestamptz; + duration_ms numeric; + total_rows bigint; + column_count int; + + -- Table/Schema name + full_table_name text; + from_clause text; + + -- Column loop + col record; + is_numeric boolean; + skip_min_max boolean; + + -- SQL Construction + select_exprs text := ''; + query_str text; + result_row record; + result_json jsonb; + + -- Arrays + fields_array jsonb := '[]'::jsonb; + values_array jsonb := '[]'::jsonb; + + -- Temp vars for each column + non_null_count bigint; + distinct_count bigint; + min_val text; + max_val text; + avg_len numeric; + max_len bigint; + avg_val numeric; + median_val numeric; + p90_val numeric; + p99_val numeric; + + -- Mode and Value Distribution + val_query text; + val_rec record; + val_obj jsonb; + mode_val text; + mode_found boolean; + field_obj jsonb; +BEGIN + start_time := clock_timestamp(); + full_table_name := target_table::text; + + -- Ensure options is not null + options := coalesce(options, '{}'::jsonb); + + -- Parse options with defaults + scan_field_values := coalesce((options ->> 'scan_field_values')::boolean, true); + min_cell_count := coalesce((options ->> 'min_cell_count')::int, 5); + max_distinct_values := coalesce((options ->> 'max_distinct_values')::int, 100); + rows_per_table := coalesce((options ->> 'rows_per_table')::int, 0); + calculate_numeric_stats := coalesce((options ->> 'calculate_numeric_stats')::boolean, true); + + -- Build FROM clause with optional sampling/limit + IF rows_per_table > 0 THEN + from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; + ELSE + from_clause := full_table_name; + END IF; + + -- Retrieve column count + SELECT COUNT(*)::int INTO column_count + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped; + + -- Build SELECT expressions for column stats + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Determine if type is numeric + is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') + OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); + + -- Determine if type does not support MIN/MAX + skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot'); + + -- Basic column stats + select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); + select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); + + -- MIN/MAX + IF skip_min_max THEN + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); + ELSE + select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + END IF; + + -- Avg and Max string length + select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); + select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + + -- Numeric stats (AVG, MEDIAN, P90, P99) + IF calculate_numeric_stats AND is_numeric THEN + select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + ELSE + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + END IF; + END LOOP; + + -- Execute main statistics query + query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; + EXECUTE query_str INTO result_row; + result_json := to_jsonb(result_row); + total_rows := (result_json ->> 'total_rows')::bigint; + + -- Execute value distribution query per column if enabled + IF scan_field_values AND total_rows > 0 THEN + FOR col IN + SELECT attname AS column_name + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + val_query := 'SELECT ' || + quote_literal(col.column_name) || ' AS column_name, ' || + 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || + 'COUNT(*) AS frequency ' || + 'FROM ' || from_clause || ' ' || + 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || + 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || + 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || + 'ORDER BY COUNT(*) DESC ' || + 'LIMIT ' || max_distinct_values; + + FOR val_rec IN EXECUTE val_query LOOP + val_obj := jsonb_build_object( + 'columnName', val_rec.column_name, + 'value', val_rec.value, + 'frequency', val_rec.frequency, + 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END LOOP; + END IF; + + -- Build final fields array + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; + distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; + min_val := result_json ->> (col.column_name || '__min'); + max_val := result_json ->> (col.column_name || '__max'); + avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; + max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; + avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; + median_val := (result_json ->> (col.column_name || '__median'))::numeric; + p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; + p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; + + -- Find mode for this column if we scanned values + mode_val := ''; + IF scan_field_values THEN + -- Find the first value in values_array for this column + -- Since values_array is sorted by frequency DESC, the first one is the mode + DECLARE + temp_val jsonb; + BEGIN + FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP + IF temp_val ->> 'columnName' = col.column_name THEN + mode_val := temp_val ->> 'value'; + EXIT; + END IF; + END LOOP; + END; + END IF; + + -- Construct field profile object matching FieldProfile interface + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', coalesce(avg_len, 0), + 'maxLength', coalesce(max_len, 0), + 'mode', mode_val, + 'pattern', @extschema@.infer_pattern(col.data_type) + ); + + -- Add optional numeric and min/max stats + IF min_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('minValue', min_val); + END IF; + IF max_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('maxValue', max_val); + END IF; + IF avg_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('avgValue', avg_val); + END IF; + IF median_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('median', median_val); + END IF; + IF p90_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p90', p90_val); + END IF; + IF p99_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p99', p99_val); + END IF; + + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + + duration_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000; + + -- Return full result matching ProfilingResult interface + RETURN jsonb_build_object( + 'tableStats', jsonb_build_object( + 'tableName', full_table_name, + 'rowCount', total_rows, + 'columnCount', column_count, + 'selected', false + ), + 'fields', fields_array, + 'values', values_array, + 'profiledAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'durationMs', round(duration_ms, 2), + 'queryMode', CASE WHEN rows_per_table > 0 THEN 'FAST' ELSE 'NORMAL' END + ); +END; +$$ LANGUAGE plpgsql VOLATILE STRICT; diff --git a/supa_profile/supa_profile.control b/supa_profile/supa_profile.control new file mode 100644 index 0000000..b849199 --- /dev/null +++ b/supa_profile/supa_profile.control @@ -0,0 +1,5 @@ +# supa_profile extension for PostgreSQL +comment = 'Database table and column statistical profiler' +default_version = '1.0.0' +relocatable = true +superuser = false diff --git a/supa_profile/test_validation.sql b/supa_profile/test_validation.sql new file mode 100644 index 0000000..5f58cbf --- /dev/null +++ b/supa_profile/test_validation.sql @@ -0,0 +1,436 @@ +-- ===================================================================== +-- SUPA_PROFILE EXTENSION VALIDATION TEST SCRIPT +-- Run this script to verify the correctness of the profiling functions. +-- ===================================================================== + +BEGIN; + +-- --------------------------------------------------------- +-- 1. Loading extension schema and functions... +-- --------------------------------------------------------- +CREATE SCHEMA IF NOT EXISTS supa_profile; + +-- Copy of helper functions for standalone execution verification +CREATE OR REPLACE FUNCTION supa_profile.infer_pattern(data_type text) +RETURNS text AS $$ +DECLARE + t text := upper(data_type); +BEGIN + IF t LIKE '%DATE%' THEN RETURN 'YYYY-MM-DD'; END IF; + IF t LIKE '%TIME%' THEN RETURN 'HH:MM:SS'; END IF; + IF t LIKE '%INT%' THEN RETURN '-?\d+'; END IF; + IF t LIKE '%FLOAT%' OR t LIKE '%NUMERIC%' OR t LIKE '%DECIMAL%' OR t LIKE '%DOUBLE%' OR t LIKE '%REAL%' THEN + RETURN '-?\d+\.?\d*'; + END IF; + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_profile.profile_table( + target_table regclass, + options jsonb DEFAULT '{}' +) +RETURNS jsonb AS $$ +DECLARE + -- Options + scan_field_values boolean; + min_cell_count int; + max_distinct_values int; + rows_per_table int; + calculate_numeric_stats boolean; + + -- Execution metadata + start_time timestamptz; + duration_ms numeric; + total_rows bigint; + column_count int; + + -- Table/Schema name + full_table_name text; + from_clause text; + + -- Column loop + col record; + is_numeric boolean; + skip_min_max boolean; + + -- SQL Construction + select_exprs text := ''; + query_str text; + result_row record; + result_json jsonb; + + -- Arrays + fields_array jsonb := '[]'::jsonb; + values_array jsonb := '[]'::jsonb; + + -- Temp vars for each column + non_null_count bigint; + distinct_count bigint; + min_val text; + max_val text; + avg_len numeric; + max_len bigint; + avg_val numeric; + median_val numeric; + p90_val numeric; + p99_val numeric; + + -- Mode and Value Distribution + val_query text; + val_rec record; + val_obj jsonb; + mode_val text; + mode_found boolean; + field_obj jsonb; +BEGIN + start_time := clock_timestamp(); + full_table_name := target_table::text; + + -- Ensure options is not null + options := coalesce(options, '{}'::jsonb); + + -- Parse options with defaults + scan_field_values := coalesce((options ->> 'scan_field_values')::boolean, true); + min_cell_count := coalesce((options ->> 'min_cell_count')::int, 5); + max_distinct_values := coalesce((options ->> 'max_distinct_values')::int, 100); + rows_per_table := coalesce((options ->> 'rows_per_table')::int, 0); + calculate_numeric_stats := coalesce((options ->> 'calculate_numeric_stats')::boolean, true); + + -- Build FROM clause with optional sampling/limit + IF rows_per_table > 0 THEN + from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; + ELSE + from_clause := full_table_name; + END IF; + + -- Retrieve column count + SELECT COUNT(*)::int INTO column_count + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped; + + -- Build SELECT expressions for column stats + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Determine if type is numeric + is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') + OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); + + -- Determine if type does not support MIN/MAX + skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot'); + + -- Basic column stats + select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); + select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); + + -- MIN/MAX + IF skip_min_max THEN + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); + ELSE + select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + END IF; + + -- Avg and Max string length + select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); + select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + + -- Numeric stats (AVG, MEDIAN, P90, P99) + IF calculate_numeric_stats AND is_numeric THEN + select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + ELSE + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + END IF; + END LOOP; + + -- Execute main statistics query + query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; + EXECUTE query_str INTO result_row; + result_json := to_jsonb(result_row); + total_rows := (result_json ->> 'total_rows')::bigint; + + -- Execute value distribution query per column if enabled + IF scan_field_values AND total_rows > 0 THEN + FOR col IN + SELECT attname AS column_name + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + val_query := 'SELECT ' || + quote_literal(col.column_name) || ' AS column_name, ' || + 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || + 'COUNT(*) AS frequency ' || + 'FROM ' || from_clause || ' ' || + 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || + 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || + 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || + 'ORDER BY COUNT(*) DESC ' || + 'LIMIT ' || max_distinct_values; + + FOR val_rec IN EXECUTE val_query LOOP + val_obj := jsonb_build_object( + 'columnName', val_rec.column_name, + 'value', val_rec.value, + 'frequency', val_rec.frequency, + 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END LOOP; + END IF; + + -- Build final fields array + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; + distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; + min_val := result_json ->> (col.column_name || '__min'); + max_val := result_json ->> (col.column_name || '__max'); + avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; + max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; + avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; + median_val := (result_json ->> (col.column_name || '__median'))::numeric; + p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; + p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; + + -- Find mode for this column if we scanned values + mode_val := ''; + IF scan_field_values THEN + -- Find the first value in values_array for this column + -- Since values_array is sorted by frequency DESC, the first one is the mode + DECLARE + temp_val jsonb; + BEGIN + FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP + IF temp_val ->> 'columnName' = col.column_name THEN + mode_val := temp_val ->> 'value'; + EXIT; + END IF; + END LOOP; + END; + END IF; + + -- Construct field profile object matching FieldProfile interface + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', coalesce(avg_len, 0), + 'maxLength', coalesce(max_len, 0), + 'mode', mode_val, + 'pattern', supa_profile.infer_pattern(col.data_type) + ); + + -- Add optional numeric and min/max stats + IF min_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('minValue', min_val); + END IF; + IF max_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('maxValue', max_val); + END IF; + IF avg_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('avgValue', avg_val); + END IF; + IF median_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('median', median_val); + END IF; + IF p90_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p90', p90_val); + END IF; + IF p99_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p99', p99_val); + END IF; + + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + + duration_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000; + + -- Return full result matching ProfilingResult interface + RETURN jsonb_build_object( + 'tableStats', jsonb_build_object( + 'tableName', full_table_name, + 'rowCount', total_rows, + 'columnCount', column_count, + 'selected', false + ), + 'fields', fields_array, + 'values', values_array, + 'profiledAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'durationMs', round(duration_ms, 2), + 'queryMode', CASE WHEN rows_per_table > 0 THEN 'FAST' ELSE 'NORMAL' END + ); +END; +$$ LANGUAGE plpgsql VOLATILE STRICT; + +-- --------------------------------------------------------- +-- 2. Create mock tables and populate data... +-- --------------------------------------------------------- + +CREATE TEMP TABLE mock_users ( + id serial PRIMARY KEY, + username text NOT NULL, + age int, + salary numeric(10, 2), + signup_date date, + extra_info jsonb, + tags text[] +); + +INSERT INTO mock_users (username, age, salary, signup_date, extra_info, tags) VALUES +('alice', 25, 80000.00, '2025-01-10'::date, '{"city": "NY"}'::jsonb, ARRAY['admin', 'staff']), +('bob', 30, 95000.00, '2025-02-15'::date, '{"city": "SF"}'::jsonb, ARRAY['staff']), +('charlie', 45, 120000.50, '2024-11-20'::date, '{"city": "NY"}'::jsonb, ARRAY['admin']), +('diana', 25, 80000.00, '2025-03-01'::date, NULL::jsonb, ARRAY['user']), +('diana', 25, NULL::numeric, NULL::date, '{"city": "LA"}'::jsonb, NULL::text[]); + +-- --------------------------------------------------------- +-- 3. Execute unit tests... +-- --------------------------------------------------------- +DO $$ +DECLARE + res jsonb; + fields jsonb; + vals jsonb; + f record; + v record; +BEGIN + -- Profile the mock table with default options + res := supa_profile.profile_table('mock_users'::regclass, '{"min_cell_count": 2}'::jsonb); + + -- Verify tableStats + ASSERT res -> 'tableStats' ->> 'tableName' = 'mock_users', 'Table stats name mismatch'; + ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 5, 'Table stats row count mismatch'; + ASSERT (res -> 'tableStats' ->> 'columnCount')::int = 7, 'Table stats column count mismatch'; + + fields := res -> 'fields'; + vals := res -> 'values'; + + -- Loop through fields and check basic validation + FOR f IN SELECT * FROM jsonb_to_recordset(fields) AS ( + "columnName" text, "dataType" text, "nullable" boolean, + "nonNullCount" int, "nullCount" int, "distinctCount" int, + "minValue" text, "maxValue" text, "avgValue" numeric, + "median" numeric, "mode" text, "pattern" text + ) LOOP + IF f."columnName" = 'id' THEN + ASSERT f."nullable" = false, 'id should not be nullable'; + ASSERT f."nonNullCount" = 5, 'id non-null count mismatch'; + ASSERT f."distinctCount" = 5, 'id distinct count mismatch'; + ASSERT f."minValue" = '1', 'id min mismatch'; + ASSERT f."maxValue" = '5', 'id max mismatch'; + ASSERT f."avgValue" = 3.00, 'id avg mismatch'; + ASSERT f."median" = 3.00, 'id median mismatch'; + ASSERT f."pattern" = '-?\d+', 'id pattern mismatch'; + ELSIF f."columnName" = 'username' THEN + ASSERT f."nullable" = false, 'username should not be nullable'; + ASSERT f."nonNullCount" = 5, 'username non-null count mismatch'; + ASSERT f."distinctCount" = 4, 'username distinct count mismatch'; + ASSERT f."minValue" = 'alice', 'username min mismatch'; + ASSERT f."maxValue" = 'diana', 'username max mismatch'; + ASSERT f."mode" = 'diana', 'username mode mismatch (should be diana, frequency = 2)'; + ELSIF f."columnName" = 'age' THEN + ASSERT f."nullable" = true, 'age should be nullable'; + ASSERT f."nonNullCount" = 5, 'age non-null count mismatch'; + ASSERT f."distinctCount" = 3, 'age distinct count mismatch'; + ASSERT f."minValue" = '25', 'age min mismatch'; + ASSERT f."maxValue" = '45', 'age max mismatch'; + ASSERT f."median" = 25.00, 'age median mismatch'; + ASSERT f."mode" = '25', 'age mode mismatch (frequency = 3)'; + ELSIF f."columnName" = 'salary' THEN + ASSERT f."nullable" = true, 'salary should be nullable'; + ASSERT f."nonNullCount" = 4, 'salary non-null count mismatch'; + ASSERT f."nullCount" = 1, 'salary null count mismatch'; + ASSERT f."minValue" = '80000.00', 'salary min mismatch'; + ASSERT f."maxValue" = '120000.50', 'salary max mismatch'; + ELSIF f."columnName" = 'extra_info' THEN + -- Verify that skipped types like jsonb have null min/max but non-null and distinct counts + ASSERT f."minValue" IS NULL, 'jsonb should not have min'; + ASSERT f."maxValue" IS NULL, 'jsonb should not have max'; + ASSERT f."nonNullCount" = 4, 'jsonb non-null count mismatch'; + ASSERT f."distinctCount" = 3, 'jsonb distinct count mismatch'; + END IF; + END LOOP; + + -- Verify value distribution (we filtered with min_cell_count = 2) + -- Values expected to be frequent: age=25 (freq=3), salary=80000.00 (freq=2), username=diana (freq=2) + DECLARE + age_25_found boolean := false; + salary_80000_found boolean := false; + username_diana_found boolean := false; + BEGIN + FOR v IN SELECT * FROM jsonb_to_recordset(vals) AS ( + "columnName" text, "value" text, "frequency" int, "percent" numeric + ) LOOP + IF v."columnName" = 'age' AND v."value" = '25' THEN + ASSERT v."frequency" = 3, 'age 25 frequency mismatch'; + ASSERT v."percent" = 0.6000, 'age 25 percent mismatch'; + age_25_found := true; + ELSIF v."columnName" = 'salary' AND v."value" = '80000.00' THEN + ASSERT v."frequency" = 2, 'salary 80000 frequency mismatch'; + ASSERT v."percent" = 0.4000, 'salary 80000 percent mismatch'; + salary_80000_found := true; + ELSIF v."columnName" = 'username' AND v."value" = 'diana' THEN + ASSERT v."frequency" = 2, 'username diana frequency mismatch'; + ASSERT v."percent" = 0.4000, 'username diana percent mismatch'; + username_diana_found := true; + END IF; + END LOOP; + + ASSERT age_25_found, 'age 25 value distribution missing'; + ASSERT salary_80000_found, 'salary 80000 value distribution missing'; + ASSERT username_diana_found, 'username diana value distribution missing'; + END; + + -- Test limiting options + res := supa_profile.profile_table('mock_users'::regclass, '{"rows_per_table": 3}'::jsonb); + ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 3, 'rows_per_table limit option failed'; + ASSERT res ->> 'queryMode' = 'FAST', 'FAST queryMode mismatch for sampled table'; + + -- Test disabling numeric stats + res := supa_profile.profile_table('mock_users'::regclass, '{"calculate_numeric_stats": false}'::jsonb); + FOR f IN SELECT * FROM jsonb_to_recordset(res -> 'fields') AS ("columnName" text, "avgValue" numeric, "median" numeric) LOOP + IF f."columnName" = 'age' THEN + ASSERT f."avgValue" IS NULL, 'avgValue should be null when numeric stats are disabled'; + ASSERT f."median" IS NULL, 'median should be null when numeric stats are disabled'; + END IF; + END LOOP; + + -- Test disabling value distribution scan + res := supa_profile.profile_table('mock_users'::regclass, '{"scan_field_values": false}'::jsonb); + ASSERT jsonb_array_length(res -> 'values') = 0, 'values array should be empty when scan_field_values is false'; + + RAISE NOTICE '✅ All supa_profile unit tests passed successfully!'; +END; +$$; + +DROP TABLE IF EXISTS mock_users; +DROP SCHEMA supa_profile CASCADE; + +ROLLBACK; From 7dcd8c172e177c7e633d65825f819e54fa92501c Mon Sep 17 00:00:00 2001 From: Jesse <16379819+JesseVent@users.noreply.github.com> Date: Mon, 8 Jun 2026 19:23:31 +0930 Subject: [PATCH 5/7] feat: implement advanced database profiling optimizations --- supa_profile/README.md | 178 ++------ supa_profile/supa_profile--1.0.0.sql | 516 +++++++++++++++------- supa_profile/test_validation.sql | 636 +++++++++++++++++---------- 3 files changed, 798 insertions(+), 532 deletions(-) diff --git a/supa_profile/README.md b/supa_profile/README.md index bd4b4a0..ece4c2e 100644 --- a/supa_profile/README.md +++ b/supa_profile/README.md @@ -1,6 +1,6 @@ # supa_profile - PostgreSQL Database Table & Column Profiler -`supa_profile` is a relocatable, pure SQL PostgreSQL **Trusted Language Extension (TLE)** designed for native table and column-level statistical profiling. It introspects schema metadata, dynamically constructs profiling queries, and compiles a comprehensive statistical report (including counts, nullability, distinctness, min/max, string length distributions, averages, medians, percentiles, and frequent value distributions) returned directly as a single structured JSONB document. +`supa_profile` is a relocatable, pure SQL PostgreSQL **Trusted Language Extension (TLE)** designed for native table and column-level statistical profiling. It introspects schema metadata, dynamically constructs parallelized queries, and compiles a comprehensive statistical report (including counts, nullability, distinctness, min/max, string length distributions, averages, medians, percentiles, and frequent value distributions) returned directly as a single structured JSONB document. The JSONB output format is fully compatible with the **`ProfilingResult`** TypeScript interface used by frontend profiling dashboards. @@ -43,17 +43,43 @@ Introspects and profiles a physical table, view, or materialized view. - `max_distinct_values` (int, default: `100`): Maximum distinct values returned per column in the value distribution list. - `rows_per_table` (int, default: `0` = unlimited): Maximum rows to scan. If greater than zero, it wraps the source in a sampled subquery (`LIMIT N`). - `calculate_numeric_stats` (boolean, default: `true`): Compute average, median, 90th percentile, and 99th percentile for numeric fields. + - `sampling_method` (text, default: `'limit'`): Sampling strategy when limiting scans. Options: + - `'limit'`: Truncates scan to first $N$ rows (`LIMIT N`). Works on all table types, views, and foreign tables. + - `'system'`: Uses PostgreSQL native block-level sampling (`TABLESAMPLE SYSTEM (percentage)`). Extremely fast, but block-based. + - `'bernoulli'`: Uses PostgreSQL native row-level random sampling (`TABLESAMPLE BERNOULLI (percentage)`). Slower than system, but fully random. + - `sample_percentage` (numeric): Percentage of rows to sample (between `0` and `100`). Only used if `sampling_method` is set to `'system'` or `'bernoulli'`. + - `use_estimated_stats` (boolean, default: `false`): Enable instant catalog-based profiling. When `true`, it bypasses active table queries entirely and extracts statistics directly from Postgres catalogs (`pg_class.reltuples` and `pg_stats`). Excellent for tables with billions of rows (completes in < 5ms). * **Example:** ```sql + -- Profile using Bernoulli row-level sampling at 10% SELECT supa_profile.profile_table( 'public.users'::regclass, - '{"rows_per_table": 10000, "min_cell_count": 10}'::jsonb + '{"sampling_method": "bernoulli", "sample_percentage": 10.0}'::jsonb + ); + + -- Profile billions of rows instantly using pg_stats estimates + SELECT supa_profile.profile_table( + 'public.huge_analytics_log'::regclass, + '{"use_estimated_stats": true}'::jsonb ); ``` --- +## 🔍 Data Pattern Classifier + +The profiler includes a data classifier that samples column values and matches them against regular expression pattern groups to identify specific fields: +- `EMAIL`: Match handles like `user@corp.com`. +- `UUID`: Match UUID strings like `a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11`. +- `IPV4`: Match IP addresses like `192.168.1.1`. +- `URL`: Match URLs like `https://supabase.com`. +- `DATE`: Match ISO date formats. +- `NUMERIC`: Match integers and floats. +- `.*`: Fallback for generic text fields. + +--- + ## 📊 Output Schema (JSONB) The function returns a JSONB object containing: @@ -62,153 +88,7 @@ The function returns a JSONB object containing: - `values`: Value frequency distribution array. - `profiledAt`: Timestamp in UTC. - `durationMs`: Total duration of the profiling process. -- `queryMode`: Current query mode (`NORMAL` or `FAST` for sampled scans). - -### Field Properties: -| Property | Type | Description | -|---|---|---| -| `columnName` | `text` | Column identifier | -| `dataType` | `text` | Column type format | -| `nullable` | `boolean` | True if nulls are allowed | -| `nonNullCount` | `bigint` | Total count of non-null cells | -| `nullCount` | `bigint` | Total count of null cells | -| `distinctCount` | `bigint` | Total unique value count | -| `minValue` | `text` | Minimum value (omitted for JSON/bytes/arrays) | -| `maxValue` | `text` | Maximum value (omitted for JSON/bytes/arrays) | -| `avgValue` | `numeric` | Average value (numeric columns only) | -| `median` | `numeric` | Median value / P50 (numeric columns only) | -| `p90` | `numeric` | 90th percentile value (numeric columns only) | -| `p99` | `numeric` | 99th percentile value (numeric columns only) | -| `avgLength` | `numeric` | Average string length of text representation | -| `maxLength` | `bigint` | Maximum string length of text representation | -| `mode` | `text` | Most frequent value (omitted if `scan_field_values` is false) | -| `pattern` | `text` | Inferred regex pattern for type verification | - ---- - -## 📖 Example Walkthrough - -### 1. Create Sample Table -```sql -CREATE TABLE public.customers ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - age INT, - country VARCHAR(3) DEFAULT 'USA', - signup_date DATE -); - -INSERT INTO public.customers (name, age, country, signup_date) VALUES -('Alice', 25, 'USA', '2025-01-10'), -('Bob', 30, 'USA', '2025-02-15'), -('Charlie', 45, 'CAN', '2024-11-20'), -('Diana', 25, 'USA', '2025-03-01'), -('Ethan', NULL, 'GBR', NULL); -``` - -### 2. Profile the Table -```sql -SELECT jsonb_pretty(supa_profile.profile_table('public.customers'::regclass, '{"min_cell_count": 1}'::jsonb)); -``` - -### 3. Example JSONB Output -```json -{ - "fields": [ - { - "mode": "", - "pattern": "-?\\d+", - "dataType": "integer", - "nullable": false, - "nullCount": 0, - "avgLength": 1.0, - "columnName": "id", - "distinctCount": 5, - "maxLength": 1, - "nonNullCount": 5, - "tableName": "public.customers", - "avgValue": 3.0, - "median": 3.0, - "p90": 5.0, - "p99": 5.0, - "minValue": "1", - "maxValue": "5" - }, - { - "mode": "", - "pattern": ".*", - "dataType": "text", - "nullable": false, - "nullCount": 0, - "avgLength": 4.6, - "columnName": "name", - "distinctCount": 5, - "maxLength": 7, - "nonNullCount": 5, - "tableName": "public.customers", - "minValue": "Alice", - "maxValue": "Ethan" - }, - { - "mode": "25", - "pattern": "-?\\d+", - "dataType": "integer", - "nullable": true, - "nullCount": 1, - "avgLength": 1.6, - "columnName": "age", - "distinctCount": 3, - "maxLength": 2, - "nonNullCount": 4, - "tableName": "public.customers", - "avgValue": 31.25, - "median": 30.0, - "p90": 45.0, - "p99": 45.0, - "minValue": "25", - "maxValue": "45" - }, - { - "mode": "USA", - "pattern": ".*", - "dataType": "character varying(3)", - "nullable": true, - "nullCount": 0, - "avgLength": 3.0, - "columnName": "country", - "distinctCount": 3, - "maxLength": 3, - "nonNullCount": 5, - "tableName": "public.customers", - "minValue": "CAN", - "maxValue": "USA" - } - ], - "values": [ - { - "value": "25", - "percent": 0.4000, - "frequency": 2, - "columnName": "age" - }, - { - "value": "USA", - "percent": 0.6000, - "frequency": 3, - "columnName": "country" - } - ], - "queryMode": "NORMAL", - "durationMs": 4.32, - "profiledAt": "2026-06-08T18:00:00Z", - "tableStats": { - "rowCount": 5, - "selected": false, - "tableName": "public.customers", - "columnCount": 5 - } -} -``` +- `queryMode`: Current query mode (`NORMAL`, `FAST` for sampled scans, or `ESTIMATED` for catalog statistics). --- diff --git a/supa_profile/supa_profile--1.0.0.sql b/supa_profile/supa_profile--1.0.0.sql index e99fc8e..88a4737 100644 --- a/supa_profile/supa_profile--1.0.0.sql +++ b/supa_profile/supa_profile--1.0.0.sql @@ -1,13 +1,70 @@ -- supa_profile extension database schema definition -- --------------------------------------------------------------------- --- 1. PATTERN INFERENCE HELPER +-- 1. PATTERN CLASSIFIER HELPER -- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.classify_values(vals text[]) +RETURNS text AS $$ +DECLARE + val text; + total int := 0; + uuid_cnt int := 0; + email_cnt int := 0; + ipv4_cnt int := 0; + url_cnt int := 0; + numeric_cnt int := 0; + date_cnt int := 0; +BEGIN + IF vals IS NULL OR array_length(vals, 1) IS NULL THEN + RETURN '.*'; + END IF; + + total := array_length(vals, 1); + + FOREACH val IN ARRAY vals LOOP + IF val IS NULL OR val = 'NULL' OR val = '' THEN + total := total - 1; + CONTINUE; + END IF; + + IF val ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' THEN + uuid_cnt := uuid_cnt + 1; + ELSIF val ~* '^[a-z0-9.!#$%&''*+/=?^_`{|}~-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$' THEN + email_cnt := email_cnt + 1; + ELSIF val ~* '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' THEN + ipv4_cnt := ipv4_cnt + 1; + ELSIF val ~* '^https?://[^\s/$.?#].[^\s]*$' THEN + url_cnt := url_cnt + 1; + ELSIF val ~* '^-?\d+$' THEN + numeric_cnt := numeric_cnt + 1; + ELSIF val ~* '^\d{4}-\d{2}-\d{2}$' OR val ~* '^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}' THEN + date_cnt := date_cnt + 1; + END IF; + END LOOP; + + IF total <= 0 THEN + RETURN '.*'; + END IF; + + -- If 80% or more match a pattern, return it + IF (uuid_cnt::numeric / total) >= 0.8 THEN RETURN 'UUID'; END IF; + IF (email_cnt::numeric / total) >= 0.8 THEN RETURN 'EMAIL'; END IF; + IF (ipv4_cnt::numeric / total) >= 0.8 THEN RETURN 'IPV4'; END IF; + IF (url_cnt::numeric / total) >= 0.8 THEN RETURN 'URL'; END IF; + IF (date_cnt::numeric / total) >= 0.8 THEN RETURN 'DATE'; END IF; + IF (numeric_cnt::numeric / total) >= 0.8 THEN RETURN 'NUMERIC'; END IF; + + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + CREATE OR REPLACE FUNCTION @extschema@.infer_pattern(data_type text) RETURNS text AS $$ DECLARE t text := upper(data_type); BEGIN + IF t = 'UUID' THEN RETURN 'UUID'; END IF; + IF t = 'INET' THEN RETURN 'IPV4'; END IF; IF t LIKE '%DATE%' THEN RETURN 'YYYY-MM-DD'; END IF; IF t LIKE '%TIME%' THEN RETURN 'HH:MM:SS'; END IF; IF t LIKE '%INT%' THEN RETURN '-?\d+'; END IF; @@ -34,6 +91,9 @@ DECLARE max_distinct_values int; rows_per_table int; calculate_numeric_stats boolean; + use_estimated_stats boolean; + sampling_method text; + sample_percentage numeric; -- Execution metadata start_time timestamptz; @@ -41,11 +101,13 @@ DECLARE total_rows bigint; column_count int; - -- Table/Schema name + -- Resolving names + schema_name_val text; + table_name_val text; full_table_name text; from_clause text; - -- Column loop + -- Column loops col record; is_numeric boolean; skip_min_max boolean; @@ -71,18 +133,34 @@ DECLARE median_val numeric; p90_val numeric; p99_val numeric; + pattern_val text; - -- Mode and Value Distribution - val_query text; + -- Value Distribution (Unified Parallelized Union Query) + val_query text := ''; val_rec record; val_obj jsonb; - mode_val text; - mode_found boolean; field_obj jsonb; + mode_val text; + sample_vals text[]; + + -- Estimated Stats variables + null_fraction numeric; + distinct_stat numeric; + width_stat int; + mcv_vals text[]; + mcf_freqs numeric[]; + i int; BEGIN start_time := clock_timestamp(); full_table_name := target_table::text; + -- Retrieve resolved schema and table name + SELECT n.nspname, c.relname + INTO schema_name_val, table_name_val + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target_table; + -- Ensure options is not null options := coalesce(options, '{}'::jsonb); @@ -92,179 +170,297 @@ BEGIN max_distinct_values := coalesce((options ->> 'max_distinct_values')::int, 100); rows_per_table := coalesce((options ->> 'rows_per_table')::int, 0); calculate_numeric_stats := coalesce((options ->> 'calculate_numeric_stats')::boolean, true); - - -- Build FROM clause with optional sampling/limit - IF rows_per_table > 0 THEN - from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; - ELSE - from_clause := full_table_name; - END IF; + use_estimated_stats := coalesce((options ->> 'use_estimated_stats')::boolean, false); + sampling_method := coalesce(lower(options ->> 'sampling_method'), 'limit'); + sample_percentage := (options ->> 'sample_percentage')::numeric; -- Retrieve column count SELECT COUNT(*)::int INTO column_count FROM pg_attribute WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped; - -- Build SELECT expressions for column stats - FOR col IN - SELECT - attname AS column_name, - format_type(atttypid, atttypmod) AS data_type, - NOT attnotnull AS is_nullable - FROM pg_attribute - WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped - ORDER BY attnum - LOOP - -- Determine if type is numeric - is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') - OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); - - -- Determine if type does not support MIN/MAX - skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot'); - - -- Basic column stats - select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); - select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); - - -- MIN/MAX - IF skip_min_max THEN - select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); - select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); - ELSE - select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); - select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + -- ----------------------------------------------------------------- + -- BRANCH A: CATALOG-BASED ESTIMATED PROFILING (Fast path) + -- ----------------------------------------------------------------- + IF use_estimated_stats THEN + -- Get estimated row count from pg_class + SELECT reltuples::bigint INTO total_rows + FROM pg_class + WHERE oid = target_table; + + -- Guard against negative estimates + IF total_rows < 0 THEN + total_rows := 0; END IF; - -- Avg and Max string length - select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); - select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Check if we have stats in pg_stats + SELECT + null_frac, n_distinct, avg_width, + most_common_vals::text::text[], most_common_freqs + INTO null_fraction, distinct_stat, width_stat, mcv_vals, mcf_freqs + FROM pg_stats + WHERE schemaname = schema_name_val AND tablename = table_name_val AND attname = col.column_name; + + IF null_fraction IS NOT NULL THEN + non_null_count := (total_rows * (1.0 - null_fraction))::bigint; + + -- Calculate distinct count + IF distinct_stat < 0 THEN + distinct_count := (abs(distinct_stat) * total_rows)::bigint; + ELSE + distinct_count := distinct_stat::bigint; + END IF; + + avg_len := width_stat::numeric; + max_len := width_stat::bigint; -- fallback estimate + ELSE + -- Fallback defaults if ANALYZE hasn't run + non_null_count := total_rows; + distinct_count := 0; + avg_len := 0; + max_len := 0; + mcv_vals := NULL; + mcf_freqs := NULL; + END IF; + + -- Build value distribution list from MCV/MCF + mode_val := ''; + IF scan_field_values AND mcv_vals IS NOT NULL AND mcf_freqs IS NOT NULL THEN + FOR i IN 1..array_length(mcv_vals, 1) LOOP + IF i = 1 THEN + mode_val := mcv_vals[1]; + END IF; + + val_obj := jsonb_build_object( + 'columnName', col.column_name, + 'value', mcv_vals[i], + 'frequency', (mcf_freqs[i] * total_rows)::bigint, + 'percent', mcf_freqs[i]::numeric(10,4) + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END IF; - -- Numeric stats (AVG, MEDIAN, P90, P99) - IF calculate_numeric_stats AND is_numeric THEN - select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); - select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); - select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); - select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + -- Classify pattern from MCV values + IF mcv_vals IS NOT NULL AND array_length(mcv_vals, 1) > 0 THEN + pattern_val := @extschema@.classify_values(mcv_vals); + ELSE + pattern_val := @extschema@.infer_pattern(col.data_type); + END IF; + + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', avg_len, + 'maxLength', max_len, + 'mode', mode_val, + 'pattern', pattern_val + ); + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + + -- ----------------------------------------------------------------- + -- BRANCH B: DYNAMIC QUERY-BASED EXACT PROFILING (Thorough path) + -- ----------------------------------------------------------------- + ELSE + -- Build FROM clause with dynamic sampling options + IF sampling_method IN ('system', 'bernoulli') AND sample_percentage > 0 AND sample_percentage <= 100 THEN + from_clause := full_table_name || ' TABLESAMPLE ' || upper(sampling_method) || ' (' || sample_percentage || ')'; + ELSIF rows_per_table > 0 THEN + from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; ELSE - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + from_clause := full_table_name; END IF; - END LOOP; - -- Execute main statistics query - query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; - EXECUTE query_str INTO result_row; - result_json := to_jsonb(result_row); - total_rows := (result_json ->> 'total_rows')::bigint; - - -- Execute value distribution query per column if enabled - IF scan_field_values AND total_rows > 0 THEN + -- Build SELECT expressions for column stats FOR col IN - SELECT attname AS column_name + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable FROM pg_attribute WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped ORDER BY attnum LOOP - val_query := 'SELECT ' || - quote_literal(col.column_name) || ' AS column_name, ' || - 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || - 'COUNT(*) AS frequency ' || - 'FROM ' || from_clause || ' ' || - 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || - 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || - 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || - 'ORDER BY COUNT(*) DESC ' || - 'LIMIT ' || max_distinct_values; - - FOR val_rec IN EXECUTE val_query LOOP - val_obj := jsonb_build_object( - 'columnName', val_rec.column_name, - 'value', val_rec.value, - 'frequency', val_rec.frequency, - 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END - ); - values_array := values_array || jsonb_build_array(val_obj); - END LOOP; + -- Determine if type is numeric + is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') + OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); + + -- Determine if type does not support MIN/MAX + skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot|uuid'); + + -- Basic column stats + select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); + select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); + + -- MIN/MAX + IF skip_min_max THEN + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); + ELSE + select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + END IF; + + -- Avg and Max string length + select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); + select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + + -- Numeric stats (AVG, MEDIAN, P90, P99) + IF calculate_numeric_stats AND is_numeric THEN + select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + ELSE + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + END IF; END LOOP; - END IF; - -- Build final fields array - FOR col IN - SELECT - attname AS column_name, - format_type(atttypid, atttypmod) AS data_type, - NOT attnotnull AS is_nullable - FROM pg_attribute - WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped - ORDER BY attnum - LOOP - non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; - distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; - min_val := result_json ->> (col.column_name || '__min'); - max_val := result_json ->> (col.column_name || '__max'); - avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; - max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; - avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; - median_val := (result_json ->> (col.column_name || '__median'))::numeric; - p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; - p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; - - -- Find mode for this column if we scanned values - mode_val := ''; - IF scan_field_values THEN - -- Find the first value in values_array for this column - -- Since values_array is sorted by frequency DESC, the first one is the mode - DECLARE - temp_val jsonb; - BEGIN - FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP - IF temp_val ->> 'columnName' = col.column_name THEN - mode_val := temp_val ->> 'value'; - EXIT; - END IF; + -- Execute main statistics query + query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; + EXECUTE query_str INTO result_row; + result_json := to_jsonb(result_row); + total_rows := (result_json ->> 'total_rows')::bigint; + + -- Build unified parallelized value distribution query if enabled + IF scan_field_values AND total_rows > 0 THEN + FOR col IN + SELECT attname AS column_name + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + IF val_query != '' THEN + val_query := val_query || ' UNION ALL '; + END IF; + val_query := val_query || + 'SELECT * FROM (' || + 'SELECT ' || + quote_literal(col.column_name) || ' AS column_name, ' || + 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || + 'COUNT(*) AS frequency ' || + 'FROM ' || from_clause || ' ' || + 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || + 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || + 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || + 'ORDER BY COUNT(*) DESC ' || + 'LIMIT ' || max_distinct_values || ') q_' || quote_ident(col.column_name); + END LOOP; + + IF val_query != '' THEN + FOR val_rec IN EXECUTE val_query LOOP + val_obj := jsonb_build_object( + 'columnName', val_rec.column_name, + 'value', val_rec.value, + 'frequency', val_rec.frequency, + 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END + ); + values_array := values_array || jsonb_build_array(val_obj); END LOOP; - END; + END IF; END IF; - -- Construct field profile object matching FieldProfile interface - field_obj := jsonb_build_object( - 'tableName', full_table_name, - 'columnName', col.column_name, - 'dataType', col.data_type, - 'nullable', col.is_nullable, - 'nonNullCount', non_null_count, - 'nullCount', total_rows - non_null_count, - 'distinctCount', distinct_count, - 'avgLength', coalesce(avg_len, 0), - 'maxLength', coalesce(max_len, 0), - 'mode', mode_val, - 'pattern', @extschema@.infer_pattern(col.data_type) - ); - - -- Add optional numeric and min/max stats - IF min_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('minValue', min_val); - END IF; - IF max_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('maxValue', max_val); - END IF; - IF avg_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('avgValue', avg_val); - END IF; - IF median_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('median', median_val); - END IF; - IF p90_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('p90', p90_val); - END IF; - IF p99_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('p99', p99_val); - END IF; + -- Build final fields array + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; + distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; + min_val := result_json ->> (col.column_name || '__min'); + max_val := result_json ->> (col.column_name || '__max'); + avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; + max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; + avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; + median_val := (result_json ->> (col.column_name || '__median'))::numeric; + p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; + p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; - fields_array := fields_array || jsonb_build_array(field_obj); - END LOOP; + -- Find mode for this column if we scanned values + mode_val := ''; + IF scan_field_values THEN + DECLARE + temp_val jsonb; + BEGIN + FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP + IF temp_val ->> 'columnName' = col.column_name THEN + mode_val := temp_val ->> 'value'; + EXIT; + END IF; + END LOOP; + END; + END IF; + + -- Collect a small sample of values to classify the data pattern + EXECUTE 'SELECT array_agg(' || quote_ident(col.column_name) || '::text) FROM (SELECT ' || quote_ident(col.column_name) || ' FROM ' || from_clause || ' WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL LIMIT 50) t' INTO sample_vals; + + IF sample_vals IS NOT NULL AND array_length(sample_vals, 1) > 0 THEN + pattern_val := @extschema@.classify_values(sample_vals); + ELSE + pattern_val := @extschema@.infer_pattern(col.data_type); + END IF; + + -- Construct field profile object matching FieldProfile interface + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', coalesce(avg_len, 0), + 'maxLength', coalesce(max_len, 0), + 'mode', mode_val, + 'pattern', pattern_val + ); + + -- Add optional numeric and min/max stats + IF min_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('minValue', min_val); + END IF; + IF max_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('maxValue', max_val); + END IF; + IF avg_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('avgValue', avg_val); + END IF; + IF median_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('median', median_val); + END IF; + IF p90_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p90', p90_val); + END IF; + IF p99_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p99', p99_val); + END IF; + + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + END IF; duration_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000; @@ -280,7 +476,11 @@ BEGIN 'values', values_array, 'profiledAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 'durationMs', round(duration_ms, 2), - 'queryMode', CASE WHEN rows_per_table > 0 THEN 'FAST' ELSE 'NORMAL' END + 'queryMode', CASE + WHEN use_estimated_stats THEN 'ESTIMATED' + WHEN rows_per_table > 0 OR sample_percentage > 0 THEN 'FAST' + ELSE 'NORMAL' + END ); END; $$ LANGUAGE plpgsql VOLATILE STRICT; diff --git a/supa_profile/test_validation.sql b/supa_profile/test_validation.sql index 5f58cbf..c349b15 100644 --- a/supa_profile/test_validation.sql +++ b/supa_profile/test_validation.sql @@ -10,12 +10,67 @@ BEGIN; -- --------------------------------------------------------- CREATE SCHEMA IF NOT EXISTS supa_profile; --- Copy of helper functions for standalone execution verification +CREATE OR REPLACE FUNCTION supa_profile.classify_values(vals text[]) +RETURNS text AS $$ +DECLARE + val text; + total int := 0; + uuid_cnt int := 0; + email_cnt int := 0; + ipv4_cnt int := 0; + url_cnt int := 0; + numeric_cnt int := 0; + date_cnt int := 0; +BEGIN + IF vals IS NULL OR array_length(vals, 1) IS NULL THEN + RETURN '.*'; + END IF; + + total := array_length(vals, 1); + + FOREACH val IN ARRAY vals LOOP + IF val IS NULL OR val = 'NULL' OR val = '' THEN + total := total - 1; + CONTINUE; + END IF; + + IF val ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' THEN + uuid_cnt := uuid_cnt + 1; + ELSIF val ~* '^[a-z0-9.!#$%&''*+/=?^_`{|}~-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$' THEN + email_cnt := email_cnt + 1; + ELSIF val ~* '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' THEN + ipv4_cnt := ipv4_cnt + 1; + ELSIF val ~* '^https?://[^\s/$.?#].[^\s]*$' THEN + url_cnt := url_cnt + 1; + ELSIF val ~* '^-?\d+$' THEN + numeric_cnt := numeric_cnt + 1; + ELSIF val ~* '^\d{4}-\d{2}-\d{2}$' OR val ~* '^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}' THEN + date_cnt := date_cnt + 1; + END IF; + END LOOP; + + IF total <= 0 THEN + RETURN '.*'; + END IF; + + IF (uuid_cnt::numeric / total) >= 0.8 THEN RETURN 'UUID'; END IF; + IF (email_cnt::numeric / total) >= 0.8 THEN RETURN 'EMAIL'; END IF; + IF (ipv4_cnt::numeric / total) >= 0.8 THEN RETURN 'IPV4'; END IF; + IF (url_cnt::numeric / total) >= 0.8 THEN RETURN 'URL'; END IF; + IF (date_cnt::numeric / total) >= 0.8 THEN RETURN 'DATE'; END IF; + IF (numeric_cnt::numeric / total) >= 0.8 THEN RETURN 'NUMERIC'; END IF; + + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + CREATE OR REPLACE FUNCTION supa_profile.infer_pattern(data_type text) RETURNS text AS $$ DECLARE t text := upper(data_type); BEGIN + IF t = 'UUID' THEN RETURN 'UUID'; END IF; + IF t = 'INET' THEN RETURN 'IPV4'; END IF; IF t LIKE '%DATE%' THEN RETURN 'YYYY-MM-DD'; END IF; IF t LIKE '%TIME%' THEN RETURN 'HH:MM:SS'; END IF; IF t LIKE '%INT%' THEN RETURN '-?\d+'; END IF; @@ -38,6 +93,9 @@ DECLARE max_distinct_values int; rows_per_table int; calculate_numeric_stats boolean; + use_estimated_stats boolean; + sampling_method text; + sample_percentage numeric; -- Execution metadata start_time timestamptz; @@ -45,11 +103,13 @@ DECLARE total_rows bigint; column_count int; - -- Table/Schema name + -- Resolving names + schema_name_val text; + table_name_val text; full_table_name text; from_clause text; - -- Column loop + -- Column loops col record; is_numeric boolean; skip_min_max boolean; @@ -75,18 +135,34 @@ DECLARE median_val numeric; p90_val numeric; p99_val numeric; + pattern_val text; - -- Mode and Value Distribution - val_query text; + -- Value Distribution (Unified Parallelized Union Query) + val_query text := ''; val_rec record; val_obj jsonb; - mode_val text; - mode_found boolean; field_obj jsonb; + mode_val text; + sample_vals text[]; + + -- Estimated Stats variables + null_fraction numeric; + distinct_stat numeric; + width_stat int; + mcv_vals text[]; + mcf_freqs numeric[]; + i int; BEGIN start_time := clock_timestamp(); full_table_name := target_table::text; + -- Retrieve resolved schema and table name + SELECT n.nspname, c.relname + INTO schema_name_val, table_name_val + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target_table; + -- Ensure options is not null options := coalesce(options, '{}'::jsonb); @@ -96,179 +172,296 @@ BEGIN max_distinct_values := coalesce((options ->> 'max_distinct_values')::int, 100); rows_per_table := coalesce((options ->> 'rows_per_table')::int, 0); calculate_numeric_stats := coalesce((options ->> 'calculate_numeric_stats')::boolean, true); - - -- Build FROM clause with optional sampling/limit - IF rows_per_table > 0 THEN - from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; - ELSE - from_clause := full_table_name; - END IF; + use_estimated_stats := coalesce((options ->> 'use_estimated_stats')::boolean, false); + sampling_method := coalesce(lower(options ->> 'sampling_method'), 'limit'); + sample_percentage := (options ->> 'sample_percentage')::numeric; -- Retrieve column count SELECT COUNT(*)::int INTO column_count FROM pg_attribute WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped; - -- Build SELECT expressions for column stats - FOR col IN - SELECT - attname AS column_name, - format_type(atttypid, atttypmod) AS data_type, - NOT attnotnull AS is_nullable - FROM pg_attribute - WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped - ORDER BY attnum - LOOP - -- Determine if type is numeric - is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') - OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); - - -- Determine if type does not support MIN/MAX - skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot'); - - -- Basic column stats - select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); - select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); - - -- MIN/MAX - IF skip_min_max THEN - select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); - select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); - ELSE - select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); - select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + -- ----------------------------------------------------------------- + -- BRANCH A: CATALOG-BASED ESTIMATED PROFILING (Fast path) + -- ----------------------------------------------------------------- + IF use_estimated_stats THEN + -- Get estimated row count from pg_class + SELECT reltuples::bigint INTO total_rows + FROM pg_class + WHERE oid = target_table; + + -- Guard against negative estimates + IF total_rows < 0 THEN + total_rows := 0; END IF; - -- Avg and Max string length - select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); - select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Check if we have stats in pg_stats + SELECT + null_frac, n_distinct, avg_width, + most_common_vals::text::text[], most_common_freqs + INTO null_fraction, distinct_stat, width_stat, mcv_vals, mcf_freqs + FROM pg_stats + WHERE schemaname = schema_name_val AND tablename = table_name_val AND attname = col.column_name; + + IF null_fraction IS NOT NULL THEN + non_null_count := (total_rows * (1.0 - null_fraction))::bigint; + + -- Calculate distinct count + IF distinct_stat < 0 THEN + distinct_count := (abs(distinct_stat) * total_rows)::bigint; + ELSE + distinct_count := distinct_stat::bigint; + END IF; + + avg_len := width_stat::numeric; + max_len := width_stat::bigint; + ELSE + non_null_count := total_rows; + distinct_count := 0; + avg_len := 0; + max_len := 0; + mcv_vals := NULL; + mcf_freqs := NULL; + END IF; - -- Numeric stats (AVG, MEDIAN, P90, P99) - IF calculate_numeric_stats AND is_numeric THEN - select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); - select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); - select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); - select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + -- Build value distribution list from MCV/MCF + mode_val := ''; + IF scan_field_values AND mcv_vals IS NOT NULL AND mcf_freqs IS NOT NULL THEN + FOR i IN 1..array_length(mcv_vals, 1) LOOP + IF i = 1 THEN + mode_val := mcv_vals[1]; + END IF; + + val_obj := jsonb_build_object( + 'columnName', col.column_name, + 'value', mcv_vals[i], + 'frequency', (mcf_freqs[i] * total_rows)::bigint, + 'percent', mcf_freqs[i]::numeric(10,4) + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END IF; + + -- Classify pattern from MCV values + IF mcv_vals IS NOT NULL AND array_length(mcv_vals, 1) > 0 THEN + pattern_val := supa_profile.classify_values(mcv_vals); + ELSE + pattern_val := supa_profile.infer_pattern(col.data_type); + END IF; + + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', avg_len, + 'maxLength', max_len, + 'mode', mode_val, + 'pattern', pattern_val + ); + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + + -- ----------------------------------------------------------------- + -- BRANCH B: DYNAMIC QUERY-BASED EXACT PROFILING (Thorough path) + -- ----------------------------------------------------------------- + ELSE + -- Build FROM clause with dynamic sampling options + IF sampling_method IN ('system', 'bernoulli') AND sample_percentage > 0 AND sample_percentage <= 100 THEN + from_clause := full_table_name || ' TABLESAMPLE ' || upper(sampling_method) || ' (' || sample_percentage || ')'; + ELSIF rows_per_table > 0 THEN + from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; ELSE - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); - select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + from_clause := full_table_name; END IF; - END LOOP; - - -- Execute main statistics query - query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; - EXECUTE query_str INTO result_row; - result_json := to_jsonb(result_row); - total_rows := (result_json ->> 'total_rows')::bigint; - -- Execute value distribution query per column if enabled - IF scan_field_values AND total_rows > 0 THEN + -- Build SELECT expressions for column stats FOR col IN - SELECT attname AS column_name + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable FROM pg_attribute WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped ORDER BY attnum LOOP - val_query := 'SELECT ' || - quote_literal(col.column_name) || ' AS column_name, ' || - 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || - 'COUNT(*) AS frequency ' || - 'FROM ' || from_clause || ' ' || - 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || - 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || - 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || - 'ORDER BY COUNT(*) DESC ' || - 'LIMIT ' || max_distinct_values; - - FOR val_rec IN EXECUTE val_query LOOP - val_obj := jsonb_build_object( - 'columnName', val_rec.column_name, - 'value', val_rec.value, - 'frequency', val_rec.frequency, - 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END - ); - values_array := values_array || jsonb_build_array(val_obj); - END LOOP; + -- Determine if type is numeric + is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') + OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); + + -- Determine if type does not support MIN/MAX + skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot|uuid'); + + -- Basic column stats + select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); + select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); + + -- MIN/MAX + IF skip_min_max THEN + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); + ELSE + select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + END IF; + + -- Avg and Max string length + select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); + select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + + -- Numeric stats (AVG, MEDIAN, P90, P99) + IF calculate_numeric_stats AND is_numeric THEN + select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + ELSE + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + END IF; END LOOP; - END IF; - -- Build final fields array - FOR col IN - SELECT - attname AS column_name, - format_type(atttypid, atttypmod) AS data_type, - NOT attnotnull AS is_nullable - FROM pg_attribute - WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped - ORDER BY attnum - LOOP - non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; - distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; - min_val := result_json ->> (col.column_name || '__min'); - max_val := result_json ->> (col.column_name || '__max'); - avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; - max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; - avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; - median_val := (result_json ->> (col.column_name || '__median'))::numeric; - p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; - p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; - - -- Find mode for this column if we scanned values - mode_val := ''; - IF scan_field_values THEN - -- Find the first value in values_array for this column - -- Since values_array is sorted by frequency DESC, the first one is the mode - DECLARE - temp_val jsonb; - BEGIN - FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP - IF temp_val ->> 'columnName' = col.column_name THEN - mode_val := temp_val ->> 'value'; - EXIT; - END IF; + -- Execute main statistics query + query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; + EXECUTE query_str INTO result_row; + result_json := to_jsonb(result_row); + total_rows := (result_json ->> 'total_rows')::bigint; + + -- Build unified parallelized value distribution query if enabled + IF scan_field_values AND total_rows > 0 THEN + FOR col IN + SELECT attname AS column_name + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + IF val_query != '' THEN + val_query := val_query || ' UNION ALL '; + END IF; + val_query := val_query || + 'SELECT * FROM (' || + 'SELECT ' || + quote_literal(col.column_name) || ' AS column_name, ' || + 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || + 'COUNT(*) AS frequency ' || + 'FROM ' || from_clause || ' ' || + 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || + 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || + 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || + 'ORDER BY COUNT(*) DESC ' || + 'LIMIT ' || max_distinct_values || ') q_' || quote_ident(col.column_name); + END LOOP; + + IF val_query != '' THEN + FOR val_rec IN EXECUTE val_query LOOP + val_obj := jsonb_build_object( + 'columnName', val_rec.column_name, + 'value', val_rec.value, + 'frequency', val_rec.frequency, + 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END + ); + values_array := values_array || jsonb_build_array(val_obj); END LOOP; - END; + END IF; END IF; - -- Construct field profile object matching FieldProfile interface - field_obj := jsonb_build_object( - 'tableName', full_table_name, - 'columnName', col.column_name, - 'dataType', col.data_type, - 'nullable', col.is_nullable, - 'nonNullCount', non_null_count, - 'nullCount', total_rows - non_null_count, - 'distinctCount', distinct_count, - 'avgLength', coalesce(avg_len, 0), - 'maxLength', coalesce(max_len, 0), - 'mode', mode_val, - 'pattern', supa_profile.infer_pattern(col.data_type) - ); - - -- Add optional numeric and min/max stats - IF min_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('minValue', min_val); - END IF; - IF max_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('maxValue', max_val); - END IF; - IF avg_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('avgValue', avg_val); - END IF; - IF median_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('median', median_val); - END IF; - IF p90_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('p90', p90_val); - END IF; - IF p99_val IS NOT NULL THEN - field_obj := field_obj || jsonb_build_object('p99', p99_val); - END IF; + -- Build final fields array + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; + distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; + min_val := result_json ->> (col.column_name || '__min'); + max_val := result_json ->> (col.column_name || '__max'); + avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; + max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; + avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; + median_val := (result_json ->> (col.column_name || '__median'))::numeric; + p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; + p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; + + -- Find mode for this column if we scanned values + mode_val := ''; + IF scan_field_values THEN + DECLARE + temp_val jsonb; + BEGIN + FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP + IF temp_val ->> 'columnName' = col.column_name THEN + mode_val := temp_val ->> 'value'; + EXIT; + END IF; + END LOOP; + END; + END IF; - fields_array := fields_array || jsonb_build_array(field_obj); - END LOOP; + -- Collect a small sample of values to classify the data pattern + EXECUTE 'SELECT array_agg(' || quote_ident(col.column_name) || '::text) FROM (SELECT ' || quote_ident(col.column_name) || ' FROM ' || from_clause || ' WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL LIMIT 50) t' INTO sample_vals; + + IF sample_vals IS NOT NULL AND array_length(sample_vals, 1) > 0 THEN + pattern_val := supa_profile.classify_values(sample_vals); + ELSE + pattern_val := supa_profile.infer_pattern(col.data_type); + END IF; + + -- Construct field profile object matching FieldProfile interface + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', coalesce(avg_len, 0), + 'maxLength', coalesce(max_len, 0), + 'mode', mode_val, + 'pattern', pattern_val + ); + + -- Add optional numeric and min/max stats + IF min_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('minValue', min_val); + END IF; + IF max_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('maxValue', max_val); + END IF; + IF avg_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('avgValue', avg_val); + END IF; + IF median_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('median', median_val); + END IF; + IF p90_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p90', p90_val); + END IF; + IF p99_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p99', p99_val); + END IF; + + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + END IF; duration_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000; @@ -284,31 +477,37 @@ BEGIN 'values', values_array, 'profiledAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 'durationMs', round(duration_ms, 2), - 'queryMode', CASE WHEN rows_per_table > 0 THEN 'FAST' ELSE 'NORMAL' END + 'queryMode', CASE + WHEN use_estimated_stats THEN 'ESTIMATED' + WHEN rows_per_table > 0 OR sample_percentage > 0 THEN 'FAST' + ELSE 'NORMAL' + END ); END; $$ LANGUAGE plpgsql VOLATILE STRICT; -- --------------------------------------------------------- --- 2. Create mock tables and populate data... +-- 2. Create mock table and populate data... -- --------------------------------------------------------- CREATE TEMP TABLE mock_users ( id serial PRIMARY KEY, username text NOT NULL, + email text, + uuid_val uuid, + ip_val text, + url_val text, age int, salary numeric(10, 2), - signup_date date, - extra_info jsonb, - tags text[] + signup_date date ); -INSERT INTO mock_users (username, age, salary, signup_date, extra_info, tags) VALUES -('alice', 25, 80000.00, '2025-01-10'::date, '{"city": "NY"}'::jsonb, ARRAY['admin', 'staff']), -('bob', 30, 95000.00, '2025-02-15'::date, '{"city": "SF"}'::jsonb, ARRAY['staff']), -('charlie', 45, 120000.50, '2024-11-20'::date, '{"city": "NY"}'::jsonb, ARRAY['admin']), -('diana', 25, 80000.00, '2025-03-01'::date, NULL::jsonb, ARRAY['user']), -('diana', 25, NULL::numeric, NULL::date, '{"city": "LA"}'::jsonb, NULL::text[]); +INSERT INTO mock_users (username, email, uuid_val, ip_val, url_val, age, salary, signup_date) VALUES +('alice', 'alice@corp.com', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::uuid, '192.168.1.1', 'https://supabase.com', 25, 80000.00, '2025-01-10'::date), +('bob', 'bob@corp.com', 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22'::uuid, '192.168.1.2', 'https://google.com', 30, 95000.00, '2025-02-15'::date), +('charlie', 'charlie@gmail.com', 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33'::uuid, '10.0.0.1', 'https://github.com', 45, 120000.50, '2024-11-20'::date), +('diana', 'diana@corp.com', 'd0eebc99-9c0b-4ef8-bb6d-6bb9bd380a44'::uuid, '10.0.0.2', 'https://database.dev', 25, 80000.00, '2025-03-01'::date), +('diana', 'diana@corp.com', 'e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a55'::uuid, '10.0.0.3', 'https://database.dev', 25, NULL::numeric, NULL::date); -- --------------------------------------------------------- -- 3. Execute unit tests... @@ -327,12 +526,12 @@ BEGIN -- Verify tableStats ASSERT res -> 'tableStats' ->> 'tableName' = 'mock_users', 'Table stats name mismatch'; ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 5, 'Table stats row count mismatch'; - ASSERT (res -> 'tableStats' ->> 'columnCount')::int = 7, 'Table stats column count mismatch'; + ASSERT (res -> 'tableStats' ->> 'columnCount')::int = 9, 'Table stats column count mismatch'; fields := res -> 'fields'; vals := res -> 'values'; - -- Loop through fields and check basic validation + -- Loop through fields and check pattern classifiers and basic validations FOR f IN SELECT * FROM jsonb_to_recordset(fields) AS ( "columnName" text, "dataType" text, "nullable" boolean, "nonNullCount" int, "nullCount" int, "distinctCount" int, @@ -343,89 +542,76 @@ BEGIN ASSERT f."nullable" = false, 'id should not be nullable'; ASSERT f."nonNullCount" = 5, 'id non-null count mismatch'; ASSERT f."distinctCount" = 5, 'id distinct count mismatch'; - ASSERT f."minValue" = '1', 'id min mismatch'; - ASSERT f."maxValue" = '5', 'id max mismatch'; - ASSERT f."avgValue" = 3.00, 'id avg mismatch'; - ASSERT f."median" = 3.00, 'id median mismatch'; - ASSERT f."pattern" = '-?\d+', 'id pattern mismatch'; + ASSERT f."pattern" = 'NUMERIC', 'id pattern classification failed (expected NUMERIC)'; ELSIF f."columnName" = 'username' THEN - ASSERT f."nullable" = false, 'username should not be nullable'; - ASSERT f."nonNullCount" = 5, 'username non-null count mismatch'; - ASSERT f."distinctCount" = 4, 'username distinct count mismatch'; - ASSERT f."minValue" = 'alice', 'username min mismatch'; - ASSERT f."maxValue" = 'diana', 'username max mismatch'; ASSERT f."mode" = 'diana', 'username mode mismatch (should be diana, frequency = 2)'; + ASSERT f."pattern" = '.*', 'username pattern classification failed (expected .*)'; + ELSIF f."columnName" = 'email' THEN + ASSERT f."pattern" = 'EMAIL', 'email pattern classification failed (expected EMAIL)'; + ELSIF f."columnName" = 'uuid_val' THEN + ASSERT f."pattern" = 'UUID', 'uuid_val pattern classification failed (expected UUID)'; + ELSIF f."columnName" = 'ip_val' THEN + ASSERT f."pattern" = 'IPV4', 'ip_val pattern classification failed (expected IPV4)'; + ELSIF f."columnName" = 'url_val' THEN + ASSERT f."pattern" = 'URL', 'url_val pattern classification failed (expected URL)'; + ELSIF f."columnName" = 'signup_date' THEN + ASSERT f."pattern" = 'DATE', 'signup_date pattern classification failed (expected DATE)'; ELSIF f."columnName" = 'age' THEN - ASSERT f."nullable" = true, 'age should be nullable'; - ASSERT f."nonNullCount" = 5, 'age non-null count mismatch'; - ASSERT f."distinctCount" = 3, 'age distinct count mismatch'; - ASSERT f."minValue" = '25', 'age min mismatch'; - ASSERT f."maxValue" = '45', 'age max mismatch'; + ASSERT f."pattern" = 'NUMERIC', 'age pattern classification failed (expected NUMERIC)'; ASSERT f."median" = 25.00, 'age median mismatch'; - ASSERT f."mode" = '25', 'age mode mismatch (frequency = 3)'; - ELSIF f."columnName" = 'salary' THEN - ASSERT f."nullable" = true, 'salary should be nullable'; - ASSERT f."nonNullCount" = 4, 'salary non-null count mismatch'; - ASSERT f."nullCount" = 1, 'salary null count mismatch'; - ASSERT f."minValue" = '80000.00', 'salary min mismatch'; - ASSERT f."maxValue" = '120000.50', 'salary max mismatch'; - ELSIF f."columnName" = 'extra_info' THEN - -- Verify that skipped types like jsonb have null min/max but non-null and distinct counts - ASSERT f."minValue" IS NULL, 'jsonb should not have min'; - ASSERT f."maxValue" IS NULL, 'jsonb should not have max'; - ASSERT f."nonNullCount" = 4, 'jsonb non-null count mismatch'; - ASSERT f."distinctCount" = 3, 'jsonb distinct count mismatch'; END IF; END LOOP; -- Verify value distribution (we filtered with min_cell_count = 2) - -- Values expected to be frequent: age=25 (freq=3), salary=80000.00 (freq=2), username=diana (freq=2) + -- Values expected to be frequent: age=25 (freq=3), email=diana@corp.com (freq=2), country/url_val=https://database.dev (freq=2) DECLARE age_25_found boolean := false; - salary_80000_found boolean := false; - username_diana_found boolean := false; + email_diana_found boolean := false; + url_dbdev_found boolean := false; BEGIN FOR v IN SELECT * FROM jsonb_to_recordset(vals) AS ( "columnName" text, "value" text, "frequency" int, "percent" numeric ) LOOP IF v."columnName" = 'age' AND v."value" = '25' THEN ASSERT v."frequency" = 3, 'age 25 frequency mismatch'; - ASSERT v."percent" = 0.6000, 'age 25 percent mismatch'; age_25_found := true; - ELSIF v."columnName" = 'salary' AND v."value" = '80000.00' THEN - ASSERT v."frequency" = 2, 'salary 80000 frequency mismatch'; - ASSERT v."percent" = 0.4000, 'salary 80000 percent mismatch'; - salary_80000_found := true; - ELSIF v."columnName" = 'username' AND v."value" = 'diana' THEN - ASSERT v."frequency" = 2, 'username diana frequency mismatch'; - ASSERT v."percent" = 0.4000, 'username diana percent mismatch'; - username_diana_found := true; + ELSIF v."columnName" = 'email' AND v."value" = 'diana@corp.com' THEN + ASSERT v."frequency" = 2, 'email diana frequency mismatch'; + email_diana_found := true; + ELSIF v."columnName" = 'url_val' AND v."value" = 'https://database.dev' THEN + ASSERT v."frequency" = 2, 'url_val frequency mismatch'; + url_dbdev_found := true; END IF; END LOOP; ASSERT age_25_found, 'age 25 value distribution missing'; - ASSERT salary_80000_found, 'salary 80000 value distribution missing'; - ASSERT username_diana_found, 'username diana value distribution missing'; + ASSERT email_diana_found, 'email diana value distribution missing'; + ASSERT url_dbdev_found, 'url_val database.dev value distribution missing'; END; - -- Test limiting options - res := supa_profile.profile_table('mock_users'::regclass, '{"rows_per_table": 3}'::jsonb); - ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 3, 'rows_per_table limit option failed'; + -- Test TABLESAMPLE system/bernoulli options + res := supa_profile.profile_table('mock_users'::regclass, '{"sampling_method": "bernoulli", "sample_percentage": 100}'::jsonb); ASSERT res ->> 'queryMode' = 'FAST', 'FAST queryMode mismatch for sampled table'; + ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 5, 'tablesample result size mismatch'; - -- Test disabling numeric stats - res := supa_profile.profile_table('mock_users'::regclass, '{"calculate_numeric_stats": false}'::jsonb); - FOR f IN SELECT * FROM jsonb_to_recordset(res -> 'fields') AS ("columnName" text, "avgValue" numeric, "median" numeric) LOOP - IF f."columnName" = 'age' THEN - ASSERT f."avgValue" IS NULL, 'avgValue should be null when numeric stats are disabled'; - ASSERT f."median" IS NULL, 'median should be null when numeric stats are disabled'; + -- Test CATALOG ESTIMATIONS + -- Execute ANALYZE so that pg_stats gets populated for our temp table + -- Temp tables need manual ANALYZE to populate statistics + ANALYZE mock_users; + + res := supa_profile.profile_table('mock_users'::regclass, '{"use_estimated_stats": true}'::jsonb); + ASSERT res ->> 'queryMode' = 'ESTIMATED', 'ESTIMATED queryMode mismatch'; + ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 5, 'Estimated row count mismatch'; + + -- Verify pattern classification on catalog estimations + FOR f IN SELECT * FROM jsonb_to_recordset(res -> 'fields') AS ("columnName" text, "pattern" text) LOOP + IF f."columnName" = 'email' THEN + ASSERT f."pattern" = 'EMAIL', 'Estimated email pattern failed'; + ELSIF f."columnName" = 'uuid_val' THEN + ASSERT f."pattern" = 'UUID', 'Estimated uuid pattern failed'; END IF; END LOOP; - -- Test disabling value distribution scan - res := supa_profile.profile_table('mock_users'::regclass, '{"scan_field_values": false}'::jsonb); - ASSERT jsonb_array_length(res -> 'values') = 0, 'values array should be empty when scan_field_values is false'; - RAISE NOTICE '✅ All supa_profile unit tests passed successfully!'; END; $$; From 15922d30dda21887d1543567e659c219257d55a6 Mon Sep 17 00:00:00 2001 From: Jesse <16379819+JesseVent@users.noreply.github.com> Date: Wed, 10 Jun 2026 00:17:32 +0930 Subject: [PATCH 6/7] feat(supa_privacy): grant schema usage/execute to Supabase roles Add guarded GRANTs so anon/authenticated can call the masking helpers directly (PostgREST RPC / invoker-side wrappers). Role-existence checks keep CREATE EXTENSION working on non-Supabase Postgres, and @extschema@ keeps it relocatable. create_masked_view runs dynamic DDL, so it stays admin-only via REVOKE from PUBLIC/anon/authenticated. --- supa_privacy/supa_privacy--1.0.0.sql | 93 +++++++++++++++++++++------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/supa_privacy/supa_privacy--1.0.0.sql b/supa_privacy/supa_privacy--1.0.0.sql index 3f91e21..92749e6 100644 --- a/supa_privacy/supa_privacy--1.0.0.sql +++ b/supa_privacy/supa_privacy--1.0.0.sql @@ -42,8 +42,8 @@ $$ LANGUAGE plpgsql IMMUTABLE STRICT; -- 2. PHONE MASKING (Formatting-Preserving) -- --------------------------------------------------------------------- CREATE OR REPLACE FUNCTION @extschema@.mask_phone_flexible( - phone text, - keep_digits int DEFAULT 4, + phone text, + keep_digits int DEFAULT 4, mask_char char DEFAULT '*' ) RETURNS text AS $$ @@ -77,7 +77,7 @@ BEGIN result := result || char_val; END IF; END LOOP; - + RETURN result; END; $$ LANGUAGE plpgsql IMMUTABLE STRICT; @@ -151,8 +151,8 @@ END; $$ LANGUAGE plpgsql VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.perturb_numeric_deterministic( - val numeric, - seed_key text, + val numeric, + seed_key text, max_deviation numeric DEFAULT 0.07 ) RETURNS numeric AS $$ @@ -166,10 +166,10 @@ BEGIN -- Convert SHA-256 hash fragment to a big integer deterministically raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; - + -- Normalize big integer to a 0.0 .. 1.0 range (divide by 2^60 - 1) normalized_rand := abs(raw_hash)::numeric / 1152921504606846975.0; - + -- Map normalized value to the deviation bounds RETURN val * (1.0 + (normalized_rand * (max_deviation * 2.0) - max_deviation)); END; @@ -190,8 +190,8 @@ END; $$ LANGUAGE plpgsql IMMUTABLE; CREATE OR REPLACE FUNCTION @extschema@.shift_date_deterministic( - val date, - seed_key text, + val date, + seed_key text, max_days int DEFAULT 30 ) RETURNS date AS $$ @@ -205,10 +205,10 @@ BEGIN -- Convert SHA-256 hash fragment to a big integer deterministically raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; - + -- Map to a shift in days between [-max_days, max_days] shift_days := (abs(raw_hash) % (max_days * 2 + 1)) - max_days; - + RETURN val + shift_days; END; $$ LANGUAGE plpgsql IMMUTABLE; @@ -262,27 +262,27 @@ BEGIN LOOP col_name := column_record.attname; col_type := column_record.type_desc; - + -- Check if there is a rule defined for this column rule := rules -> col_name; - + IF rule IS NOT NULL THEN rule_type := rule ->> 'type'; - + CASE rule_type WHEN 'email' THEN select_expr := '@extschema@.mask_email(' || quote_ident(col_name) || '::text)'; - + WHEN 'phone' THEN IF (rule ->> 'keep_digits') IS NOT NULL THEN select_expr := '@extschema@.mask_phone_flexible(' || quote_ident(col_name) || '::text, ' || (rule ->> 'keep_digits') || ')'; ELSE select_expr := '@extschema@.mask_phone(' || quote_ident(col_name) || '::text)'; END IF; - + WHEN 'hash' THEN select_expr := '@extschema@.salted_hash(' || quote_ident(col_name) || '::text, ' || quote_literal(coalesce(rule ->> 'salt', '')) || ')'; - + WHEN 'perturb' THEN seed_col := rule ->> 'seed_column'; IF seed_col IS NOT NULL THEN @@ -290,7 +290,7 @@ BEGIN ELSE select_expr := '@extschema@.perturb_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'variance', '0.07') || ')'; END IF; - + WHEN 'shift_date' THEN seed_col := rule ->> 'seed_column'; IF seed_col IS NOT NULL THEN @@ -298,29 +298,29 @@ BEGIN ELSE select_expr := '@extschema@.shift_date_deterministic(' || quote_ident(col_name) || '::date, ' || quote_literal('default_seed') || ', ' || coalesce(rule ->> 'days', '30') || ')'; END IF; - + WHEN 'generalize_numeric' THEN select_expr := '@extschema@.generalize_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'bucket', '10') || ')'; - + WHEN 'generalize_date' THEN select_expr := '@extschema@.generalize_date(' || quote_ident(col_name) || '::date, ' || quote_literal(coalesce(rule ->> 'bucket', 'month')) || ')'; - + WHEN 'redact' THEN IF (rule ->> 'value') IS NULL OR (rule ->> 'value') = 'NULL' THEN select_expr := 'NULL'; ELSE select_expr := quote_literal(rule ->> 'value'); END IF; - + WHEN 'custom' THEN -- Replace placeholder {col} with the actual quoted column identifier select_expr := replace(rule ->> 'expression', '{col}', quote_ident(col_name)); - + ELSE -- Unknown rule type: default to as-is select_expr := quote_ident(col_name); END CASE; - + -- Ensure expression is cast back to the original column type select_expr := '(' || select_expr || ')::' || col_type; ELSE @@ -340,3 +340,48 @@ BEGIN EXECUTE sql_stmt; END; $$ LANGUAGE plpgsql VOLATILE; + + +-- --------------------------------------------------------------------- +-- 9. ROLE GRANTS (Supabase) +-- --------------------------------------------------------------------- +-- The masking helpers execute with the *invoker's* privileges, so any +-- role that calls them directly (PostgREST RPC, or an invoker-side +-- wrapper function such as the get_secured_customers() pattern) needs +-- USAGE on the extension schema and EXECUTE on the functions. +-- +-- NOTE: querying a masked VIEW does NOT require these grants -- a view +-- accesses its referenced functions/tables as the view OWNER, so the +-- client only needs SELECT on the view itself. +-- +-- Wrapped in role-existence checks so CREATE EXTENSION still succeeds on +-- non-Supabase Postgres where anon/authenticated do not exist. +-- @extschema@ is substituted by pg_tle with the install schema, so this +-- stays relocatable. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'anon') THEN + GRANT USAGE ON SCHEMA @extschema@ TO anon; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA @extschema@ TO anon; + END IF; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticated') THEN + GRANT USAGE ON SCHEMA @extschema@ TO authenticated; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA @extschema@ TO authenticated; + END IF; + + -- create_masked_view() runs dynamic DDL (CREATE VIEW). Keep it out of + -- reach of untrusted client roles: an admin/owner should create the + -- masked view once, then expose it via GRANT SELECT on the view. + -- Remove these REVOKEs if you deliberately want clients to build their + -- own masked views. + REVOKE EXECUTE ON FUNCTION @extschema@.create_masked_view(regclass, text, jsonb) FROM PUBLIC; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'anon') THEN + REVOKE EXECUTE ON FUNCTION @extschema@.create_masked_view(regclass, text, jsonb) FROM anon; + END IF; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticated') THEN + REVOKE EXECUTE ON FUNCTION @extschema@.create_masked_view(regclass, text, jsonb) FROM authenticated; + END IF; +END $$; From 8bfa15ec70f094f5e06247426ba820cdc5ef60b7 Mon Sep 17 00:00:00 2001 From: Jesse <16379819+JesseVent@users.noreply.github.com> Date: Wed, 10 Jun 2026 03:10:51 +0930 Subject: [PATCH 7/7] chore(supa_privacy): bump version to 1.0.1 for dbdev --- supa_privacy/README.md | 4 ++-- .../{supa_privacy--1.0.0.sql => supa_privacy--1.0.1.sql} | 0 supa_privacy/supa_privacy.control | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename supa_privacy/{supa_privacy--1.0.0.sql => supa_privacy--1.0.1.sql} (100%) diff --git a/supa_privacy/README.md b/supa_privacy/README.md index 72adb4c..abb1cb7 100644 --- a/supa_privacy/README.md +++ b/supa_privacy/README.md @@ -25,13 +25,13 @@ CREATE EXTENSION IF NOT EXISTS "pg_tle"; Using the [dbdev CLI](https://supabase.github.io/dbdev): ```bash -dbdev add -o ./migrations -s extensions -v 1.0.0 package -n "jvent@supa_privacy" +dbdev add -o ./migrations -s extensions -v 1.0.1 package -n "jvent@supa_privacy" ``` This will generate a migration file in your `./migrations` folder containing the SQL required to load the extension. After applying the migration, enable the extension: ```sql -CREATE EXTENSION "jvent@supa_privacy" VERSION '1.0.0' SCHEMA supa_privacy; +CREATE EXTENSION "jvent@supa_privacy" VERSION '1.0.1' SCHEMA supa_privacy; ``` --- diff --git a/supa_privacy/supa_privacy--1.0.0.sql b/supa_privacy/supa_privacy--1.0.1.sql similarity index 100% rename from supa_privacy/supa_privacy--1.0.0.sql rename to supa_privacy/supa_privacy--1.0.1.sql diff --git a/supa_privacy/supa_privacy.control b/supa_privacy/supa_privacy.control index cd8547a..60351a7 100644 --- a/supa_privacy/supa_privacy.control +++ b/supa_privacy/supa_privacy.control @@ -1,5 +1,5 @@ # supa_privacy extension for PostgreSQL comment = 'Formatting-preserving anonymisation and data masking' -default_version = '1.0.0' +default_version = '1.0.1' relocatable = true superuser = false