diff --git a/Makefile b/Makefile index cd6b7a7..52e74bb 100644 --- a/Makefile +++ b/Makefile @@ -14,17 +14,19 @@ reset: .PHONY: tle.install tle.install: \ tle.install.pg_idkit \ - tle.install.is_even + tle.install.is_even \ + tle.install.supa_privacy \ + tle.install.supa_profile @echo "\n\nDone!" tle.install.%: @echo "\n\nInstalling $*" - dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres --path $(REPO_DIR)/$* + dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres path --directory $(REPO_DIR)/$* PGPASSWORD=postgres psql -U postgres -d postgres -h localhost -p 54322 -c "CREATE EXTENSION $*;" tle.update.%: @echo "\n\Updating $*" - dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres --path $(REPO_DIR)/$* + dbdev install --connection postgres://postgres:postgres@localhost:54322/postgres path --directory $(REPO_DIR)/$* dbdev.publish.%: @echo "\n\nPublishing $* \n" diff --git a/supa_privacy/README.md b/supa_privacy/README.md new file mode 100644 index 0000000..abb1cb7 --- /dev/null +++ b/supa_privacy/README.md @@ -0,0 +1,287 @@ +# supa_privacy - It's not just private, it's supa private. + +## PostgreSQL Data Anonymisation TLE Extension + +`supa_privacy` is a relocatable, pure SQL PostgreSQL **Trusted Language Extension (TLE)** designed for database-side data de-identification, compliance, and sandbox security. It provides standard masking, hashing, and perturbation functions and is fully optimized for **[database.dev](https://database.dev)** and **Supabase**. + +It is particularly useful for: +- Preparing production database schemas for staging or development environments. +- Sharing data with external analytics teams while respecting privacy regulations (GDPR, HIPAA, CCPA). +- Implementing Dynamic Data Masking (DDM) for low-privilege database roles. + +--- + +## 🚀 Installation + +### 1. Enable TLE Support +`supa_privacy` runs as a Trusted Language Extension and requires the `pg_tle` extension to be enabled in your PostgreSQL database: + +```sql +CREATE EXTENSION IF NOT EXISTS "pg_tle"; +``` + +### 2. Installing from database.dev + +Using the [dbdev CLI](https://supabase.github.io/dbdev): + +```bash +dbdev add -o ./migrations -s extensions -v 1.0.1 package -n "jvent@supa_privacy" +``` + +This will generate a migration file in your `./migrations` folder containing the SQL required to load the extension. After applying the migration, enable the extension: + +```sql +CREATE EXTENSION "jvent@supa_privacy" VERSION '1.0.1' SCHEMA supa_privacy; +``` + +--- + +## 🛠️ API Reference + +All functions are located under the schema where the extension is installed (e.g. `supa_privacy`). + +### 1. Email Masking +Masks the username part of an email address while keeping the domain. +* **Function:** `supa_privacy.mask_email(email text) RETURNS text` +* **Behavior:** + - `jesse@vent.com` ➡️ `j***e@vent.com` + - `ab@vent.com` ➡️ `a*@vent.com` + - `a@vent.com` ➡️ `*@vent.com` +* **Example:** + ```sql + SELECT supa_privacy.mask_email('jesse@vent.com'); -- Returns: j***e@vent.com + ``` + +### 2. Phone Masking +Masks digit characters while preserving formatting characters (such as spaces, hyphens, plus signs, and brackets). +* **Format-Preserving Masking:** `supa_privacy.mask_phone_flexible(phone text, keep_digits int DEFAULT 4, mask_char char DEFAULT '*') RETURNS text` +* **Default Wrapper:** `supa_privacy.mask_phone(phone text) RETURNS text` (wrapper calling `mask_phone_flexible` keeping 4 digits). +* **Behavior:** + - `+1 (202) 555-0143` ➡️ `+* (***) ***-0143` + - `+61412345678` ➡️ `+*******5678` +* **Example:** + ```sql + SELECT supa_privacy.mask_phone_flexible('+1 (202) 555-0143', 4); -- Returns: +* (***) ***-0143 + ``` + +### 3. Text Masking & Redaction +Fully or partially obscures text strings. +* **Full Masking:** `supa_privacy.mask_text(val text, mask_char char DEFAULT '*') RETURNS text` + - `secret` ➡️ `******` +* **Partial Masking:** `supa_privacy.partial_mask(val text, prefix_keep int, suffix_keep int, mask_char char DEFAULT '*') RETURNS text` + - `supa_privacy.partial_mask('1234-5678-9012', 4, 4, '*')` ➡️ `1234******9012` +* **Examples:** + ```sql + SELECT supa_privacy.mask_text('secret'); -- Returns: ****** + SELECT supa_privacy.partial_mask('1234-5678-9012', 4, 4, '*'); -- Returns: 1234******9012 + ``` + +### 4. Cryptographic Salted Hashing (Deterministic) +Generates a deterministic SHA-256 hash using a secret salt. +* **Function:** `supa_privacy.salted_hash(val text, salt text) RETURNS text` +* **Example:** + ```sql + SELECT supa_privacy.salted_hash('user_id_123', 'secret_salt_key'); + -- Returns hex-encoded SHA-256 hash: 39de1a88b56f... + ``` + +### 5. Numeric Perturbation (Noise Injection) +Adds bounded random variance (noise) to numeric fields to protect details while preserving statistical averages. +* **Volatile Perturbation:** `supa_privacy.perturb_numeric(val numeric, max_deviation numeric DEFAULT 0.07) RETURNS numeric` + - Noise changes on every query execution. +* **Deterministic Perturbation:** `supa_privacy.perturb_numeric_deterministic(val numeric, seed_key text, max_deviation numeric DEFAULT 0.07) RETURNS numeric` + - Noise is seeded by `seed_key` (typically the row's primary key) so it remains identical across multiple query executions (critical for analytical query stability). +* **Examples:** + ```sql + -- Volatile noise + SELECT supa_privacy.perturb_numeric(100.0, 0.10); -- Returns a value between 90.0 and 110.0 + -- Deterministic noise (always returns same value for 'user_1') + SELECT supa_privacy.perturb_numeric_deterministic(100.0, 'user_1', 0.10); + ``` + +### 6. Date Generalization & Shifting +Obfuscates absolute dates via truncation or chronological offsets. +* **Date Generalization (Bucketing):** `supa_privacy.generalize_date(val date, bucket text DEFAULT 'month') RETURNS date` + - Truncates dates to `'year'`, `'quarter'`, `'month'`, or `'week'`. +* **Deterministic Date Shifting:** `supa_privacy.shift_date_deterministic(val date, seed_key text, max_days int DEFAULT 30) RETURNS date` + - Shifts date back/forward by a deterministic number of days based on a seed key. Ideal for HIPAA compliance (preserves chronological order of events within a user profile while masking the absolute dates). +* **Examples:** + ```sql + SELECT supa_privacy.generalize_date('2026-06-08'::date, 'year'); -- Returns: 2026-01-01 + SELECT supa_privacy.shift_date_deterministic('2026-06-08'::date, 'user_123_salt', 15); -- Shifts by +/- 15 days deterministically + ``` + +### 7. Numeric Generalization (Bucketing) +Groups numbers into custom bucket intervals (useful for ages, salaries). +* **Function:** `supa_privacy.generalize_numeric(val numeric, bucket_size numeric) RETURNS numeric` +* **Example:** + ```sql + SELECT supa_privacy.generalize_numeric(27, 5); -- Returns: 25 (rounds to nearest multiple of 5) + ``` + +### 8. Dynamic Masked View Generator (Extensible) +Scans a physical table and automatically generates a secure VIEW that replaces sensitive fields with masking expressions while leaving other columns untouched. +* **Function:** `supa_privacy.create_masked_view(source_table regclass, view_name text, rules jsonb) RETURNS void` +* **Rules JSONB Format:** + A JSON object mapping column names to rule configurations. Supported rule configurations: + - `{"type": "email"}` + - `{"type": "phone", "keep_digits": 4}` (uses formatting-preserving masking) + - `{"type": "hash", "salt": "my_salt"}` + - `{"type": "perturb", "variance": 0.05, "seed_column": "id"}` (deterministic noise) + - `{"type": "shift_date", "days": 15, "seed_column": "id"}` (deterministic date shifting) + - `{"type": "generalize_numeric", "bucket": 10}` + - `{"type": "generalize_date", "bucket": "year"}` + - `{"type": "redact", "value": "NULL"}` (replaces value with constant or NULL) + - `{"type": "custom", "expression": "upper(reverse({col}))"}` (interpolates `{col}` with column name) + +--- + +## 📖 End-to-End Example + +Here is a full walkthrough showing how to de-identify a customer table. + +### 1. Create a Source Table and Insert Mock Data +```sql +CREATE TABLE public.customers ( + id SERIAL PRIMARY KEY, + full_name TEXT NOT NULL, + email TEXT UNIQUE NOT NULL, + phone TEXT, + age INT, + yearly_salary NUMERIC(10, 2), + signup_date DATE NOT NULL +); + +INSERT INTO public.customers (full_name, email, phone, age, yearly_salary, signup_date) VALUES +('John Doe', 'john.doe@gmail.com', '+1 (202) 555-0143', 27, 85000.00, '2025-03-12'), +('Jane Smith', 'jane_smith@corp.com', '+61 412 000 222', 43, 142000.00, '2024-11-20'), +('Bob Johnson', 'bjohnson@yahoo.com', '0491 570 156', 58, 62500.50, '2026-01-05'); +``` + +### 2. Generate a Masked View +Define anonymisation rules in JSON and invoke `create_masked_view`: + +```sql +SELECT supa_privacy.create_masked_view( + 'public.customers'::regclass, + 'public.v_customers_deidentified', + '{ + "full_name": {"type": "hash", "salt": "AppSaltKey123"}, + "email": {"type": "email"}, + "phone": {"type": "phone", "keep_digits": 4}, + "age": {"type": "generalize_numeric", "bucket": 10}, + "yearly_salary": {"type": "perturb", "variance": 0.07, "seed_column": "id"}, + "signup_date": {"type": "shift_date", "days": 15, "seed_column": "id"} + }'::jsonb +); +``` + +### 3. Query the Masked View +Query the generated view: + +```sql +SELECT * FROM public.v_customers_deidentified; +``` + +**Result:** +| id | full_name | email | phone | age | yearly_salary | signup_date | +|---|---|---|---|---|---|---| +| 1 | `e97a3a9489f...` | `j***e@gmail.com` | `+* (***) ***-0143` | 30 | `87140.23` (Stable) | `2025-03-14` (Shifted) | +| 2 | `a902b4d812d...` | `j***h@corp.com` | `+** *** *** 0222` | 40 | `138402.12` (Stable) | `2024-11-12` (Shifted) | +| 3 | `d8293bcde11...` | `b***n@yahoo.com` | `**** *** 0156` | 60 | `64102.50` (Stable) | `2026-01-08` (Shifted) | + +--- + +## 🔐 RLS & Access Control Integration Patterns + +For production environments (like **Supabase**), you should combine `supa_privacy`'s Column-level Dynamic Data Masking (DDM) with PostgreSQL **Row-Level Security (RLS)** for comprehensive access control. + +### 1. Dual-Layer Masking + RLS Pattern +**Best Practice:** Always apply RLS to both the base table and the generated view. This ensures that users see only the rows they are allowed to see, with sensitive columns masked. + +```sql +-- 1. Create source table with RLS +CREATE TABLE public.customers ( + id SERIAL PRIMARY KEY, + full_name TEXT NOT NULL, + email TEXT UNIQUE NOT NULL, + phone TEXT, + department_id INT, -- RLS filter key + yearly_salary NUMERIC(10, 2) +); + +ALTER TABLE public.customers ENABLE ROW LEVEL SECURITY; + +-- 2. Create RLS policy for the base table +CREATE POLICY cust_department_policy ON public.customers + FOR SELECT + USING (department_id = current_setting('app.current_department_id')::int); + +-- 3. Generate the masked view (DDM layer) +SELECT supa_privacy.create_masked_view( + 'public.customers'::regclass, + 'public.v_customers_masked', + '{ + "full_name": {"type": "hash", "salt": "SecureAppSaltKey"}, + "email": {"type": "email"}, + "phone": {"type": "phone"}, + "yearly_salary": {"type": "perturb", "variance": 0.07} + }'::jsonb +); + +-- 4. Enable RLS on the masked view (CRITICAL!) +ALTER TABLE public.v_customers_masked ENABLE ROW LEVEL SECURITY; + +CREATE POLICY mask_department_policy ON public.v_customers_masked + FOR SELECT + USING (department_id = current_setting('app.current_department_id')::int); + +-- 5. Grant permissions to database role +GRANT SELECT ON public.v_customers_masked TO app_user; +``` + +### 2. Alternative: Role-Based Transparent Masking +If you want users to query the base table directly but have masking applied conditionally based on the user's role or session variables: + +```sql +-- Wrapper function to conditionally mask based on session variables +CREATE OR REPLACE FUNCTION public.get_secured_customers() +RETURNS TABLE( + id INT, + full_name TEXT, + email TEXT, + phone TEXT, + department_id INT, + yearly_salary NUMERIC(10, 2) +) AS $$ +BEGIN + RETURN QUERY + SELECT + c.id, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.mask_text(c.full_name, '*') + ELSE c.full_name + END, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.mask_email(c.email) + ELSE c.email + END, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.mask_phone(c.phone) + ELSE c.phone + END, + c.department_id, + CASE WHEN current_setting('app.masking_enabled', true) = 'true' + THEN supa_privacy.perturb_numeric(c.yearly_salary, 0.07) + ELSE c.yearly_salary + END + FROM public.customers c + WHERE c.department_id = current_setting('app.current_department_id', true)::int; +END; +$$ LANGUAGE plpgsql; +``` + +--- + +## 🛡️ License +MIT License. Feel free to use and distribute. diff --git a/supa_privacy/supa_privacy--1.0.1.sql b/supa_privacy/supa_privacy--1.0.1.sql new file mode 100644 index 0000000..92749e6 --- /dev/null +++ b/supa_privacy/supa_privacy--1.0.1.sql @@ -0,0 +1,387 @@ +-- supa_privacy extension database schema definition + +-- --------------------------------------------------------------------- +-- 1. EMAIL MASKING +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.mask_email(email text) +RETURNS text AS $$ +DECLARE + parts text[]; + username text; + domain text; + len int; +BEGIN + IF email IS NULL OR email = '' THEN + RETURN email; + END IF; + + -- Split email by '@' + parts := string_to_array(email, '@'); + IF array_length(parts, 1) != 2 THEN + -- Fallback if format is not standard email + RETURN regexp_replace(email, '.', '*', 'g'); + END IF; + + username := parts[1]; + domain := parts[2]; + len := length(username); + + IF len <= 1 THEN + RETURN '*' || '@' || domain; + ELSIF len = 2 THEN + RETURN left(username, 1) || '*' || '@' || domain; + ELSE + -- e.g. jesse@vent.com -> j***e@vent.com + RETURN left(username, 1) || '***' || right(username, 1) || '@' || domain; + END IF; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + + +-- --------------------------------------------------------------------- +-- 2. PHONE MASKING (Formatting-Preserving) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.mask_phone_flexible( + phone text, + keep_digits int DEFAULT 4, + mask_char char DEFAULT '*' +) +RETURNS text AS $$ +DECLARE + result text := ''; + char_val text; + digit_count int := 0; + total_digits int := 0; + digits_to_mask int; +BEGIN + IF phone IS NULL OR phone = '' THEN + RETURN phone; + END IF; + + -- Count total digits in the string + total_digits := length(regexp_replace(phone, '\D', '', 'g')); + digits_to_mask := total_digits - keep_digits; + + -- Iterate and mask digits selectively + FOR i IN 1..length(phone) LOOP + char_val := substr(phone, i, 1); + IF char_val ~ '[0-9]' THEN + digit_count := digit_count + 1; + IF digit_count <= digits_to_mask THEN + result := result || mask_char; + ELSE + result := result || char_val; + END IF; + ELSE + -- Keep formatting spaces, hyphens, plus signs, brackets + result := result || char_val; + END IF; + END LOOP; + + RETURN result; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION @extschema@.mask_phone(phone text) +RETURNS text AS $$ +BEGIN + -- Backward-compatible wrapper that defaults to format-preserving masking + RETURN @extschema@.mask_phone_flexible(phone, 4, '*'); +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + + +-- --------------------------------------------------------------------- +-- 3. TEXT MASKING & REDACTION +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.mask_text(val text, mask_char char DEFAULT '*') +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN repeat(mask_char, length(val)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION @extschema@.partial_mask(val text, prefix_keep int, suffix_keep int, mask_char char DEFAULT '*') +RETURNS text AS $$ +DECLARE + len int; +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + len := length(val); + IF len <= (prefix_keep + suffix_keep) THEN + RETURN repeat(mask_char, len); + END IF; + RETURN left(val, prefix_keep) || repeat(mask_char, len - prefix_keep - suffix_keep) || right(val, suffix_keep); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 4. CRYPTOGRAPHIC HASHING (Salted & Deterministic) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.salted_hash(val text, salt text) +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + -- Uses built-in sha256 to hash concatenation of value and salt, and encodes to hex + RETURN encode(sha256(convert_to(val || salt, 'UTF8')), 'hex'); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 5. NUMERIC PERTURBATION (Volatile & Deterministic Noise) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.perturb_numeric(val numeric, max_deviation numeric DEFAULT 0.07) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + -- deviation is a random number between -max_deviation and +max_deviation (volatile per run) + RETURN val * (1.0 + (random() * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE OR REPLACE FUNCTION @extschema@.perturb_numeric_deterministic( + val numeric, + seed_key text, + max_deviation numeric DEFAULT 0.07 +) +RETURNS numeric AS $$ +DECLARE + raw_hash bigint; + normalized_rand numeric; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + + -- Convert SHA-256 hash fragment to a big integer deterministically + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + + -- Normalize big integer to a 0.0 .. 1.0 range (divide by 2^60 - 1) + normalized_rand := abs(raw_hash)::numeric / 1152921504606846975.0; + + -- Map normalized value to the deviation bounds + RETURN val * (1.0 + (normalized_rand * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 6. DATE GENERALIZATION & SHIFTING +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.generalize_date(val date, bucket text DEFAULT 'month') +RETURNS date AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN date_trunc(bucket, val)::date; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION @extschema@.shift_date_deterministic( + val date, + seed_key text, + max_days int DEFAULT 30 +) +RETURNS date AS $$ +DECLARE + raw_hash bigint; + shift_days int; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + + -- Convert SHA-256 hash fragment to a big integer deterministically + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + + -- Map to a shift in days between [-max_days, max_days] + shift_days := (abs(raw_hash) % (max_days * 2 + 1)) - max_days; + + RETURN val + shift_days; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 7. NUMERIC GENERALIZATION (Bucketing) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.generalize_numeric(val numeric, bucket_size numeric) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL OR bucket_size IS NULL OR bucket_size <= 0 THEN + RETURN val; + END IF; + -- Rounds value to the nearest multiple of bucket_size + RETURN round(val / bucket_size) * bucket_size; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- --------------------------------------------------------------------- +-- 8. DYNAMIC MASKED VIEW GENERATOR (Enhanced & Extensible) +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.create_masked_view( + source_table regclass, + view_name text, + rules jsonb +) +RETURNS void AS $$ +DECLARE + column_record record; + col_name text; + col_type text; + rule jsonb; + rule_type text; + select_expr text; + select_list text := ''; + sql_stmt text; + full_table_name text; + seed_col text; +BEGIN + -- Resolve full schema-qualified table name + full_table_name := source_table::text; + + -- Loop through all columns of the source table + FOR column_record IN + SELECT attname, format_type(atttypid, atttypmod) AS type_desc + FROM pg_attribute + WHERE attrelid = source_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + col_name := column_record.attname; + col_type := column_record.type_desc; + + -- Check if there is a rule defined for this column + rule := rules -> col_name; + + IF rule IS NOT NULL THEN + rule_type := rule ->> 'type'; + + CASE rule_type + WHEN 'email' THEN + select_expr := '@extschema@.mask_email(' || quote_ident(col_name) || '::text)'; + + WHEN 'phone' THEN + IF (rule ->> 'keep_digits') IS NOT NULL THEN + select_expr := '@extschema@.mask_phone_flexible(' || quote_ident(col_name) || '::text, ' || (rule ->> 'keep_digits') || ')'; + ELSE + select_expr := '@extschema@.mask_phone(' || quote_ident(col_name) || '::text)'; + END IF; + + WHEN 'hash' THEN + select_expr := '@extschema@.salted_hash(' || quote_ident(col_name) || '::text, ' || quote_literal(coalesce(rule ->> 'salt', '')) || ')'; + + WHEN 'perturb' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := '@extschema@.perturb_numeric_deterministic(' || quote_ident(col_name) || '::numeric, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + ELSE + select_expr := '@extschema@.perturb_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + END IF; + + WHEN 'shift_date' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := '@extschema@.shift_date_deterministic(' || quote_ident(col_name) || '::date, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'days', '30') || ')'; + ELSE + select_expr := '@extschema@.shift_date_deterministic(' || quote_ident(col_name) || '::date, ' || quote_literal('default_seed') || ', ' || coalesce(rule ->> 'days', '30') || ')'; + END IF; + + WHEN 'generalize_numeric' THEN + select_expr := '@extschema@.generalize_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'bucket', '10') || ')'; + + WHEN 'generalize_date' THEN + select_expr := '@extschema@.generalize_date(' || quote_ident(col_name) || '::date, ' || quote_literal(coalesce(rule ->> 'bucket', 'month')) || ')'; + + WHEN 'redact' THEN + IF (rule ->> 'value') IS NULL OR (rule ->> 'value') = 'NULL' THEN + select_expr := 'NULL'; + ELSE + select_expr := quote_literal(rule ->> 'value'); + END IF; + + WHEN 'custom' THEN + -- Replace placeholder {col} with the actual quoted column identifier + select_expr := replace(rule ->> 'expression', '{col}', quote_ident(col_name)); + + ELSE + -- Unknown rule type: default to as-is + select_expr := quote_ident(col_name); + END CASE; + + -- Ensure expression is cast back to the original column type + select_expr := '(' || select_expr || ')::' || col_type; + ELSE + -- No rule defined: select as-is + select_expr := quote_ident(col_name); + END IF; + + -- Append to select list + IF select_list != '' THEN + select_list := select_list || ', '; + END IF; + select_list := select_list || select_expr || ' AS ' || quote_ident(col_name); + END LOOP; + + -- Build and execute CREATE VIEW statement + sql_stmt := 'CREATE OR REPLACE VIEW ' || quote_ident(view_name) || ' AS SELECT ' || select_list || ' FROM ' || full_table_name; + EXECUTE sql_stmt; +END; +$$ LANGUAGE plpgsql VOLATILE; + + +-- --------------------------------------------------------------------- +-- 9. ROLE GRANTS (Supabase) +-- --------------------------------------------------------------------- +-- The masking helpers execute with the *invoker's* privileges, so any +-- role that calls them directly (PostgREST RPC, or an invoker-side +-- wrapper function such as the get_secured_customers() pattern) needs +-- USAGE on the extension schema and EXECUTE on the functions. +-- +-- NOTE: querying a masked VIEW does NOT require these grants -- a view +-- accesses its referenced functions/tables as the view OWNER, so the +-- client only needs SELECT on the view itself. +-- +-- Wrapped in role-existence checks so CREATE EXTENSION still succeeds on +-- non-Supabase Postgres where anon/authenticated do not exist. +-- @extschema@ is substituted by pg_tle with the install schema, so this +-- stays relocatable. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'anon') THEN + GRANT USAGE ON SCHEMA @extschema@ TO anon; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA @extschema@ TO anon; + END IF; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticated') THEN + GRANT USAGE ON SCHEMA @extschema@ TO authenticated; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA @extschema@ TO authenticated; + END IF; + + -- create_masked_view() runs dynamic DDL (CREATE VIEW). Keep it out of + -- reach of untrusted client roles: an admin/owner should create the + -- masked view once, then expose it via GRANT SELECT on the view. + -- Remove these REVOKEs if you deliberately want clients to build their + -- own masked views. + REVOKE EXECUTE ON FUNCTION @extschema@.create_masked_view(regclass, text, jsonb) FROM PUBLIC; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'anon') THEN + REVOKE EXECUTE ON FUNCTION @extschema@.create_masked_view(regclass, text, jsonb) FROM anon; + END IF; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticated') THEN + REVOKE EXECUTE ON FUNCTION @extschema@.create_masked_view(regclass, text, jsonb) FROM authenticated; + END IF; +END $$; diff --git a/supa_privacy/supa_privacy.control b/supa_privacy/supa_privacy.control new file mode 100644 index 0000000..60351a7 --- /dev/null +++ b/supa_privacy/supa_privacy.control @@ -0,0 +1,5 @@ +# supa_privacy extension for PostgreSQL +comment = 'Formatting-preserving anonymisation and data masking' +default_version = '1.0.1' +relocatable = true +superuser = false diff --git a/supa_privacy/test_validation.sql b/supa_privacy/test_validation.sql new file mode 100644 index 0000000..5c0774c --- /dev/null +++ b/supa_privacy/test_validation.sql @@ -0,0 +1,394 @@ +-- ===================================================================== +-- SUPA_PRIVACY EXTENSION VALIDATION TEST SCRIPT +-- Run this script in any PostgreSQL database (version 13+) to verify +-- the correctness of the anonymisation functions and view generation. +-- ===================================================================== + +BEGIN; + +-- --------------------------------------------------------- +-- 1. Loading extension schema and functions... +-- --------------------------------------------------------- + +CREATE SCHEMA IF NOT EXISTS supa_privacy; + +-- Copy of functions for standalone execution verification + +CREATE OR REPLACE FUNCTION supa_privacy.mask_email(email text) +RETURNS text AS $$ +DECLARE + parts text[]; + username text; + domain text; + len int; +BEGIN + IF email IS NULL OR email = '' THEN + RETURN email; + END IF; + parts := string_to_array(email, '@'); + IF array_length(parts, 1) != 2 THEN + RETURN regexp_replace(email, '.', '*', 'g'); + END IF; + username := parts[1]; + domain := parts[2]; + len := length(username); + IF len <= 1 THEN + RETURN '*' || '@' || domain; + ELSIF len = 2 THEN + RETURN left(username, 1) || '*' || '@' || domain; + ELSE + RETURN left(username, 1) || '***' || right(username, 1) || '@' || domain; + END IF; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_privacy.mask_phone_flexible( + phone text, + keep_digits int DEFAULT 4, + mask_char char DEFAULT '*' +) +RETURNS text AS $$ +DECLARE + result text := ''; + char_val text; + digit_count int := 0; + total_digits int := 0; + digits_to_mask int; +BEGIN + IF phone IS NULL OR phone = '' THEN + RETURN phone; + END IF; + total_digits := length(regexp_replace(phone, '\D', '', 'g')); + digits_to_mask := total_digits - keep_digits; + FOR i IN 1..length(phone) LOOP + char_val := substr(phone, i, 1); + IF char_val ~ '[0-9]' THEN + digit_count := digit_count + 1; + IF digit_count <= digits_to_mask THEN + result := result || mask_char; + ELSE + result := result || char_val; + END IF; + ELSE + result := result || char_val; + END IF; + END LOOP; + RETURN result; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_privacy.mask_phone(phone text) +RETURNS text AS $$ +BEGIN + RETURN supa_privacy.mask_phone_flexible(phone, 4, '*'); +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_privacy.mask_text(val text, mask_char char DEFAULT '*') +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN repeat(mask_char, length(val)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.partial_mask(val text, prefix_keep int, suffix_keep int, mask_char char DEFAULT '*') +RETURNS text AS $$ +DECLARE + len int; +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + len := length(val); + IF len <= (prefix_keep + suffix_keep) THEN + RETURN repeat(mask_char, len); + END IF; + RETURN left(val, prefix_keep) || repeat(mask_char, len - prefix_keep - suffix_keep) || right(val, suffix_keep); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.salted_hash(val text, salt text) +RETURNS text AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN encode(sha256(convert_to(val || salt, 'UTF8')), 'hex'); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.perturb_numeric(val numeric, max_deviation numeric DEFAULT 0.07) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN val * (1.0 + (random() * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE OR REPLACE FUNCTION supa_privacy.perturb_numeric_deterministic( + val numeric, + seed_key text, + max_deviation numeric DEFAULT 0.07 +) +RETURNS numeric AS $$ +DECLARE + raw_hash bigint; + normalized_rand numeric; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + normalized_rand := abs(raw_hash)::numeric / 1152921504606846975.0; + RETURN val * (1.0 + (normalized_rand * (max_deviation * 2.0) - max_deviation)); +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.generalize_date(val date, bucket text DEFAULT 'month') +RETURNS date AS $$ +BEGIN + IF val IS NULL THEN + RETURN NULL; + END IF; + RETURN date_trunc(bucket, val)::date; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.shift_date_deterministic( + val date, + seed_key text, + max_days int DEFAULT 30 +) +RETURNS date AS $$ +DECLARE + raw_hash bigint; + shift_days int; +BEGIN + IF val IS NULL OR seed_key IS NULL THEN + RETURN val; + END IF; + raw_hash := ('x' || left(encode(sha256(convert_to(seed_key, 'UTF8')), 'hex'), 15))::bit(60)::bigint; + shift_days := (abs(raw_hash) % (max_days * 2 + 1)) - max_days; + RETURN val + shift_days; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.generalize_numeric(val numeric, bucket_size numeric) +RETURNS numeric AS $$ +BEGIN + IF val IS NULL OR bucket_size IS NULL OR bucket_size <= 0 THEN + RETURN val; + END IF; + RETURN round(val / bucket_size) * bucket_size; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_privacy.create_masked_view( + source_table regclass, + view_name text, + rules jsonb +) +RETURNS void AS $$ +DECLARE + column_record record; + col_name text; + col_type text; + rule jsonb; + rule_type text; + select_expr text; + select_list text := ''; + sql_stmt text; + full_table_name text; + seed_col text; +BEGIN + full_table_name := source_table::text; + FOR column_record IN + SELECT attname, format_type(atttypid, atttypmod) AS type_desc + FROM pg_attribute + WHERE attrelid = source_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + col_name := column_record.attname; + col_type := column_record.type_desc; + rule := rules -> col_name; + IF rule IS NOT NULL THEN + rule_type := rule ->> 'type'; + CASE rule_type + WHEN 'email' THEN + select_expr := 'supa_privacy.mask_email(' || quote_ident(col_name) || '::text)'; + WHEN 'phone' THEN + IF (rule ->> 'keep_digits') IS NOT NULL THEN + select_expr := 'supa_privacy.mask_phone_flexible(' || quote_ident(col_name) || '::text, ' || (rule ->> 'keep_digits') || ')'; + ELSE + select_expr := 'supa_privacy.mask_phone(' || quote_ident(col_name) || '::text)'; + END IF; + WHEN 'hash' THEN + select_expr := 'supa_privacy.salted_hash(' || quote_ident(col_name) || '::text, ' || quote_literal(coalesce(rule ->> 'salt', '')) || ')'; + WHEN 'perturb' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := 'supa_privacy.perturb_numeric_deterministic(' || quote_ident(col_name) || '::numeric, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + ELSE + select_expr := 'supa_privacy.perturb_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'variance', '0.07') || ')'; + END IF; + WHEN 'shift_date' THEN + seed_col := rule ->> 'seed_column'; + IF seed_col IS NOT NULL THEN + select_expr := 'supa_privacy.shift_date_deterministic(' || quote_ident(col_name) || '::date, ' || quote_ident(seed_col) || '::text, ' || coalesce(rule ->> 'days', '30') || ')'; + ELSE + select_expr := 'supa_privacy.shift_date_deterministic(' || quote_ident(col_name) || '::date, ''default_seed'', ' || coalesce(rule ->> 'days', '30') || ')'; + END IF; + WHEN 'generalize_numeric' THEN + select_expr := 'supa_privacy.generalize_numeric(' || quote_ident(col_name) || '::numeric, ' || coalesce(rule ->> 'bucket', '10') || ')'; + WHEN 'generalize_date' THEN + select_expr := 'supa_privacy.generalize_date(' || quote_ident(col_name) || '::date, ' || quote_literal(coalesce(rule ->> 'bucket', 'month')) || ')'; + WHEN 'redact' THEN + IF (rule ->> 'value') IS NULL OR (rule ->> 'value') = 'NULL' THEN + select_expr := 'NULL'; + ELSE + select_expr := quote_literal(rule ->> 'value'); + END IF; + WHEN 'custom' THEN + select_expr := replace(rule ->> 'expression', '{col}', quote_ident(col_name)); + ELSE + select_expr := quote_ident(col_name); + END CASE; + select_expr := '(' || select_expr || ')::' || col_type; + ELSE + select_expr := quote_ident(col_name); + END IF; + IF select_list != '' THEN + select_list := select_list || ', '; + END IF; + select_list := select_list || select_expr || ' AS ' || quote_ident(col_name); + END LOOP; + sql_stmt := 'CREATE OR REPLACE VIEW ' || quote_ident(view_name) || ' AS SELECT ' || select_list || ' FROM ' || full_table_name; + EXECUTE sql_stmt; +END; +$$ LANGUAGE plpgsql VOLATILE; + +-- --------------------------------------------------------- +-- 2. Running Function Unit Tests... +-- --------------------------------------------------------- + +DO $$ +BEGIN + -- test email masking + ASSERT supa_privacy.mask_email('jesse@vent.com') = 'j***e@vent.com', 'mask_email failed for normal username'; + ASSERT supa_privacy.mask_email('ab@vent.com') = 'a*@vent.com', 'mask_email failed for length 2 username'; + ASSERT supa_privacy.mask_email('a@vent.com') = '*@vent.com', 'mask_email failed for length 1 username'; + ASSERT supa_privacy.mask_email('') = '', 'mask_email failed for empty string'; + ASSERT supa_privacy.mask_email(NULL) IS NULL, 'mask_email failed for NULL'; + + -- test standard and flexible phone masking + ASSERT supa_privacy.mask_phone('+61412345678') = '+*******5678', 'mask_phone failed for +61412345678'; + ASSERT supa_privacy.mask_phone_flexible('+1 (202) 555-0143', 4) = '+* (***) ***-0143', 'mask_phone_flexible failed for formatted phone'; + ASSERT supa_privacy.mask_phone_flexible('12345', 2, '#') = '###45', 'mask_phone_flexible failed for custom mask char'; + + -- test text masking + ASSERT supa_privacy.mask_text('secret') = '******', 'mask_text failed'; + ASSERT supa_privacy.partial_mask('1234-5678-9012', 4, 4, '*') = '1234******9012', 'partial_mask failed'; + + -- test salted hash (should be deterministic) + ASSERT supa_privacy.salted_hash('test_value', 'my_salt') = supa_privacy.salted_hash('test_value', 'my_salt'), 'salted_hash is not deterministic'; + ASSERT supa_privacy.salted_hash('test_value', 'my_salt') != supa_privacy.salted_hash('test_value', 'different_salt'), 'salted_hash does not respect salt'; + ASSERT length(supa_privacy.salted_hash('test_value', 'my_salt')) = 64, 'salted_hash output is not 64-char sha256 hex string'; + + -- test numeric perturbation (volatile vs deterministic) + ASSERT supa_privacy.perturb_numeric(100.0, 0.0) = 100.0, 'perturb_numeric variance 0.0 deviation failed'; + + -- verify deterministic perturbation is stable + ASSERT supa_privacy.perturb_numeric_deterministic(100.0, 'seed1', 0.07) = supa_privacy.perturb_numeric_deterministic(100.0, 'seed1', 0.07), 'deterministic perturbation is not stable'; + ASSERT supa_privacy.perturb_numeric_deterministic(100.0, 'seed1', 0.07) != supa_privacy.perturb_numeric_deterministic(100.0, 'seed2', 0.07), 'deterministic perturbation did not respect seed'; + + -- test date generalization and shifting + ASSERT supa_privacy.generalize_date('2026-06-08'::date, 'year') = '2026-01-01'::date, 'generalize_date year failed'; + ASSERT supa_privacy.generalize_date('2026-06-08'::date, 'month') = '2026-06-01'::date, 'generalize_date month failed'; + + -- verify deterministic date shifting + ASSERT supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30) = supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30), 'deterministic date shift is not stable'; + ASSERT supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30) != supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed2', 30), 'deterministic date shift did not respect seed'; + ASSERT abs(supa_privacy.shift_date_deterministic('2026-06-08'::date, 'seed1', 30) - '2026-06-08'::date) <= 30, 'deterministic date shift out of bounds'; + + -- test numeric generalization + ASSERT supa_privacy.generalize_numeric(27, 5) = 25, 'generalize_numeric 27/5 failed'; + ASSERT supa_privacy.generalize_numeric(84200, 10000) = 80000, 'generalize_numeric salary failed'; + + RAISE NOTICE '✅ All individual unit tests passed successfully!'; +END; +$$; + +-- --------------------------------------------------------- +-- 3. Testing Masked View Generator... +-- --------------------------------------------------------- + +-- Create temporary mock table +CREATE TEMP TABLE test_users ( + id int PRIMARY KEY, + name text, + email text, + phone text, + salary numeric, + birth_date date, + secret_code text +); + +INSERT INTO test_users VALUES +(1, 'Alice Smith', 'alice@gmail.com', '+1 (202) 555-0143', 95000.50, '1990-04-15', 'top_secret_code_123'), +(2, 'Bob Jones', 'bob@corp.com', '+61 412 000 222', 125000.00, '1985-09-22', 'top_secret_code_456'); + +-- Generate masked view with advanced and custom rules +SELECT supa_privacy.create_masked_view( + 'test_users'::regclass, + 'v_test_users_masked', + '{ + "name": {"type": "hash", "salt": "testsalt"}, + "email": {"type": "email"}, + "phone": {"type": "phone", "keep_digits": 4}, + "salary": {"type": "perturb", "variance": 0.05, "seed_column": "id"}, + "birth_date": {"type": "shift_date", "days": 15, "seed_column": "id"}, + "secret_code": {"type": "custom", "expression": "upper(reverse({col}))"} + }'::jsonb +); + +-- Check that view exists and columns are masked correctly +DO $$ +DECLARE + r1 record; + r2 record; +BEGIN + SELECT * INTO r1 FROM v_test_users_masked WHERE id = 1; + SELECT * INTO r2 FROM v_test_users_masked WHERE id = 1; -- query view again to check deterministic stability + + -- Verify types and values + ASSERT r1.id = 1; + ASSERT r1.email = 'a***e@gmail.com', 'View email masking failed'; + ASSERT r1.phone = '+* (***) ***-0143', 'View phone formatting preservation failed'; + ASSERT abs(r1.birth_date - '1990-04-15'::date) <= 15, 'View birth date shifting failed'; + ASSERT length(r1.name) = 64, 'View name hashing failed'; + ASSERT r1.salary >= 90250.0 AND r1.salary <= 99751.0, 'View salary perturbation failed'; + ASSERT r1.secret_code = '321_EDOC_TERCES_POT', 'View custom expression execution failed'; + + -- Verify deterministic stability (r1 values MUST equal r2 values exactly) + ASSERT r1.salary = r2.salary, 'Deterministic perturbation is not stable in view'; + ASSERT r1.birth_date = r2.birth_date, 'Deterministic date shifting is not stable in view'; + + RAISE NOTICE '✅ Masked View Generator advanced test passed successfully!'; +END; +$$; + +-- Cleanup view and tables +DROP VIEW IF EXISTS v_test_users_masked; +DROP TABLE IF EXISTS test_users; +DROP SCHEMA supa_privacy CASCADE; + +-- --------------------------------------------------------- +-- ✅ All tests passed successfully! +-- --------------------------------------------------------- + +ROLLBACK; -- rollback to keep the test clean diff --git a/supa_profile/README.md b/supa_profile/README.md new file mode 100644 index 0000000..ece4c2e --- /dev/null +++ b/supa_profile/README.md @@ -0,0 +1,96 @@ +# supa_profile - PostgreSQL Database Table & Column Profiler + +`supa_profile` is a relocatable, pure SQL PostgreSQL **Trusted Language Extension (TLE)** designed for native table and column-level statistical profiling. It introspects schema metadata, dynamically constructs parallelized queries, and compiles a comprehensive statistical report (including counts, nullability, distinctness, min/max, string length distributions, averages, medians, percentiles, and frequent value distributions) returned directly as a single structured JSONB document. + +The JSONB output format is fully compatible with the **`ProfilingResult`** TypeScript interface used by frontend profiling dashboards. + +--- + +## 🚀 Installation + +### 1. Enable TLE Support +`supa_profile` runs as a Trusted Language Extension and requires the `pg_tle` extension to be enabled: + +```sql +CREATE EXTENSION IF NOT EXISTS "pg_tle"; +``` + +### 2. Installing from database.dev + +Using the [dbdev CLI](https://supabase.github.io/dbdev): + +```bash +dbdev add -o ./migrations -s extensions -v 1.0.0 package -n "jvent@supa_profile" +``` + +This generates a migration script to load the TLE. Apply the migration, then enable the extension: + +```sql +CREATE EXTENSION "jvent@supa_profile" VERSION '1.0.0' SCHEMA supa_profile; +``` + +--- + +## 🛠️ API Reference + +### 1. Profile Table +Introspects and profiles a physical table, view, or materialized view. +* **Function:** `supa_profile.profile_table(target_table regclass, options jsonb DEFAULT '{}') RETURNS jsonb` +* **Options Parameter (`options`):** + A JSON object supporting the following parameters: + - `scan_field_values` (boolean, default: `true`): Scan column values to compute top value frequency distributions. + - `min_cell_count` (int, default: `5`): Minimum occurrence count of a value to include it in the value distribution list. + - `max_distinct_values` (int, default: `100`): Maximum distinct values returned per column in the value distribution list. + - `rows_per_table` (int, default: `0` = unlimited): Maximum rows to scan. If greater than zero, it wraps the source in a sampled subquery (`LIMIT N`). + - `calculate_numeric_stats` (boolean, default: `true`): Compute average, median, 90th percentile, and 99th percentile for numeric fields. + - `sampling_method` (text, default: `'limit'`): Sampling strategy when limiting scans. Options: + - `'limit'`: Truncates scan to first $N$ rows (`LIMIT N`). Works on all table types, views, and foreign tables. + - `'system'`: Uses PostgreSQL native block-level sampling (`TABLESAMPLE SYSTEM (percentage)`). Extremely fast, but block-based. + - `'bernoulli'`: Uses PostgreSQL native row-level random sampling (`TABLESAMPLE BERNOULLI (percentage)`). Slower than system, but fully random. + - `sample_percentage` (numeric): Percentage of rows to sample (between `0` and `100`). Only used if `sampling_method` is set to `'system'` or `'bernoulli'`. + - `use_estimated_stats` (boolean, default: `false`): Enable instant catalog-based profiling. When `true`, it bypasses active table queries entirely and extracts statistics directly from Postgres catalogs (`pg_class.reltuples` and `pg_stats`). Excellent for tables with billions of rows (completes in < 5ms). + +* **Example:** + ```sql + -- Profile using Bernoulli row-level sampling at 10% + SELECT supa_profile.profile_table( + 'public.users'::regclass, + '{"sampling_method": "bernoulli", "sample_percentage": 10.0}'::jsonb + ); + + -- Profile billions of rows instantly using pg_stats estimates + SELECT supa_profile.profile_table( + 'public.huge_analytics_log'::regclass, + '{"use_estimated_stats": true}'::jsonb + ); + ``` + +--- + +## 🔍 Data Pattern Classifier + +The profiler includes a data classifier that samples column values and matches them against regular expression pattern groups to identify specific fields: +- `EMAIL`: Match handles like `user@corp.com`. +- `UUID`: Match UUID strings like `a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11`. +- `IPV4`: Match IP addresses like `192.168.1.1`. +- `URL`: Match URLs like `https://supabase.com`. +- `DATE`: Match ISO date formats. +- `NUMERIC`: Match integers and floats. +- `.*`: Fallback for generic text fields. + +--- + +## 📊 Output Schema (JSONB) + +The function returns a JSONB object containing: +- `tableStats`: Table name, row count, column count. +- `fields`: Array of column-level statistics. +- `values`: Value frequency distribution array. +- `profiledAt`: Timestamp in UTC. +- `durationMs`: Total duration of the profiling process. +- `queryMode`: Current query mode (`NORMAL`, `FAST` for sampled scans, or `ESTIMATED` for catalog statistics). + +--- + +## 🛡️ License +MIT License. Feel free to use and distribute. diff --git a/supa_profile/supa_profile--1.0.0.sql b/supa_profile/supa_profile--1.0.0.sql new file mode 100644 index 0000000..88a4737 --- /dev/null +++ b/supa_profile/supa_profile--1.0.0.sql @@ -0,0 +1,486 @@ +-- supa_profile extension database schema definition + +-- --------------------------------------------------------------------- +-- 1. PATTERN CLASSIFIER HELPER +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.classify_values(vals text[]) +RETURNS text AS $$ +DECLARE + val text; + total int := 0; + uuid_cnt int := 0; + email_cnt int := 0; + ipv4_cnt int := 0; + url_cnt int := 0; + numeric_cnt int := 0; + date_cnt int := 0; +BEGIN + IF vals IS NULL OR array_length(vals, 1) IS NULL THEN + RETURN '.*'; + END IF; + + total := array_length(vals, 1); + + FOREACH val IN ARRAY vals LOOP + IF val IS NULL OR val = 'NULL' OR val = '' THEN + total := total - 1; + CONTINUE; + END IF; + + IF val ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' THEN + uuid_cnt := uuid_cnt + 1; + ELSIF val ~* '^[a-z0-9.!#$%&''*+/=?^_`{|}~-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$' THEN + email_cnt := email_cnt + 1; + ELSIF val ~* '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' THEN + ipv4_cnt := ipv4_cnt + 1; + ELSIF val ~* '^https?://[^\s/$.?#].[^\s]*$' THEN + url_cnt := url_cnt + 1; + ELSIF val ~* '^-?\d+$' THEN + numeric_cnt := numeric_cnt + 1; + ELSIF val ~* '^\d{4}-\d{2}-\d{2}$' OR val ~* '^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}' THEN + date_cnt := date_cnt + 1; + END IF; + END LOOP; + + IF total <= 0 THEN + RETURN '.*'; + END IF; + + -- If 80% or more match a pattern, return it + IF (uuid_cnt::numeric / total) >= 0.8 THEN RETURN 'UUID'; END IF; + IF (email_cnt::numeric / total) >= 0.8 THEN RETURN 'EMAIL'; END IF; + IF (ipv4_cnt::numeric / total) >= 0.8 THEN RETURN 'IPV4'; END IF; + IF (url_cnt::numeric / total) >= 0.8 THEN RETURN 'URL'; END IF; + IF (date_cnt::numeric / total) >= 0.8 THEN RETURN 'DATE'; END IF; + IF (numeric_cnt::numeric / total) >= 0.8 THEN RETURN 'NUMERIC'; END IF; + + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION @extschema@.infer_pattern(data_type text) +RETURNS text AS $$ +DECLARE + t text := upper(data_type); +BEGIN + IF t = 'UUID' THEN RETURN 'UUID'; END IF; + IF t = 'INET' THEN RETURN 'IPV4'; END IF; + IF t LIKE '%DATE%' THEN RETURN 'YYYY-MM-DD'; END IF; + IF t LIKE '%TIME%' THEN RETURN 'HH:MM:SS'; END IF; + IF t LIKE '%INT%' THEN RETURN '-?\d+'; END IF; + IF t LIKE '%FLOAT%' OR t LIKE '%NUMERIC%' OR t LIKE '%DECIMAL%' OR t LIKE '%DOUBLE%' OR t LIKE '%REAL%' THEN + RETURN '-?\d+\.?\d*'; + END IF; + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + + +-- --------------------------------------------------------------------- +-- 2. TABLE PROFILER FUNCTION +-- --------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.profile_table( + target_table regclass, + options jsonb DEFAULT '{}' +) +RETURNS jsonb AS $$ +DECLARE + -- Options + scan_field_values boolean; + min_cell_count int; + max_distinct_values int; + rows_per_table int; + calculate_numeric_stats boolean; + use_estimated_stats boolean; + sampling_method text; + sample_percentage numeric; + + -- Execution metadata + start_time timestamptz; + duration_ms numeric; + total_rows bigint; + column_count int; + + -- Resolving names + schema_name_val text; + table_name_val text; + full_table_name text; + from_clause text; + + -- Column loops + col record; + is_numeric boolean; + skip_min_max boolean; + + -- SQL Construction + select_exprs text := ''; + query_str text; + result_row record; + result_json jsonb; + + -- Arrays + fields_array jsonb := '[]'::jsonb; + values_array jsonb := '[]'::jsonb; + + -- Temp vars for each column + non_null_count bigint; + distinct_count bigint; + min_val text; + max_val text; + avg_len numeric; + max_len bigint; + avg_val numeric; + median_val numeric; + p90_val numeric; + p99_val numeric; + pattern_val text; + + -- Value Distribution (Unified Parallelized Union Query) + val_query text := ''; + val_rec record; + val_obj jsonb; + field_obj jsonb; + mode_val text; + sample_vals text[]; + + -- Estimated Stats variables + null_fraction numeric; + distinct_stat numeric; + width_stat int; + mcv_vals text[]; + mcf_freqs numeric[]; + i int; +BEGIN + start_time := clock_timestamp(); + full_table_name := target_table::text; + + -- Retrieve resolved schema and table name + SELECT n.nspname, c.relname + INTO schema_name_val, table_name_val + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target_table; + + -- Ensure options is not null + options := coalesce(options, '{}'::jsonb); + + -- Parse options with defaults + scan_field_values := coalesce((options ->> 'scan_field_values')::boolean, true); + min_cell_count := coalesce((options ->> 'min_cell_count')::int, 5); + max_distinct_values := coalesce((options ->> 'max_distinct_values')::int, 100); + rows_per_table := coalesce((options ->> 'rows_per_table')::int, 0); + calculate_numeric_stats := coalesce((options ->> 'calculate_numeric_stats')::boolean, true); + use_estimated_stats := coalesce((options ->> 'use_estimated_stats')::boolean, false); + sampling_method := coalesce(lower(options ->> 'sampling_method'), 'limit'); + sample_percentage := (options ->> 'sample_percentage')::numeric; + + -- Retrieve column count + SELECT COUNT(*)::int INTO column_count + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped; + + -- ----------------------------------------------------------------- + -- BRANCH A: CATALOG-BASED ESTIMATED PROFILING (Fast path) + -- ----------------------------------------------------------------- + IF use_estimated_stats THEN + -- Get estimated row count from pg_class + SELECT reltuples::bigint INTO total_rows + FROM pg_class + WHERE oid = target_table; + + -- Guard against negative estimates + IF total_rows < 0 THEN + total_rows := 0; + END IF; + + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Check if we have stats in pg_stats + SELECT + null_frac, n_distinct, avg_width, + most_common_vals::text::text[], most_common_freqs + INTO null_fraction, distinct_stat, width_stat, mcv_vals, mcf_freqs + FROM pg_stats + WHERE schemaname = schema_name_val AND tablename = table_name_val AND attname = col.column_name; + + IF null_fraction IS NOT NULL THEN + non_null_count := (total_rows * (1.0 - null_fraction))::bigint; + + -- Calculate distinct count + IF distinct_stat < 0 THEN + distinct_count := (abs(distinct_stat) * total_rows)::bigint; + ELSE + distinct_count := distinct_stat::bigint; + END IF; + + avg_len := width_stat::numeric; + max_len := width_stat::bigint; -- fallback estimate + ELSE + -- Fallback defaults if ANALYZE hasn't run + non_null_count := total_rows; + distinct_count := 0; + avg_len := 0; + max_len := 0; + mcv_vals := NULL; + mcf_freqs := NULL; + END IF; + + -- Build value distribution list from MCV/MCF + mode_val := ''; + IF scan_field_values AND mcv_vals IS NOT NULL AND mcf_freqs IS NOT NULL THEN + FOR i IN 1..array_length(mcv_vals, 1) LOOP + IF i = 1 THEN + mode_val := mcv_vals[1]; + END IF; + + val_obj := jsonb_build_object( + 'columnName', col.column_name, + 'value', mcv_vals[i], + 'frequency', (mcf_freqs[i] * total_rows)::bigint, + 'percent', mcf_freqs[i]::numeric(10,4) + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END IF; + + -- Classify pattern from MCV values + IF mcv_vals IS NOT NULL AND array_length(mcv_vals, 1) > 0 THEN + pattern_val := @extschema@.classify_values(mcv_vals); + ELSE + pattern_val := @extschema@.infer_pattern(col.data_type); + END IF; + + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', avg_len, + 'maxLength', max_len, + 'mode', mode_val, + 'pattern', pattern_val + ); + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + + -- ----------------------------------------------------------------- + -- BRANCH B: DYNAMIC QUERY-BASED EXACT PROFILING (Thorough path) + -- ----------------------------------------------------------------- + ELSE + -- Build FROM clause with dynamic sampling options + IF sampling_method IN ('system', 'bernoulli') AND sample_percentage > 0 AND sample_percentage <= 100 THEN + from_clause := full_table_name || ' TABLESAMPLE ' || upper(sampling_method) || ' (' || sample_percentage || ')'; + ELSIF rows_per_table > 0 THEN + from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; + ELSE + from_clause := full_table_name; + END IF; + + -- Build SELECT expressions for column stats + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Determine if type is numeric + is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') + OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); + + -- Determine if type does not support MIN/MAX + skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot|uuid'); + + -- Basic column stats + select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); + select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); + + -- MIN/MAX + IF skip_min_max THEN + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); + ELSE + select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + END IF; + + -- Avg and Max string length + select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); + select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + + -- Numeric stats (AVG, MEDIAN, P90, P99) + IF calculate_numeric_stats AND is_numeric THEN + select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + ELSE + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + END IF; + END LOOP; + + -- Execute main statistics query + query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; + EXECUTE query_str INTO result_row; + result_json := to_jsonb(result_row); + total_rows := (result_json ->> 'total_rows')::bigint; + + -- Build unified parallelized value distribution query if enabled + IF scan_field_values AND total_rows > 0 THEN + FOR col IN + SELECT attname AS column_name + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + IF val_query != '' THEN + val_query := val_query || ' UNION ALL '; + END IF; + val_query := val_query || + 'SELECT * FROM (' || + 'SELECT ' || + quote_literal(col.column_name) || ' AS column_name, ' || + 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || + 'COUNT(*) AS frequency ' || + 'FROM ' || from_clause || ' ' || + 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || + 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || + 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || + 'ORDER BY COUNT(*) DESC ' || + 'LIMIT ' || max_distinct_values || ') q_' || quote_ident(col.column_name); + END LOOP; + + IF val_query != '' THEN + FOR val_rec IN EXECUTE val_query LOOP + val_obj := jsonb_build_object( + 'columnName', val_rec.column_name, + 'value', val_rec.value, + 'frequency', val_rec.frequency, + 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END IF; + END IF; + + -- Build final fields array + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; + distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; + min_val := result_json ->> (col.column_name || '__min'); + max_val := result_json ->> (col.column_name || '__max'); + avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; + max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; + avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; + median_val := (result_json ->> (col.column_name || '__median'))::numeric; + p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; + p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; + + -- Find mode for this column if we scanned values + mode_val := ''; + IF scan_field_values THEN + DECLARE + temp_val jsonb; + BEGIN + FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP + IF temp_val ->> 'columnName' = col.column_name THEN + mode_val := temp_val ->> 'value'; + EXIT; + END IF; + END LOOP; + END; + END IF; + + -- Collect a small sample of values to classify the data pattern + EXECUTE 'SELECT array_agg(' || quote_ident(col.column_name) || '::text) FROM (SELECT ' || quote_ident(col.column_name) || ' FROM ' || from_clause || ' WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL LIMIT 50) t' INTO sample_vals; + + IF sample_vals IS NOT NULL AND array_length(sample_vals, 1) > 0 THEN + pattern_val := @extschema@.classify_values(sample_vals); + ELSE + pattern_val := @extschema@.infer_pattern(col.data_type); + END IF; + + -- Construct field profile object matching FieldProfile interface + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', coalesce(avg_len, 0), + 'maxLength', coalesce(max_len, 0), + 'mode', mode_val, + 'pattern', pattern_val + ); + + -- Add optional numeric and min/max stats + IF min_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('minValue', min_val); + END IF; + IF max_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('maxValue', max_val); + END IF; + IF avg_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('avgValue', avg_val); + END IF; + IF median_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('median', median_val); + END IF; + IF p90_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p90', p90_val); + END IF; + IF p99_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p99', p99_val); + END IF; + + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + END IF; + + duration_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000; + + -- Return full result matching ProfilingResult interface + RETURN jsonb_build_object( + 'tableStats', jsonb_build_object( + 'tableName', full_table_name, + 'rowCount', total_rows, + 'columnCount', column_count, + 'selected', false + ), + 'fields', fields_array, + 'values', values_array, + 'profiledAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'durationMs', round(duration_ms, 2), + 'queryMode', CASE + WHEN use_estimated_stats THEN 'ESTIMATED' + WHEN rows_per_table > 0 OR sample_percentage > 0 THEN 'FAST' + ELSE 'NORMAL' + END + ); +END; +$$ LANGUAGE plpgsql VOLATILE STRICT; diff --git a/supa_profile/supa_profile.control b/supa_profile/supa_profile.control new file mode 100644 index 0000000..b849199 --- /dev/null +++ b/supa_profile/supa_profile.control @@ -0,0 +1,5 @@ +# supa_profile extension for PostgreSQL +comment = 'Database table and column statistical profiler' +default_version = '1.0.0' +relocatable = true +superuser = false diff --git a/supa_profile/test_validation.sql b/supa_profile/test_validation.sql new file mode 100644 index 0000000..c349b15 --- /dev/null +++ b/supa_profile/test_validation.sql @@ -0,0 +1,622 @@ +-- ===================================================================== +-- SUPA_PROFILE EXTENSION VALIDATION TEST SCRIPT +-- Run this script to verify the correctness of the profiling functions. +-- ===================================================================== + +BEGIN; + +-- --------------------------------------------------------- +-- 1. Loading extension schema and functions... +-- --------------------------------------------------------- +CREATE SCHEMA IF NOT EXISTS supa_profile; + +CREATE OR REPLACE FUNCTION supa_profile.classify_values(vals text[]) +RETURNS text AS $$ +DECLARE + val text; + total int := 0; + uuid_cnt int := 0; + email_cnt int := 0; + ipv4_cnt int := 0; + url_cnt int := 0; + numeric_cnt int := 0; + date_cnt int := 0; +BEGIN + IF vals IS NULL OR array_length(vals, 1) IS NULL THEN + RETURN '.*'; + END IF; + + total := array_length(vals, 1); + + FOREACH val IN ARRAY vals LOOP + IF val IS NULL OR val = 'NULL' OR val = '' THEN + total := total - 1; + CONTINUE; + END IF; + + IF val ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' THEN + uuid_cnt := uuid_cnt + 1; + ELSIF val ~* '^[a-z0-9.!#$%&''*+/=?^_`{|}~-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$' THEN + email_cnt := email_cnt + 1; + ELSIF val ~* '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' THEN + ipv4_cnt := ipv4_cnt + 1; + ELSIF val ~* '^https?://[^\s/$.?#].[^\s]*$' THEN + url_cnt := url_cnt + 1; + ELSIF val ~* '^-?\d+$' THEN + numeric_cnt := numeric_cnt + 1; + ELSIF val ~* '^\d{4}-\d{2}-\d{2}$' OR val ~* '^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}' THEN + date_cnt := date_cnt + 1; + END IF; + END LOOP; + + IF total <= 0 THEN + RETURN '.*'; + END IF; + + IF (uuid_cnt::numeric / total) >= 0.8 THEN RETURN 'UUID'; END IF; + IF (email_cnt::numeric / total) >= 0.8 THEN RETURN 'EMAIL'; END IF; + IF (ipv4_cnt::numeric / total) >= 0.8 THEN RETURN 'IPV4'; END IF; + IF (url_cnt::numeric / total) >= 0.8 THEN RETURN 'URL'; END IF; + IF (date_cnt::numeric / total) >= 0.8 THEN RETURN 'DATE'; END IF; + IF (numeric_cnt::numeric / total) >= 0.8 THEN RETURN 'NUMERIC'; END IF; + + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION supa_profile.infer_pattern(data_type text) +RETURNS text AS $$ +DECLARE + t text := upper(data_type); +BEGIN + IF t = 'UUID' THEN RETURN 'UUID'; END IF; + IF t = 'INET' THEN RETURN 'IPV4'; END IF; + IF t LIKE '%DATE%' THEN RETURN 'YYYY-MM-DD'; END IF; + IF t LIKE '%TIME%' THEN RETURN 'HH:MM:SS'; END IF; + IF t LIKE '%INT%' THEN RETURN '-?\d+'; END IF; + IF t LIKE '%FLOAT%' OR t LIKE '%NUMERIC%' OR t LIKE '%DECIMAL%' OR t LIKE '%DOUBLE%' OR t LIKE '%REAL%' THEN + RETURN '-?\d+\.?\d*'; + END IF; + RETURN '.*'; +END; +$$ LANGUAGE plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION supa_profile.profile_table( + target_table regclass, + options jsonb DEFAULT '{}' +) +RETURNS jsonb AS $$ +DECLARE + -- Options + scan_field_values boolean; + min_cell_count int; + max_distinct_values int; + rows_per_table int; + calculate_numeric_stats boolean; + use_estimated_stats boolean; + sampling_method text; + sample_percentage numeric; + + -- Execution metadata + start_time timestamptz; + duration_ms numeric; + total_rows bigint; + column_count int; + + -- Resolving names + schema_name_val text; + table_name_val text; + full_table_name text; + from_clause text; + + -- Column loops + col record; + is_numeric boolean; + skip_min_max boolean; + + -- SQL Construction + select_exprs text := ''; + query_str text; + result_row record; + result_json jsonb; + + -- Arrays + fields_array jsonb := '[]'::jsonb; + values_array jsonb := '[]'::jsonb; + + -- Temp vars for each column + non_null_count bigint; + distinct_count bigint; + min_val text; + max_val text; + avg_len numeric; + max_len bigint; + avg_val numeric; + median_val numeric; + p90_val numeric; + p99_val numeric; + pattern_val text; + + -- Value Distribution (Unified Parallelized Union Query) + val_query text := ''; + val_rec record; + val_obj jsonb; + field_obj jsonb; + mode_val text; + sample_vals text[]; + + -- Estimated Stats variables + null_fraction numeric; + distinct_stat numeric; + width_stat int; + mcv_vals text[]; + mcf_freqs numeric[]; + i int; +BEGIN + start_time := clock_timestamp(); + full_table_name := target_table::text; + + -- Retrieve resolved schema and table name + SELECT n.nspname, c.relname + INTO schema_name_val, table_name_val + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target_table; + + -- Ensure options is not null + options := coalesce(options, '{}'::jsonb); + + -- Parse options with defaults + scan_field_values := coalesce((options ->> 'scan_field_values')::boolean, true); + min_cell_count := coalesce((options ->> 'min_cell_count')::int, 5); + max_distinct_values := coalesce((options ->> 'max_distinct_values')::int, 100); + rows_per_table := coalesce((options ->> 'rows_per_table')::int, 0); + calculate_numeric_stats := coalesce((options ->> 'calculate_numeric_stats')::boolean, true); + use_estimated_stats := coalesce((options ->> 'use_estimated_stats')::boolean, false); + sampling_method := coalesce(lower(options ->> 'sampling_method'), 'limit'); + sample_percentage := (options ->> 'sample_percentage')::numeric; + + -- Retrieve column count + SELECT COUNT(*)::int INTO column_count + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped; + + -- ----------------------------------------------------------------- + -- BRANCH A: CATALOG-BASED ESTIMATED PROFILING (Fast path) + -- ----------------------------------------------------------------- + IF use_estimated_stats THEN + -- Get estimated row count from pg_class + SELECT reltuples::bigint INTO total_rows + FROM pg_class + WHERE oid = target_table; + + -- Guard against negative estimates + IF total_rows < 0 THEN + total_rows := 0; + END IF; + + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Check if we have stats in pg_stats + SELECT + null_frac, n_distinct, avg_width, + most_common_vals::text::text[], most_common_freqs + INTO null_fraction, distinct_stat, width_stat, mcv_vals, mcf_freqs + FROM pg_stats + WHERE schemaname = schema_name_val AND tablename = table_name_val AND attname = col.column_name; + + IF null_fraction IS NOT NULL THEN + non_null_count := (total_rows * (1.0 - null_fraction))::bigint; + + -- Calculate distinct count + IF distinct_stat < 0 THEN + distinct_count := (abs(distinct_stat) * total_rows)::bigint; + ELSE + distinct_count := distinct_stat::bigint; + END IF; + + avg_len := width_stat::numeric; + max_len := width_stat::bigint; + ELSE + non_null_count := total_rows; + distinct_count := 0; + avg_len := 0; + max_len := 0; + mcv_vals := NULL; + mcf_freqs := NULL; + END IF; + + -- Build value distribution list from MCV/MCF + mode_val := ''; + IF scan_field_values AND mcv_vals IS NOT NULL AND mcf_freqs IS NOT NULL THEN + FOR i IN 1..array_length(mcv_vals, 1) LOOP + IF i = 1 THEN + mode_val := mcv_vals[1]; + END IF; + + val_obj := jsonb_build_object( + 'columnName', col.column_name, + 'value', mcv_vals[i], + 'frequency', (mcf_freqs[i] * total_rows)::bigint, + 'percent', mcf_freqs[i]::numeric(10,4) + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END IF; + + -- Classify pattern from MCV values + IF mcv_vals IS NOT NULL AND array_length(mcv_vals, 1) > 0 THEN + pattern_val := supa_profile.classify_values(mcv_vals); + ELSE + pattern_val := supa_profile.infer_pattern(col.data_type); + END IF; + + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', avg_len, + 'maxLength', max_len, + 'mode', mode_val, + 'pattern', pattern_val + ); + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + + -- ----------------------------------------------------------------- + -- BRANCH B: DYNAMIC QUERY-BASED EXACT PROFILING (Thorough path) + -- ----------------------------------------------------------------- + ELSE + -- Build FROM clause with dynamic sampling options + IF sampling_method IN ('system', 'bernoulli') AND sample_percentage > 0 AND sample_percentage <= 100 THEN + from_clause := full_table_name || ' TABLESAMPLE ' || upper(sampling_method) || ' (' || sample_percentage || ')'; + ELSIF rows_per_table > 0 THEN + from_clause := '(SELECT * FROM ' || full_table_name || ' LIMIT ' || rows_per_table || ') AS sampled_table'; + ELSE + from_clause := full_table_name; + END IF; + + -- Build SELECT expressions for column stats + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + -- Determine if type is numeric + is_numeric := (col.data_type IN ('smallint', 'integer', 'bigint', 'decimal', 'numeric', 'real', 'double precision') + OR col.data_type ~* 'int|numeric|decimal|real|double|float|number'); + + -- Determine if type does not support MIN/MAX + skip_min_max := (col.data_type ~* '\[\]|json|jsonb|bytea|xml|geometry|geography|box|circle|line|lseg|path|point|polygon|xid|cid|oid|tid|txid_snapshot|uuid'); + + -- Basic column stats + select_exprs := select_exprs || ', COUNT(' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__non_null'); + select_exprs := select_exprs || ', COUNT(DISTINCT ' || quote_ident(col.column_name) || ') AS ' || quote_ident(col.column_name || '__distinct'); + + -- MIN/MAX + IF skip_min_max THEN + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', NULL::text AS ' || quote_ident(col.column_name || '__max'); + ELSE + select_exprs := select_exprs || ', MIN(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__min'); + select_exprs := select_exprs || ', MAX(' || quote_ident(col.column_name) || ')::text AS ' || quote_ident(col.column_name || '__max'); + END IF; + + -- Avg and Max string length + select_exprs := select_exprs || ', AVG(length(' || quote_ident(col.column_name) || '::text))::numeric AS ' || quote_ident(col.column_name || '__avg_len'); + select_exprs := select_exprs || ', MAX(length(' || quote_ident(col.column_name) || '::text))::bigint AS ' || quote_ident(col.column_name || '__max_len'); + + -- Numeric stats (AVG, MEDIAN, P90, P99) + IF calculate_numeric_stats AND is_numeric THEN + select_exprs := select_exprs || ', AVG(' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', percentile_disc(0.5) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', percentile_disc(0.9) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', percentile_disc(0.99) WITHIN GROUP (ORDER BY ' || quote_ident(col.column_name) || ')::numeric AS ' || quote_ident(col.column_name || '__p99'); + ELSE + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__avg'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__median'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p90'); + select_exprs := select_exprs || ', NULL::numeric AS ' || quote_ident(col.column_name || '__p99'); + END IF; + END LOOP; + + -- Execute main statistics query + query_str := 'SELECT COUNT(*) AS total_rows' || select_exprs || ' FROM ' || from_clause; + EXECUTE query_str INTO result_row; + result_json := to_jsonb(result_row); + total_rows := (result_json ->> 'total_rows')::bigint; + + -- Build unified parallelized value distribution query if enabled + IF scan_field_values AND total_rows > 0 THEN + FOR col IN + SELECT attname AS column_name + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + IF val_query != '' THEN + val_query := val_query || ' UNION ALL '; + END IF; + val_query := val_query || + 'SELECT * FROM (' || + 'SELECT ' || + quote_literal(col.column_name) || ' AS column_name, ' || + 'coalesce(' || quote_ident(col.column_name) || '::text, ''NULL'') AS value, ' || + 'COUNT(*) AS frequency ' || + 'FROM ' || from_clause || ' ' || + 'WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL ' || + 'GROUP BY ' || quote_ident(col.column_name) || '::text ' || + 'HAVING COUNT(*) >= ' || min_cell_count || ' ' || + 'ORDER BY COUNT(*) DESC ' || + 'LIMIT ' || max_distinct_values || ') q_' || quote_ident(col.column_name); + END LOOP; + + IF val_query != '' THEN + FOR val_rec IN EXECUTE val_query LOOP + val_obj := jsonb_build_object( + 'columnName', val_rec.column_name, + 'value', val_rec.value, + 'frequency', val_rec.frequency, + 'percent', CASE WHEN total_rows > 0 THEN (val_rec.frequency::numeric / total_rows)::numeric(10,4) ELSE 0 END + ); + values_array := values_array || jsonb_build_array(val_obj); + END LOOP; + END IF; + END IF; + + -- Build final fields array + FOR col IN + SELECT + attname AS column_name, + format_type(atttypid, atttypmod) AS data_type, + NOT attnotnull AS is_nullable + FROM pg_attribute + WHERE attrelid = target_table AND attnum > 0 AND NOT attisdropped + ORDER BY attnum + LOOP + non_null_count := (result_json ->> (col.column_name || '__non_null'))::bigint; + distinct_count := (result_json ->> (col.column_name || '__distinct'))::bigint; + min_val := result_json ->> (col.column_name || '__min'); + max_val := result_json ->> (col.column_name || '__max'); + avg_len := (result_json ->> (col.column_name || '__avg_len'))::numeric; + max_len := (result_json ->> (col.column_name || '__max_len'))::bigint; + avg_val := (result_json ->> (col.column_name || '__avg'))::numeric; + median_val := (result_json ->> (col.column_name || '__median'))::numeric; + p90_val := (result_json ->> (col.column_name || '__p90'))::numeric; + p99_val := (result_json ->> (col.column_name || '__p99'))::numeric; + + -- Find mode for this column if we scanned values + mode_val := ''; + IF scan_field_values THEN + DECLARE + temp_val jsonb; + BEGIN + FOR temp_val IN SELECT * FROM jsonb_array_elements(values_array) LOOP + IF temp_val ->> 'columnName' = col.column_name THEN + mode_val := temp_val ->> 'value'; + EXIT; + END IF; + END LOOP; + END; + END IF; + + -- Collect a small sample of values to classify the data pattern + EXECUTE 'SELECT array_agg(' || quote_ident(col.column_name) || '::text) FROM (SELECT ' || quote_ident(col.column_name) || ' FROM ' || from_clause || ' WHERE ' || quote_ident(col.column_name) || ' IS NOT NULL LIMIT 50) t' INTO sample_vals; + + IF sample_vals IS NOT NULL AND array_length(sample_vals, 1) > 0 THEN + pattern_val := supa_profile.classify_values(sample_vals); + ELSE + pattern_val := supa_profile.infer_pattern(col.data_type); + END IF; + + -- Construct field profile object matching FieldProfile interface + field_obj := jsonb_build_object( + 'tableName', full_table_name, + 'columnName', col.column_name, + 'dataType', col.data_type, + 'nullable', col.is_nullable, + 'nonNullCount', non_null_count, + 'nullCount', total_rows - non_null_count, + 'distinctCount', distinct_count, + 'avgLength', coalesce(avg_len, 0), + 'maxLength', coalesce(max_len, 0), + 'mode', mode_val, + 'pattern', pattern_val + ); + + -- Add optional numeric and min/max stats + IF min_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('minValue', min_val); + END IF; + IF max_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('maxValue', max_val); + END IF; + IF avg_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('avgValue', avg_val); + END IF; + IF median_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('median', median_val); + END IF; + IF p90_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p90', p90_val); + END IF; + IF p99_val IS NOT NULL THEN + field_obj := field_obj || jsonb_build_object('p99', p99_val); + END IF; + + fields_array := fields_array || jsonb_build_array(field_obj); + END LOOP; + END IF; + + duration_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000; + + -- Return full result matching ProfilingResult interface + RETURN jsonb_build_object( + 'tableStats', jsonb_build_object( + 'tableName', full_table_name, + 'rowCount', total_rows, + 'columnCount', column_count, + 'selected', false + ), + 'fields', fields_array, + 'values', values_array, + 'profiledAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'durationMs', round(duration_ms, 2), + 'queryMode', CASE + WHEN use_estimated_stats THEN 'ESTIMATED' + WHEN rows_per_table > 0 OR sample_percentage > 0 THEN 'FAST' + ELSE 'NORMAL' + END + ); +END; +$$ LANGUAGE plpgsql VOLATILE STRICT; + +-- --------------------------------------------------------- +-- 2. Create mock table and populate data... +-- --------------------------------------------------------- + +CREATE TEMP TABLE mock_users ( + id serial PRIMARY KEY, + username text NOT NULL, + email text, + uuid_val uuid, + ip_val text, + url_val text, + age int, + salary numeric(10, 2), + signup_date date +); + +INSERT INTO mock_users (username, email, uuid_val, ip_val, url_val, age, salary, signup_date) VALUES +('alice', 'alice@corp.com', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::uuid, '192.168.1.1', 'https://supabase.com', 25, 80000.00, '2025-01-10'::date), +('bob', 'bob@corp.com', 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22'::uuid, '192.168.1.2', 'https://google.com', 30, 95000.00, '2025-02-15'::date), +('charlie', 'charlie@gmail.com', 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33'::uuid, '10.0.0.1', 'https://github.com', 45, 120000.50, '2024-11-20'::date), +('diana', 'diana@corp.com', 'd0eebc99-9c0b-4ef8-bb6d-6bb9bd380a44'::uuid, '10.0.0.2', 'https://database.dev', 25, 80000.00, '2025-03-01'::date), +('diana', 'diana@corp.com', 'e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a55'::uuid, '10.0.0.3', 'https://database.dev', 25, NULL::numeric, NULL::date); + +-- --------------------------------------------------------- +-- 3. Execute unit tests... +-- --------------------------------------------------------- +DO $$ +DECLARE + res jsonb; + fields jsonb; + vals jsonb; + f record; + v record; +BEGIN + -- Profile the mock table with default options + res := supa_profile.profile_table('mock_users'::regclass, '{"min_cell_count": 2}'::jsonb); + + -- Verify tableStats + ASSERT res -> 'tableStats' ->> 'tableName' = 'mock_users', 'Table stats name mismatch'; + ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 5, 'Table stats row count mismatch'; + ASSERT (res -> 'tableStats' ->> 'columnCount')::int = 9, 'Table stats column count mismatch'; + + fields := res -> 'fields'; + vals := res -> 'values'; + + -- Loop through fields and check pattern classifiers and basic validations + FOR f IN SELECT * FROM jsonb_to_recordset(fields) AS ( + "columnName" text, "dataType" text, "nullable" boolean, + "nonNullCount" int, "nullCount" int, "distinctCount" int, + "minValue" text, "maxValue" text, "avgValue" numeric, + "median" numeric, "mode" text, "pattern" text + ) LOOP + IF f."columnName" = 'id' THEN + ASSERT f."nullable" = false, 'id should not be nullable'; + ASSERT f."nonNullCount" = 5, 'id non-null count mismatch'; + ASSERT f."distinctCount" = 5, 'id distinct count mismatch'; + ASSERT f."pattern" = 'NUMERIC', 'id pattern classification failed (expected NUMERIC)'; + ELSIF f."columnName" = 'username' THEN + ASSERT f."mode" = 'diana', 'username mode mismatch (should be diana, frequency = 2)'; + ASSERT f."pattern" = '.*', 'username pattern classification failed (expected .*)'; + ELSIF f."columnName" = 'email' THEN + ASSERT f."pattern" = 'EMAIL', 'email pattern classification failed (expected EMAIL)'; + ELSIF f."columnName" = 'uuid_val' THEN + ASSERT f."pattern" = 'UUID', 'uuid_val pattern classification failed (expected UUID)'; + ELSIF f."columnName" = 'ip_val' THEN + ASSERT f."pattern" = 'IPV4', 'ip_val pattern classification failed (expected IPV4)'; + ELSIF f."columnName" = 'url_val' THEN + ASSERT f."pattern" = 'URL', 'url_val pattern classification failed (expected URL)'; + ELSIF f."columnName" = 'signup_date' THEN + ASSERT f."pattern" = 'DATE', 'signup_date pattern classification failed (expected DATE)'; + ELSIF f."columnName" = 'age' THEN + ASSERT f."pattern" = 'NUMERIC', 'age pattern classification failed (expected NUMERIC)'; + ASSERT f."median" = 25.00, 'age median mismatch'; + END IF; + END LOOP; + + -- Verify value distribution (we filtered with min_cell_count = 2) + -- Values expected to be frequent: age=25 (freq=3), email=diana@corp.com (freq=2), country/url_val=https://database.dev (freq=2) + DECLARE + age_25_found boolean := false; + email_diana_found boolean := false; + url_dbdev_found boolean := false; + BEGIN + FOR v IN SELECT * FROM jsonb_to_recordset(vals) AS ( + "columnName" text, "value" text, "frequency" int, "percent" numeric + ) LOOP + IF v."columnName" = 'age' AND v."value" = '25' THEN + ASSERT v."frequency" = 3, 'age 25 frequency mismatch'; + age_25_found := true; + ELSIF v."columnName" = 'email' AND v."value" = 'diana@corp.com' THEN + ASSERT v."frequency" = 2, 'email diana frequency mismatch'; + email_diana_found := true; + ELSIF v."columnName" = 'url_val' AND v."value" = 'https://database.dev' THEN + ASSERT v."frequency" = 2, 'url_val frequency mismatch'; + url_dbdev_found := true; + END IF; + END LOOP; + + ASSERT age_25_found, 'age 25 value distribution missing'; + ASSERT email_diana_found, 'email diana value distribution missing'; + ASSERT url_dbdev_found, 'url_val database.dev value distribution missing'; + END; + + -- Test TABLESAMPLE system/bernoulli options + res := supa_profile.profile_table('mock_users'::regclass, '{"sampling_method": "bernoulli", "sample_percentage": 100}'::jsonb); + ASSERT res ->> 'queryMode' = 'FAST', 'FAST queryMode mismatch for sampled table'; + ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 5, 'tablesample result size mismatch'; + + -- Test CATALOG ESTIMATIONS + -- Execute ANALYZE so that pg_stats gets populated for our temp table + -- Temp tables need manual ANALYZE to populate statistics + ANALYZE mock_users; + + res := supa_profile.profile_table('mock_users'::regclass, '{"use_estimated_stats": true}'::jsonb); + ASSERT res ->> 'queryMode' = 'ESTIMATED', 'ESTIMATED queryMode mismatch'; + ASSERT (res -> 'tableStats' ->> 'rowCount')::int = 5, 'Estimated row count mismatch'; + + -- Verify pattern classification on catalog estimations + FOR f IN SELECT * FROM jsonb_to_recordset(res -> 'fields') AS ("columnName" text, "pattern" text) LOOP + IF f."columnName" = 'email' THEN + ASSERT f."pattern" = 'EMAIL', 'Estimated email pattern failed'; + ELSIF f."columnName" = 'uuid_val' THEN + ASSERT f."pattern" = 'UUID', 'Estimated uuid pattern failed'; + END IF; + END LOOP; + + RAISE NOTICE '✅ All supa_profile unit tests passed successfully!'; +END; +$$; + +DROP TABLE IF EXISTS mock_users; +DROP SCHEMA supa_profile CASCADE; + +ROLLBACK;