Better gigamap queries

[hrstoyanov]

Is your feature request related to a problem? Please describe.

@fh-ms @hg-ms @zdenek-jonas : This is not a bug, but a feature suggestion sketch, that I use in my code and I think the ES community can benefit from, if incorporated in ES.

Currently the GigaMap queries, while comprehensive, expose the indexers to clients that use them. Indexers are consider internal implementation detail for a persistent entity, not to be exposed in queries api - think how indexes work in relation/sql databases - they are invisible to queries.

Describe the solution you'd like

I want to share some piece of code that mitigates this issue, by adding a thin layer shielding the calls sites from having to reference internal indexers for queries. This leads to much nicer-to-read code and the benefits compound if you have lot of queries.
This code also applies to mutating operations as well.

Describe alternatives you've considered

Consider the attached code - I propose this to be incorporated in ES at some point.
There are two components - Store and Query builder.

1. Store allows for bolier-plate operations - create(if missing) , update, remove. It also creates a Query builders for use by entity clients.
2. Query Builder provides fluent, indexer-hinding way to construct much nicer queries.

---

Examples:

Say you have a an product rating entity:

/// Caches the aggregated rating statistics (average score, total reviews) for a Product.
public final class AggregateRating implements WithTsid {
    
    private final Tsid id;
    private final Tsid productId;
    private float averageRating;
    private int totalReviews;
    
    // ... enum Score {TERRIBLE(1),POOR(2),AVERAGE(3),VERY_GOOD(4),EXCELLENT(5);
    private final Map<Rating.Score, Integer> ratingDistribution;

 // some code  omitted

    public static class Store extends AbstractEntityStore<Tsid, AggregateRating> {

       //add your indexers - they are private!
      
        private static final BinaryIndexerTsid<AggregateRating> PRODUCT_INDEXER = new BinaryIndexerTsid<>() {
            @Override
            protected Tsid getTsid(AggregateRating entity) {
                return entity.productTsid();
            }
        };

        private static final IndexerFloat<AggregateRating> AVERAGE_RATING_INDEXER = new IndexerFloat.Abstract<>() {
            @Override
            public Float getFloat(AggregateRating entity) {
                return entity.averageRating();
            }
        };

        private static final IndexerInteger<AggregateRating> TOTAL_REVIEWS_INDEXER = new IndexerInteger.Abstract<>() {
            @Override
            public Integer getInteger(AggregateRating entity) {
                return entity.totalReviews();
            }
        };

        public Store() {
            super(GigaMap.<AggregateRating>Builder()
                    .withBitmapIdentityIndex(WithTsid.TSID_INDEXER_BINARY)
                    .withBitmapIndex(PRODUCT_INDEXER)
                    .withBitmapIndex(AVERAGE_RATING_INDEXER)
                    .withBitmapIndex(TOTAL_REVIEWS_INDEXER)
                    .build());
        }

        @Override
        public Optional<AggregateRating> findById(Tsid id) {
            return WithTsid.findOne(gigaMap, id);
        }

        @Override
        public Optional<AggregateRating> addOrGetExisting(AggregateRating entity) {
            var existing = findByProduct(entity.productTsid());
            if (existing.isPresent()) return existing;
            add(entity);
            return Optional.empty();
        }

        @Override
        public void add(AggregateRating entity) {
            super.add(entity);
            RootData.getEclipseStoreContext().addAllToPersist(entity,gigaMap);
        }

        @Override
        public void update(AggregateRating entity, Consumer<AggregateRating> updater) {
            super.update(entity, updater);
            TourBizData.getEclipseStoreContext().addAllToPersist(entity,gigaMap);
        }

        @Override
        public boolean remove(AggregateRating entity) {
            if (super.remove(entity)) {
                TourBizData.getEclipseStoreContext().addToPersist(gigaMap);
                return true;
            }
            return false;
        }

        public Optional<AggregateRating> findByProduct(Tsid productId) {
            return gigaMap.query(PRODUCT_INDEXER.is(productId)).findFirst();
        }

        public void updateMetric(Tsid productTsid, Consumer<AggregateRating> metricUpdater) {
            var agg = findByProduct(productTsid).orElseGet(() -> {
                var newAgg = new AggregateRating(productTsid);
                add(newAgg);
                return newAgg;
            });
            update(agg, metricUpdater);
        }      

        //Create a query builder off this store 
        public QueryBuilder query() {
            return new QueryBuilder();
        }

        public class QueryBuilder extends GigaQueryBuilder<AggregateRating, QueryBuilder> {
            public QueryBuilder() {
                super(Store.this.gigaMap);
            }

            @Override
            protected QueryBuilder self() {
                return this;
            }

            public QueryBuilder product(Tsid... productIds) {
                return where(PRODUCT_INDEXER, productIds);
            }

            public QueryBuilder averageRatingGreaterThan(float rating) {
                add(AVERAGE_RATING_INDEXER.greaterThan(rating));
                return this;
            }

            public QueryBuilder totalReviewsGreaterThan(int count) {
                add(TOTAL_REVIEWS_INDEXER.greaterThan(count));
                return this;
            }
        }
    }
}

Then you can query it like that:

var store = dataRoot.aggreagteRationsStore()
var results = store.query()
                        .product(1,2,3)
                        .averageRatingGreaterThan(3.0)
                        .totalReviewsGreaterThan(1000)
  .toList();

Additional context

1. GigaQueryBuilder place nicely with GigaMap queries - it can be extended with inheritance or composition of gigamap queries

2. At some point the Store/QueryBuilder can be autogenerated much like the code generators for layered entities

3. If you look at Spring Data and Jakarta EE Data specs - theyr are going in the same directions - convinience by default, that can be overridden and extended.

Note - attached files. EclipseContextStore.java is included in another feature request

Creator.java
AbstractEntityStore.java
GigaQueryBuilder.java

[hrstoyanov]

Btw, the above can be greatly simplified with a annotation processing/code generators, like the ones ES already has for entity interfaces.

[fh-ms]

Btw, the above can be greatly simplified with a annotation processing/code generators, like the ones ES already has for entity interfaces.

Have a look at #699

[hrstoyanov]

Thanks @fh-ms - this is close to what I want - a way to hide the indexers at the caller site, when coding queries.

How does this work with another ES (older) feature I use a lot - layered entities and versioning? Can you, please, add some junit tests to demonstrate interoperability and clean integration with layered entities codegen as part of #699 ?

[hrstoyanov]

@fh-ms - Here is an example of how I use layered entities with built-in Gigamap wrapper (see the inner class Store and QueryBuildr) - it would make sense if you also look at the other code I attached above ^^^.

A lot of the code is boilerplate . Ideally I would prefer if I can use the layered entities codegen + a gigamap query code generator to shrink my code

import com.github.f4b6a3.tsid.Tsid;
import org.apache.lucene.document.Document;
import org.eclipse.serializer.entity.EntityVersionContext;
import org.eclipse.serializer.reference.Lazy;
import org.eclipse.store.gigamap.types.Indexer;
import org.eclipse.store.gigamap.types.IndexerMultiValue;
import peruncs.eclipsestore.core.*;
import peruncs.security.audit.CRUDAuditEvent;
import peruncs.security.authz.Actor;
import peruncs.security.authz.ElectronicAddress;
import peruncs.utilities.TypeSafeMixedMap;
import peruncs.utilities.WithId;
import peruncs.utilities.geocode.Country;
import tourbiz.datamodel.VersionedBaseEntity;
import tourbiz.datamodel.customer._Customer.CustomerCreator;
import tourbiz.datamodel.customer._Customer.CustomerUpdater;

import java.time.Instant;
import java.time.LocalDate;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;

/// A participant consumer of a booking.
public interface Customer extends VersionedBaseEntity {

    /// Logger for Customer entity operations.
    ESEntityLogger ENTITY_LOGGER = new ESEntityLogger(Customer.class.getName());

    /// Creates a new [CustomerCreator] for building Customer instances.
    ///
    /// @return a new [CustomerCreator] instance.
    static CustomerCreator creator() {
        return CustomerCreator.New()
                .addLayer(ENTITY_LOGGER)
                .addLayer(EntityVersionContext.AutoIncrementingLong())
                .id(TsidGenerator.next())
                .created(CRUDAuditEvent.created("Initial creation").register())
                .name("New Customer")
                .userId(null)
                .lazyContactAddresses(Lazy.Reference(List.of()))
                .dateOfBirth(null)
                .gender(null)
                .anonymizedAt(null)
                .lazyCountryOfCitizenship(Lazy.Reference(Set.of()))
                .lazyCountryOfResidence(Lazy.Reference(Set.of()))
                .lazyTravelDocuments(Lazy.Reference(Map.of()))
                .lazySpecialNeeds(Lazy.Reference(Set.of()))
                .lazyAdditionalFields(Lazy.Reference(TypeSafeMixedMap.of()))
                .lazyNotes(Lazy.Reference(null))
                .lazyInternalNotes(Lazy.Reference(null))
                .lazyEmergencyContactPerson(Lazy.Reference(null));
    }

    /// Creates a new [CustomerUpdater] for modifying an existing Customer instance.
    static CustomerUpdater updater(Customer customerEntity) {
        return CustomerUpdater.New(customerEntity);
    }

    ///  Optional TourBiz user registration.
    Actor userId();

    Lazy<List<ElectronicAddress>> lazyContactAddresses();

    /// Contact electronic addresses for this customer.
    default List<ElectronicAddress> contactAddresses() {
        return Lazy.get(lazyContactAddresses());
    }

    ///  Optional Date of birth
    LocalDate dateOfBirth();

    /// Optional gender.
    Gender gender();

    /// Timestamp when the customer data was anonymized (null if active/valid).
    Instant anonymizedAt();

    Lazy<Set<Country>> lazyCountryOfCitizenship();

    ///  Countries of citizenship.
    default Set<Country> countryOfCitizenship() {
        return Lazy.get(lazyCountryOfCitizenship());
    }

    Lazy<Set<Country>> lazyCountryOfResidence();

    ///  Countries of residence.
    default Set<Country> countryOfResidence() {
        return Lazy.get(lazyCountryOfResidence());
    }

    Lazy<Map<Country, TravelDocument>> lazyTravelDocuments();

    /// Government issues documents, visas, etc.
    default Map<Country, TravelDocument> travelDocuments() {
        return Lazy.get(lazyTravelDocuments());
    }

    Lazy<Set<SpecialNeed>> lazySpecialNeeds();

    /// Special needs
    default Set<SpecialNeed> specialNeeds() {
        return Lazy.get(lazySpecialNeeds());
    }

    Lazy<TypeSafeMixedMap<String>> lazyAdditionalFields();

    /// Open-ended list of additional attributes
    default TypeSafeMixedMap<String> additionalFields() {
        return Lazy.get(lazyAdditionalFields());
    }

    Lazy<ContactPerson> lazyEmergencyContactPerson();

    /// Emergency contact person for this customer,cannot be the customer him/her-self!
    default ContactPerson emergencyContactPerson() {
        return Lazy.get(lazyEmergencyContactPerson());
    }

    // Customer's gender
    enum Gender {
        MALE, FEMALE, OTHER;

        private final String i18nKey;

        Gender() {
            this.i18nKey = "customer.gender." + name().toLowerCase(Locale.ROOT);
        }

        public String i18nKey() {
            return i18nKey;
        }
    }

    //Customer's special needs.
    enum SpecialNeed {
        VISUAL_IMPAIRMENT, HEARING_IMPAIRMENT, WHEELCHAIR, SERVICE_ANIMAL, OTHER;

        private final String i18nKey;

        SpecialNeed() {
            this.i18nKey = "customer.special_need." + name().toLowerCase(Locale.ROOT);
        }

        public String i18nKey() {
            return i18nKey;
        }
    }

    //Customer's travel document
    record TravelDocument(Country country, Type type, LocalDate issuesOn, LocalDate expiresOn, String id) {
        public enum Type {
            PASSPORT, ID_CARD, DRIVING_LICENSE, RESIDENT_CARD, RESIDENCE_DOCUMENT, TOURIST_VISA, BUSINESS_VISA, GOVERNMENT_VISA, REFUGEE_VISA, OTHER;

            private final String i18nKey;

            Type() {
                this.i18nKey = "customer.travel_document.type." + name().toLowerCase(Locale.ROOT);
            }

            public String i18nKey() {
                return i18nKey;
            }
        }

        public static Builder creator(Country country, Type type) {
            return new Builder().country(country).type(type);
        }

        public static final class Builder implements peruncs.utilities.Creator<Builder, TravelDocument> {
            public Builder() {}

            private Country country;
            private Type type;
            private LocalDate issuesOn;
            private LocalDate expiresOn;
            private String id;

            public Builder country(Country country) { this.country = country; return this; }
            public Builder type(Type type) { this.type = type; return this; }
            public Builder issuesOn(LocalDate issuesOn) { this.issuesOn = issuesOn; return this; }
            public Builder expiresOn(LocalDate expiresOn) { this.expiresOn = expiresOn; return this; }
            public Builder id(String id) { this.id = id; return this; }

            @Override
            public TravelDocument create() {
                return new TravelDocument(country, type, issuesOn, expiresOn, id);
            }
        }
    }

    //Customer's contact person
    record ContactPerson(String name, Relation relation, Set<ElectronicAddress> electronicAddresses) {
        public enum Relation {
            SPOUSE, CHILD, PARENT, OTHER;

            private final String i18nKey;

            Relation() {
                this.i18nKey = "customer.relation." + name().toLowerCase(Locale.ROOT);
            }

            public String i18nKey() {
                return i18nKey;
            }
        }

        public ContactPerson {
            electronicAddresses = Set.copyOf(electronicAddresses);
        }
    }

    /// Data store and GigaMap management for [Customer] entities.
    class Store extends VersionedBaseEntity.Store<Customer> {

        private static final NullableBinaryIndexerTsid<Customer> USER_ID_INDEXER = new NullableBinaryIndexerTsid<>() {
            @Override
            protected Tsid getTsid(Customer entity) {
                return WithId.extractId(entity::userId);
            }
        };

        private static final Indexer<Customer, Gender> GENDER_INDEXER = new Indexer.Abstract<>() {
            @Override
            public Class<Gender> keyType() {
                return Gender.class;
            }

            @Override
            public Gender index(Customer entity) {
                return entity.gender();
            }
        };

        private static final IndexerCountry.MultiValue.Iso3Code<Customer> CITIZENSHIP_COUNTRY_INDEXER = new IndexerCountry.MultiValue.Iso3Code<>() {
            @Override
            public Iterable<Country> getCountries(Customer entity) {
                return entity.countryOfCitizenship();
            }
        };

        private static final IndexerCountry.MultiValue.Iso3Code<Customer> RESIDENCE_COUNTRY_INDEXER = new IndexerCountry.MultiValue.Iso3Code<>() {
            @Override
            public Iterable<Country> getCountries(Customer entity) {
                return entity.countryOfResidence();
            }
        };

        private static final IndexerMultiValue<Customer, SpecialNeed> SPECIAL_NEEDS_INDEXER = new IndexerMultiValue.Abstract<>() {
            @Override
            public Class<SpecialNeed> keyType() {
                return SpecialNeed.class;
            }

            @Override
            public Iterable<SpecialNeed> indexEntityMultiValue(Customer entity) {
                return entity.specialNeeds();
            }
        };

        public Store(EclipseStoreContext ctx) {
            super(ctx, "customers");
            var index = gigaMap.index().bitmap();
            index.ensure(USER_ID_INDEXER);
            index.ensure(GENDER_INDEXER);
            index.ensure(CITIZENSHIP_COUNTRY_INDEXER);
            index.ensure(RESIDENCE_COUNTRY_INDEXER);
            index.ensure(SPECIAL_NEEDS_INDEXER);
        }

        @Override
        public void populate(Document document, Customer entity) {
            super.populate(document, entity);
        }

        @Override
        public QueryBuilder query() {
            return new QueryBuilder();
        }

        public class QueryBuilder extends StoreQueryBuilder<QueryBuilder> {
            private QueryBuilder() {
                super();
            }

            @Override
            protected QueryBuilder self() {
                return this;
            }

            public QueryBuilder userId(Tsid... userIds) {
                return where(USER_ID_INDEXER, userIds);
            }

            public QueryBuilder user(Actor... users) {
                return where(USER_ID_INDEXER, Actor::id, Tsid[]::new, users);
            }

            public QueryBuilder gender(Gender... genders) {
                return where(GENDER_INDEXER, genders);
            }

            public QueryBuilder citizenshipCountryISO3(String... countryCodes) {
                return where(CITIZENSHIP_COUNTRY_INDEXER, countryCodes);
            }

            public QueryBuilder residenceCountryISO3(String... countryCodes) {
                return where(RESIDENCE_COUNTRY_INDEXER, countryCodes);
            }

            public QueryBuilder specialNeeds(SpecialNeed... needs) {
                return where(SPECIAL_NEEDS_INDEXER, needs);
            }
        }
    }
}

[hrstoyanov]

VersionedBaseEntity.java