diff --git a/cdm/pom.xml b/cdm/pom.xml index 6f9048ceb..0d01e4d1c 100644 --- a/cdm/pom.xml +++ b/cdm/pom.xml @@ -11,7 +11,7 @@ .. - 5.10.0-SNAPSHOT + 5.10.0 edal-cdm jar diff --git a/cdm/src/main/java/uk/ac/rdg/resc/edal/dataset/vtk/OnDemandVtkDataSource.java b/cdm/src/main/java/uk/ac/rdg/resc/edal/dataset/vtk/OnDemandVtkDataSource.java index 7a2a18d7a..52a904f17 100644 --- a/cdm/src/main/java/uk/ac/rdg/resc/edal/dataset/vtk/OnDemandVtkDataSource.java +++ b/cdm/src/main/java/uk/ac/rdg/resc/edal/dataset/vtk/OnDemandVtkDataSource.java @@ -42,13 +42,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.sf.ehcache.Cache; -import net.sf.ehcache.Element; -import net.sf.ehcache.config.CacheConfiguration; -import net.sf.ehcache.config.PersistenceConfiguration; -import net.sf.ehcache.config.CacheConfiguration.TransactionalMode; -import net.sf.ehcache.config.PersistenceConfiguration.Strategy; -import net.sf.ehcache.store.MemoryStoreEvictionPolicy; +import org.ehcache.Cache; +import org.ehcache.config.builders.CacheConfigurationBuilder; +import org.ehcache.config.builders.ResourcePoolsBuilder; import uk.ac.rdg.resc.edal.cache.EdalCache; import uk.ac.rdg.resc.edal.dataset.DataSource; import uk.ac.rdg.resc.edal.dataset.vtk.HydromodelVtkDatasetFactory.TimestepInfo; @@ -68,9 +64,10 @@ protected Number[] getData1D(TimestepInfo timestepInfo, String variableId) throw */ Number[] data1d; DataCacheKey key = new DataCacheKey(timestepInfo.file, variableId); - if (vtkGridDatasetCache.isKeyInCache(key)) { + Number[] cached = vtkGridDatasetCache.get(key); + if (cached != null) { log.debug("Getting timestep data from cache"); - data1d = (Number[]) vtkGridDatasetCache.get(key).getObjectValue(); + data1d = cached; } else { log.debug("Data not in cache, reading from VTK file: " + timestepInfo.file.getAbsolutePath()); @@ -138,7 +135,7 @@ protected Number[] getData1D(TimestepInfo timestepInfo, String variableId) throw if (dataStr != null) { data1d = VtkUtils.parseDataString(dataStr, dataFormat, dataType, timestepInfo.fillValues); - vtkGridDatasetCache.put(new Element(key, data1d)); + vtkGridDatasetCache.put(key, data1d); } else { throw new DataReadingException("No data for variable " + variableId + " found in file: " + timestepInfo.file); @@ -164,27 +161,19 @@ public void close() throws DataReadingException { */ private static final String CACHE_NAME = "vtkDataCache"; private static final int MAX_HEAP_ENTRIES = 50; - private static final MemoryStoreEvictionPolicy EVICTION_POLICY = MemoryStoreEvictionPolicy.LFU; - private static final Strategy PERSISTENCE_STRATEGY = Strategy.NONE; - private static final TransactionalMode TRANSACTIONAL_MODE = TransactionalMode.OFF; - private static Cache vtkGridDatasetCache = null; + private static final Cache vtkGridDatasetCache; static { - if (EdalCache.cacheManager.cacheExists(CACHE_NAME) == false) { - /* - * Configure cache - */ - log.debug( - "Creating vtkDataCache, with maximum " + MAX_HEAP_ENTRIES + " entries"); - CacheConfiguration config = new CacheConfiguration(CACHE_NAME, MAX_HEAP_ENTRIES) - .eternal(true).memoryStoreEvictionPolicy(EVICTION_POLICY) - .persistence(new PersistenceConfiguration().strategy(PERSISTENCE_STRATEGY)) - .transactionalMode(TRANSACTIONAL_MODE); - vtkGridDatasetCache = new Cache(config); - EdalCache.cacheManager.addCache(vtkGridDatasetCache); + Cache existing = EdalCache.cacheManager.getCache(CACHE_NAME, + DataCacheKey.class, Number[].class); + if (existing == null) { + log.debug("Creating vtkDataCache, with maximum " + MAX_HEAP_ENTRIES + " entries"); + vtkGridDatasetCache = EdalCache.cacheManager.createCache(CACHE_NAME, + CacheConfigurationBuilder.newCacheConfigurationBuilder(DataCacheKey.class, + Number[].class, ResourcePoolsBuilder.heap(MAX_HEAP_ENTRIES))); } else { log.debug("Loading existing vtkGridDatasetCache"); - vtkGridDatasetCache = EdalCache.cacheManager.getCache(CACHE_NAME); + vtkGridDatasetCache = existing; } } @@ -204,7 +193,7 @@ public int hashCode() { int result = 1; result = prime * result + ((file == null) ? 0 : file.hashCode()); result = prime * result + ((varId == null) ? 0 : varId.hashCode()); - return result; + return EdalCache.murmur3Finalize(result); } @Override diff --git a/common/pom.xml b/common/pom.xml index 27084a197..36f7e6803 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -38,9 +38,16 @@ 2.2.220 - net.sf.ehcache + org.ehcache ehcache - 2.10.6 + 3.12.0 + jakarta + + + org.glassfish.jaxb + jaxb-runtime + + diff --git a/common/src/main/java/uk/ac/rdg/resc/edal/cache/EdalCache.java b/common/src/main/java/uk/ac/rdg/resc/edal/cache/EdalCache.java index 118b723da..8dbf205d0 100644 --- a/common/src/main/java/uk/ac/rdg/resc/edal/cache/EdalCache.java +++ b/common/src/main/java/uk/ac/rdg/resc/edal/cache/EdalCache.java @@ -28,40 +28,58 @@ package uk.ac.rdg.resc.edal.cache; -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.config.Configuration; -import net.sf.ehcache.config.SizeOfPolicyConfiguration; +import org.ehcache.CacheManager; +import org.ehcache.config.builders.CacheManagerBuilder; +/** + * Holds the singleton Ehcache 3.x {@link CacheManager} used by EDAL. + * + *

+ * In Ehcache 2.x the {@code CacheManager} could be configured with a name and a + * global "size of policy" (so the cache could be sized by bytes-on-heap rather + * than entry count). Ehcache 3.x has a different model: caches are typed + * ({@code Cache}), and any optional XML configuration is loaded directly + * into a {@link CacheManager} via + * {@link org.ehcache.xml.XmlConfiguration}. Therefore this class simply + * exposes a singleton {@link CacheManager} which can be augmented at runtime by + * the various EDAL components (Domain2DMapper, HorizontalMesh4dDataset, + * OnDemandVtkDataSource, DataCatalogue, ...). + * + *

+ * The cached objects are typically large (gridded map features with + * 256*256 ~= 65,000 values, or collections of point features containing tens + * of thousands of features). Cache sizing for these objects is therefore + * handled per-cache, in units of MB on heap, rather than by entry count. + */ public class EdalCache { - private static final String CACHE_MANAGER = "EDAL-CacheManager"; - private static final int MAX_CACHE_DEPTH = 4_000_000; - - /* - * We are using an in-memory cache with a configured memory size (as opposed - * to a configured number of items in memory). This has the advantage that - * we will get a hard limit on the amount of memory the cache consumes. The - * disadvantage is that the size of each object needs to be calculated prior - * to inserting it into the cache. - * - * The maxDepth property specified the maximum number of object references - * to count before a warning is given. - * - * Now, we are generally caching 2 things: - * - * 1) Gridded map features which will generally have 256*256 ~= 65,000 - * values, but could easily be bigger - * - * 2) Collections of point features. A year's worth of EN3 data could - * typically contain >15,000 features, each with a number of properties - * - * These can need to count a very large number of object references. - * However, this calculation is actually pretty quick. Setting the max depth - * to 4,000,000 seems to suppress the vast majority of warnings, and doesn't - * impact performance noticeably. - * - * Cache configuration specified in resources/ehcache.xml + /** + * The shared, application-wide Ehcache 3 {@link CacheManager}. Caches are + * registered against this manager by the various EDAL modules. It is built + * (and initialised) eagerly so that callers can rely on it being usable + * immediately. */ - public static final CacheManager cacheManager = CacheManager - .newInstance(new Configuration().name(CACHE_MANAGER) - .sizeOfPolicy(new SizeOfPolicyConfiguration().maxDepth(MAX_CACHE_DEPTH))); + public static final CacheManager cacheManager = CacheManagerBuilder.newCacheManagerBuilder() + .build(true); + + /** + * The finalization step from MurmurHash3, used to scramble an integer hash + * so that the resulting bits are well distributed. + * + *

+ * This is the same finalizer used by Spring's {@code SimpleKey} (see + * SimpleKey.java). + * See + * issue #171, comment 2708650091 for the discussion that motivated this. + * + * @param hash the input combined hash code + * @return a strongly mixed hash code derived from {@code hash} + */ + public static int murmur3Finalize(int hash) { + hash ^= (hash >>> 16); + hash *= 0x85ebca6b; + hash ^= (hash >>> 13); + hash *= 0xc2b2ae35; + hash ^= (hash >>> 16); + return hash; + } } diff --git a/common/src/main/java/uk/ac/rdg/resc/edal/dataset/Domain2DMapper.java b/common/src/main/java/uk/ac/rdg/resc/edal/dataset/Domain2DMapper.java index 413218f65..e7fc79da7 100644 --- a/common/src/main/java/uk/ac/rdg/resc/edal/dataset/Domain2DMapper.java +++ b/common/src/main/java/uk/ac/rdg/resc/edal/dataset/Domain2DMapper.java @@ -30,13 +30,9 @@ import java.util.List; -import net.sf.ehcache.Cache; -import net.sf.ehcache.Element; -import net.sf.ehcache.config.CacheConfiguration; -import net.sf.ehcache.config.CacheConfiguration.TransactionalMode; -import net.sf.ehcache.config.PersistenceConfiguration; -import net.sf.ehcache.config.PersistenceConfiguration.Strategy; -import net.sf.ehcache.store.MemoryStoreEvictionPolicy; +import org.ehcache.Cache; +import org.ehcache.config.builders.CacheConfigurationBuilder; +import org.ehcache.config.builders.ResourcePoolsBuilder; import uk.ac.rdg.resc.edal.cache.EdalCache; import uk.ac.rdg.resc.edal.grid.GridCell2D; import uk.ac.rdg.resc.edal.grid.HorizontalGrid; @@ -114,8 +110,9 @@ public int getTargetYSize() { */ public static Domain2DMapper forGrid(HorizontalGrid sourceGrid, final HorizontalGrid targetGrid) { Domain2DMapperCacheKey key = new Domain2DMapperCacheKey(sourceGrid, targetGrid); - if (domainMapperCache.isKeyInCache(key)) { - return (Domain2DMapper) domainMapperCache.get(key).getObjectValue(); + Domain2DMapper cached = domainMapperCache.get(key); + if (cached != null) { + return cached; } Domain2DMapper ret; if (sourceGrid instanceof RectilinearGrid @@ -140,7 +137,7 @@ public static Domain2DMapper forGrid(HorizontalGrid sourceGrid, final Horizontal */ ret = forGeneralGrids(sourceGrid, targetGrid); } - domainMapperCache.put(new Element(key, ret)); + domainMapperCache.put(key, ret); return ret; } @@ -225,27 +222,20 @@ private static Domain2DMapper forGeneralGrids(HorizontalGrid sourceGrid, */ private static final String CACHE_NAME = "domainMapperCache"; private static final int MAX_HEAP_ENTRIES = 100; - private static final MemoryStoreEvictionPolicy EVICTION_POLICY = MemoryStoreEvictionPolicy.LFU; - private static final Strategy PERSISTENCE_STRATEGY = Strategy.NONE; - private static final TransactionalMode TRANSACTIONAL_MODE = TransactionalMode.OFF; - private static Cache domainMapperCache; + private static final Cache domainMapperCache; static { - if (EdalCache.cacheManager.cacheExists(CACHE_NAME) == false) { - /* - * Configure cache - */ - log.debug("Creating domainMapperCache, with maximum "+MAX_HEAP_ENTRIES+" entries"); - CacheConfiguration config = new CacheConfiguration(CACHE_NAME, MAX_HEAP_ENTRIES) - .eternal(true) - .memoryStoreEvictionPolicy(EVICTION_POLICY) - .persistence(new PersistenceConfiguration().strategy(PERSISTENCE_STRATEGY)) - .transactionalMode(TRANSACTIONAL_MODE); - domainMapperCache = new Cache(config); - EdalCache.cacheManager.addCache(domainMapperCache); + Cache existing = EdalCache.cacheManager + .getCache(CACHE_NAME, Domain2DMapperCacheKey.class, Domain2DMapper.class); + if (existing == null) { + log.debug("Creating domainMapperCache, with maximum " + MAX_HEAP_ENTRIES + " entries"); + domainMapperCache = EdalCache.cacheManager.createCache(CACHE_NAME, + CacheConfigurationBuilder.newCacheConfigurationBuilder( + Domain2DMapperCacheKey.class, Domain2DMapper.class, + ResourcePoolsBuilder.heap(MAX_HEAP_ENTRIES))); } else { log.debug("Loading existing domainMapperCache"); - domainMapperCache = EdalCache.cacheManager.getCache(CACHE_NAME); + domainMapperCache = existing; } } @@ -265,7 +255,7 @@ public int hashCode() { int result = 1; result = prime * result + ((source == null) ? 0 : source.hashCode()); result = prime * result + ((target == null) ? 0 : target.hashCode()); - return result; + return EdalCache.murmur3Finalize(result); } @Override diff --git a/common/src/main/java/uk/ac/rdg/resc/edal/dataset/HorizontalMesh4dDataset.java b/common/src/main/java/uk/ac/rdg/resc/edal/dataset/HorizontalMesh4dDataset.java index 6531ff663..2ecea25ca 100755 --- a/common/src/main/java/uk/ac/rdg/resc/edal/dataset/HorizontalMesh4dDataset.java +++ b/common/src/main/java/uk/ac/rdg/resc/edal/dataset/HorizontalMesh4dDataset.java @@ -37,13 +37,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.sf.ehcache.Cache; -import net.sf.ehcache.Element; -import net.sf.ehcache.config.CacheConfiguration; -import net.sf.ehcache.config.CacheConfiguration.TransactionalMode; -import net.sf.ehcache.config.PersistenceConfiguration; -import net.sf.ehcache.config.PersistenceConfiguration.Strategy; -import net.sf.ehcache.store.MemoryStoreEvictionPolicy; +import org.ehcache.Cache; +import org.ehcache.config.builders.CacheConfigurationBuilder; +import org.ehcache.config.builders.ResourcePoolsBuilder; import uk.ac.rdg.resc.edal.cache.EdalCache; import uk.ac.rdg.resc.edal.dataset.HZTDataSource.MeshCoordinates3D; import uk.ac.rdg.resc.edal.exceptions.DataReadingException; @@ -115,9 +111,9 @@ protected Array2D extractHorizontalData(HorizontalMesh4dVariableMetadata MeshDatasetCacheElement meshDatasetCacheElement; MeshCacheKey key = new MeshCacheKey(targetGrid, grid); - if (meshDatasetCache.isKeyInCache(key)) { - meshDatasetCacheElement = (MeshDatasetCacheElement) meshDatasetCache.get(key) - .getObjectValue(); + MeshDatasetCacheElement cached = meshDatasetCache.get(key); + if (cached != null) { + meshDatasetCacheElement = cached; outputCoords = meshDatasetCacheElement.getOutputCoords(); coordsToRead = meshDatasetCacheElement.getCoordsToRead(); } else { @@ -132,7 +128,7 @@ protected Array2D extractHorizontalData(HorizontalMesh4dVariableMetadata coordsToRead.add(meshCoords); } meshDatasetCacheElement = new MeshDatasetCacheElement(outputCoords, coordsToRead); - meshDatasetCache.put(new Element(key, meshDatasetCacheElement)); + meshDatasetCache.put(key, meshDatasetCacheElement); } /* @@ -230,10 +226,7 @@ protected Number extractPoint(HorizontalMesh4dVariableMetadata metadata, int t, */ private static final String CACHE_NAME = "meshDatasetCache"; private static final int MAX_HEAP_ENTRIES = 50; - private static final MemoryStoreEvictionPolicy EVICTION_POLICY = MemoryStoreEvictionPolicy.LFU; - private static final Strategy PERSISTENCE_STRATEGY = Strategy.NONE; - private static final TransactionalMode TRANSACTIONAL_MODE = TransactionalMode.OFF; - private static Cache meshDatasetCache = null; + private static final Cache meshDatasetCache; private static class MeshCacheKey { private HorizontalGrid target; @@ -251,7 +244,7 @@ public int hashCode() { int result = 1; result = prime * result + ((source == null) ? 0 : source.hashCode()); result = prime * result + ((target == null) ? 0 : target.hashCode()); - return result; + return EdalCache.murmur3Finalize(result); } @Override @@ -278,20 +271,17 @@ public boolean equals(Object obj) { } static { - if (EdalCache.cacheManager.cacheExists(CACHE_NAME) == false) { - /* - * Configure cache - */ + Cache existing = EdalCache.cacheManager + .getCache(CACHE_NAME, MeshCacheKey.class, MeshDatasetCacheElement.class); + if (existing == null) { log.debug("Creating meshDatasetCache, with maximum " + MAX_HEAP_ENTRIES + " entries"); - CacheConfiguration config = new CacheConfiguration(CACHE_NAME, MAX_HEAP_ENTRIES) - .eternal(true).memoryStoreEvictionPolicy(EVICTION_POLICY) - .persistence(new PersistenceConfiguration().strategy(PERSISTENCE_STRATEGY)) - .transactionalMode(TRANSACTIONAL_MODE); - meshDatasetCache = new Cache(config); - EdalCache.cacheManager.addCache(meshDatasetCache); + meshDatasetCache = EdalCache.cacheManager.createCache(CACHE_NAME, + CacheConfigurationBuilder.newCacheConfigurationBuilder(MeshCacheKey.class, + MeshDatasetCacheElement.class, + ResourcePoolsBuilder.heap(MAX_HEAP_ENTRIES))); } else { log.debug("Loading existing meshDatasetCache"); - meshDatasetCache = EdalCache.cacheManager.getCache(CACHE_NAME); + meshDatasetCache = existing; } } } diff --git a/graphics/pom.xml b/graphics/pom.xml index 2f2c3e9ef..c047f9d10 100644 --- a/graphics/pom.xml +++ b/graphics/pom.xml @@ -89,7 +89,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.1 + 3.6.2 package diff --git a/pom.xml b/pom.xml index ad9243013..983ee6736 100644 --- a/pom.xml +++ b/pom.xml @@ -56,8 +56,8 @@ org.apache.maven.plugins maven-compiler-plugin - 11 - 11 + 17 + 17 3.8.1 diff --git a/wms/src/main/java/uk/ac/rdg/resc/edal/wms/WmsContextListener.java b/wms/src/main/java/uk/ac/rdg/resc/edal/wms/WmsContextListener.java index 15cc18faf..33f61dd5b 100755 --- a/wms/src/main/java/uk/ac/rdg/resc/edal/wms/WmsContextListener.java +++ b/wms/src/main/java/uk/ac/rdg/resc/edal/wms/WmsContextListener.java @@ -54,7 +54,7 @@ public void contextDestroyed(ServletContextEvent sce) { /* * Shut down all cache threads */ - EdalCache.cacheManager.shutdown(); + EdalCache.cacheManager.close(); /* * TODO Close any DB connections diff --git a/xml-catalogue/src/main/java/uk/ac/rdg/resc/edal/catalogue/DataCatalogue.java b/xml-catalogue/src/main/java/uk/ac/rdg/resc/edal/catalogue/DataCatalogue.java index e230bae97..3fa1f08d2 100755 --- a/xml-catalogue/src/main/java/uk/ac/rdg/resc/edal/catalogue/DataCatalogue.java +++ b/xml-catalogue/src/main/java/uk/ac/rdg/resc/edal/catalogue/DataCatalogue.java @@ -28,9 +28,11 @@ package uk.ac.rdg.resc.edal.catalogue; +import java.io.File; import java.io.IOException; import java.io.Serializable; -import java.lang.management.ManagementFactory; +import java.net.MalformedURLException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -40,24 +42,22 @@ import java.util.List; import java.util.Map; -import javax.management.MBeanServer; -import javax.management.MalformedObjectNameException; -import javax.management.ObjectName; - +import org.ehcache.Cache; +import org.ehcache.CacheManager; +import org.ehcache.config.CacheConfiguration; +import org.ehcache.config.ResourceType; +import org.ehcache.config.SizedResourcePool; +import org.ehcache.config.builders.CacheConfigurationBuilder; +import org.ehcache.config.builders.CacheManagerBuilder; +import org.ehcache.config.builders.ExpiryPolicyBuilder; +import org.ehcache.config.builders.ResourcePoolsBuilder; +import org.ehcache.config.units.MemoryUnit; +import org.ehcache.expiry.ExpiryPolicy; +import org.ehcache.xml.XmlConfiguration; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; -import net.sf.ehcache.config.CacheConfiguration; -import net.sf.ehcache.config.CacheConfiguration.TransactionalMode; -import net.sf.ehcache.config.MemoryUnit; -import net.sf.ehcache.config.PersistenceConfiguration; -import net.sf.ehcache.config.PersistenceConfiguration.Strategy; -import net.sf.ehcache.management.ManagementService; -import net.sf.ehcache.store.MemoryStoreEvictionPolicy; import uk.ac.rdg.resc.edal.cache.EdalCache; import uk.ac.rdg.resc.edal.catalogue.jaxb.CacheInfo; import uk.ac.rdg.resc.edal.catalogue.jaxb.CatalogueConfig; @@ -94,14 +94,10 @@ public class DataCatalogue implements DatasetCatalogue, DatasetStorage, FeatureC private static final String CACHE_NAME = "featureCache"; private static final long CACHE_SIZE_MB = 512; private static final int LIFETIME_SECONDS = 0; - final MemoryStoreEvictionPolicy EVICTION_POLICY = MemoryStoreEvictionPolicy.LFU; - private static final Strategy PERSISTENCE_STRATEGY = Strategy.NONE; - private static final TransactionalMode TRANSACTIONAL_MODE = TransactionalMode.OFF; private boolean cachingEnabled; - private Cache featureCache = null; - private static MBeanServer mBeanServer; - private static ObjectName cacheManagerObjectName; + @SuppressWarnings("rawtypes") + private Cache featureCache = null; protected final CatalogueConfig config; protected Map datasets; @@ -139,36 +135,48 @@ public DataCatalogue(CatalogueConfig config, LayerNameMapper layerNameMapper) if (ehcache_file != null && !ehcache_file.isEmpty()) { /* * We want to load the caches from the XML file into the EDAL cache - * manager + * manager. Ehcache 3 loads XML configurations through + * XmlConfiguration; we transfer each cache definition into the + * shared EDAL CacheManager so the rest of EDAL sees them. */ log.debug("Loading cache definitions from file"); - CacheManager cacheManager = CacheManager.newInstance(System.getProperty(WMS_CACHE_CONFIG)); - for (String cacheName : cacheManager.getCacheNames()) { - if(EdalCache.cacheManager.cacheExists(cacheName)) { - /* - * Remove any existing cache - */ - EdalCache.cacheManager.removeCache(cacheName); + try { + XmlConfiguration xmlConfig = new XmlConfiguration( + new File(ehcache_file).toURI().toURL()); + for (String cacheName : xmlConfig.getCacheConfigurations().keySet()) { + if (EdalCache.cacheManager.getRuntimeConfiguration() + .getCacheConfigurations().containsKey(cacheName)) { + /* + * Remove any existing cache + */ + EdalCache.cacheManager.removeCache(cacheName); + } + EdalCache.cacheManager.createCache(cacheName, + xmlConfig.getCacheConfigurations().get(cacheName)); } - EdalCache.cacheManager.addCache(new Cache(cacheManager.getCache(cacheName).getCacheConfiguration())); + } catch (MalformedURLException e) { + throw new EdalException("Invalid ehcache.config path: " + ehcache_file, e); } - cacheManager.shutdown(); } if (cachingEnabled) { - if (EdalCache.cacheManager.cacheExists(CACHE_NAME)) { + @SuppressWarnings("rawtypes") + Cache existing = EdalCache.cacheManager.getCache(CACHE_NAME, + CacheKey.class, Collection.class); + if (existing != null) { /* * Use parameters for featureCache from ehcache.xml config file - * if passed in as JVM parameter wmsCache.config - Update cache - * params in NwcmsConfig + * if passed in as JVM parameter ehcache.config - Update cache + * params in CatalogueConfig */ - featureCache = EdalCache.cacheManager.getCache(CACHE_NAME); + featureCache = existing; CacheInfo catalogueCacheInfo = config.getCacheSettings(); - CacheConfiguration featureCacheConfiguration = featureCache.getCacheConfiguration(); - catalogueCacheInfo.setInMemorySizeMB( - (int) (featureCacheConfiguration.getMaxBytesLocalHeap() / (1024 * 1024))); - catalogueCacheInfo.setElementLifetimeMinutes( - featureCacheConfiguration.getTimeToLiveSeconds() / 60); + CacheConfiguration featureCacheConfiguration = EdalCache.cacheManager + .getRuntimeConfiguration().getCacheConfigurations().get(CACHE_NAME); + catalogueCacheInfo + .setInMemorySizeMB((int) getHeapSizeMB(featureCacheConfiguration)); + catalogueCacheInfo + .setElementLifetimeMinutes(getTtlSeconds(featureCacheConfiguration) / 60f); catalogueCacheInfo.setEnabled(true); } else { /* @@ -176,34 +184,60 @@ public DataCatalogue(CatalogueConfig config, LayerNameMapper layerNameMapper) * define "featureCache". In this case, configure with values * from config.xml */ - CacheConfiguration cacheConfig = new CacheConfiguration(CACHE_NAME, 0) - .eternal(cacheLifetimeSeconds == 0).timeToLiveSeconds(cacheLifetimeSeconds) - .maxBytesLocalHeap(config.getCacheSettings().getInMemorySizeMB(), - MemoryUnit.MEGABYTES) - .memoryStoreEvictionPolicy(EVICTION_POLICY) - .persistence(new PersistenceConfiguration().strategy(PERSISTENCE_STRATEGY)) - .transactionalMode(TRANSACTIONAL_MODE); - - featureCache = new Cache(cacheConfig); - EdalCache.cacheManager.addCache(featureCache); + featureCache = createFeatureCache(config.getCacheSettings().getInMemorySizeMB(), + cacheLifetimeSeconds); } + } + } - /* - * Used to gather statistics about Ehcache - */ - mBeanServer = ManagementFactory.getPlatformMBeanServer(); - try { - cacheManagerObjectName = new ObjectName("net.sf.ehcache:type=CacheManager,name=" - + EdalCache.cacheManager.getName()); - } catch (MalformedObjectNameException e) { - throw new EdalException("unable to form cacheManager ObjectName", e); - } + /** + * Creates the feature cache in the shared {@link EdalCache#cacheManager} + * with the specified heap size (MB) and time-to-live (seconds; 0 means + * never expire). + */ + @SuppressWarnings("rawtypes") + private static Cache createFeatureCache(long sizeMB, + long lifetimeSeconds) { + CacheConfigurationBuilder builder = CacheConfigurationBuilder + .newCacheConfigurationBuilder(CacheKey.class, Collection.class, + ResourcePoolsBuilder.newResourcePoolsBuilder().heap(sizeMB, MemoryUnit.MB)); + if (lifetimeSeconds <= 0) { + builder = builder.withExpiry(ExpiryPolicyBuilder.noExpiration()); + } else { + builder = builder.withExpiry(ExpiryPolicyBuilder + .timeToLiveExpiration(Duration.ofSeconds(lifetimeSeconds))); + } + return EdalCache.cacheManager.createCache(CACHE_NAME, builder); + } - if (!mBeanServer.isRegistered(cacheManagerObjectName)) { - ManagementService.registerMBeans(EdalCache.cacheManager, mBeanServer, true, true, - true, true); - } + private static long getHeapSizeMB(CacheConfiguration cacheConfiguration) { + if (cacheConfiguration == null) { + return 0; + } + SizedResourcePool heap = cacheConfiguration.getResourcePools() + .getPoolForResource(ResourceType.Core.HEAP); + if (heap == null) { + return 0; + } + org.ehcache.config.units.MemoryUnit unit = (org.ehcache.config.units.MemoryUnit) heap + .getUnit(); + return unit.toBytes(heap.getSize()) / (1024 * 1024); + } + + private static long getTtlSeconds(CacheConfiguration cacheConfiguration) { + if (cacheConfiguration == null) { + return 0; } + @SuppressWarnings({ "unchecked", "rawtypes" }) + ExpiryPolicy expiry = (ExpiryPolicy) cacheConfiguration.getExpiryPolicy(); + if (expiry == null) { + return 0; + } + Duration ttl = expiry.getExpiryForCreation(null, null); + if (ttl == null || ExpiryPolicy.INFINITE.equals(ttl)) { + return 0; + } + return ttl.getSeconds(); } public CatalogueConfig getConfig() { @@ -222,19 +256,15 @@ public void shutdown() { * null */ public void setCache(CacheInfo cacheConfig) { - MemoryStoreEvictionPolicy memoryStoreEviction; - Strategy persistenceStrategy; - TransactionalMode transactionalMode; - long cacheSizeMB; long configCacheSizeMB = cacheConfig.getInMemorySizeMB(); - long lifetimeSeconds; long configLifetimeSeconds = (long) (cacheConfig.getElementLifetimeMinutes() * 60); + CacheConfiguration currentConfig = EdalCache.cacheManager.getRuntimeConfiguration() + .getCacheConfigurations().get(CACHE_NAME); if (featureCache != null && cachingEnabled == cacheConfig.isEnabled() - && configCacheSizeMB == featureCache.getCacheConfiguration().getMaxBytesLocalHeap() - / (1024 * 1024) - && configLifetimeSeconds == featureCache.getCacheConfiguration() - .getTimeToLiveSeconds()) { + && currentConfig != null + && configCacheSizeMB == getHeapSizeMB(currentConfig) + && configLifetimeSeconds == getTtlSeconds(currentConfig)) { /* * We are not changing anything about the cache. */ @@ -244,77 +274,54 @@ public void setCache(CacheInfo cacheConfig) { cachingEnabled = cacheConfig.isEnabled(); if (cachingEnabled) { - if (EdalCache.cacheManager.cacheExists(CACHE_NAME)) { - /* - * Update cache configuration - */ - CacheConfiguration featureCacheConfig = featureCache.getCacheConfiguration(); - featureCacheConfig.setTimeToLiveSeconds(configLifetimeSeconds); - featureCacheConfig.setMaxBytesLocalHeap(configCacheSizeMB * 1024 * 1024); - } else { - /*- - * Precedence: - * - Admin config - * - XML file "ehcache.config" - * - Default values - */ + /*- + * Ehcache 3 cache configurations are immutable, so any + * resize/TTL change is implemented by removing and recreating + * the cache. + * + * Precedence: + * - Admin config + * - XML file "ehcache.config" + * - Default values + */ - /* - * Default values - */ - cacheSizeMB = CACHE_SIZE_MB; - lifetimeSeconds = LIFETIME_SECONDS; - memoryStoreEviction = EVICTION_POLICY; - persistenceStrategy = PERSISTENCE_STRATEGY; - transactionalMode = TRANSACTIONAL_MODE; + /* + * Default values + */ + long cacheSizeMB = CACHE_SIZE_MB; + long lifetimeSeconds = LIFETIME_SECONDS; - /* - * XML config - */ - String ehcache_file = System.getProperty("ehcache.config"); - if (ehcache_file != null && !ehcache_file.isEmpty()) { - Cache tmpfeatureCache = EdalCache.cacheManager.getCache(CACHE_NAME); - cacheSizeMB = tmpfeatureCache.getCacheConfiguration().getMaxBytesLocalHeap() - / (1024 * 1024); - lifetimeSeconds = tmpfeatureCache.getCacheConfiguration() - .getTimeToLiveSeconds(); - memoryStoreEviction = tmpfeatureCache.getCacheConfiguration() - .getMemoryStoreEvictionPolicy(); - persistenceStrategy = tmpfeatureCache.getCacheConfiguration() - .getPersistenceConfiguration().getStrategy(); - transactionalMode = tmpfeatureCache.getCacheConfiguration() - .getTransactionalMode(); - } + /* + * XML config: pull values from the existing (XML-loaded) + * featureCache, if it is currently registered. + */ + if (currentConfig != null) { + cacheSizeMB = getHeapSizeMB(currentConfig); + lifetimeSeconds = getTtlSeconds(currentConfig); + } - /* - * Admin - */ - if (cacheConfig.getInMemorySizeMB() != 0) { - cacheSizeMB = configCacheSizeMB; - } - if (cacheConfig.getElementLifetimeMinutes() != 0) { - lifetimeSeconds = configLifetimeSeconds; - } + /* + * Admin + */ + if (cacheConfig.getInMemorySizeMB() != 0) { + cacheSizeMB = configCacheSizeMB; + } + if (cacheConfig.getElementLifetimeMinutes() != 0) { + lifetimeSeconds = configLifetimeSeconds; + } - /* - * Configure and create cache - */ - CacheConfiguration config = new CacheConfiguration(CACHE_NAME, 0) - .eternal(lifetimeSeconds == 0) - .maxBytesLocalHeap(cacheSizeMB, MemoryUnit.MEGABYTES) - .timeToLiveSeconds(lifetimeSeconds) - .memoryStoreEvictionPolicy(memoryStoreEviction) - .persistence(new PersistenceConfiguration().strategy(persistenceStrategy)) - .transactionalMode(transactionalMode); - - featureCache = new Cache(config); - EdalCache.cacheManager.addCache(featureCache); + if (currentConfig != null) { + EdalCache.cacheManager.removeCache(CACHE_NAME); } + featureCache = createFeatureCache(cacheSizeMB, lifetimeSeconds); } else { /* * Remove existing cache to free up memory */ - EdalCache.cacheManager.removeCache(CACHE_NAME); + if (currentConfig != null) { + EdalCache.cacheManager.removeCache(CACHE_NAME); + } + featureCache = null; } } @@ -351,16 +358,23 @@ public synchronized void datasetLoaded(Dataset dataset, Collection>; collect keys to + * a temporary list before removing to avoid concurrent modification. + */ + List toRemove = new ArrayList<>(); + for (@SuppressWarnings("rawtypes") + Cache.Entry entry : featureCache) { + CacheKey cacheKey = entry.getKey(); String datasetId = layerNameMapper.getDatasetIdFromLayerName(cacheKey.layerName); - if(dataset.getId().equals(datasetId)) { - featureCache.remove(cacheKey); + if (dataset.getId().equals(datasetId)) { + toRemove.add(cacheKey); } } + for (CacheKey cacheKey : toRemove) { + featureCache.remove(cacheKey); + } } /* * If we already have a dataset with this ID, it will be replaced. This @@ -477,20 +491,20 @@ public FeaturesAndMemberName getFeaturesForLayer(String layerName, PlottingDomai throws EdalException { String variable = layerNameMapper.getVariableIdFromLayerName(layerName); Collection> mapFeatures; - if (cachingEnabled) { + if (cachingEnabled && featureCache != null) { CacheKey key = new CacheKey(layerName, params); - Element element = featureCache.get(key); + @SuppressWarnings("rawtypes") + Collection cached = featureCache.get(key); - if (element != null && element.getObjectValue() != null) { + if (cached != null) { /* * This is why we added the SuppressWarnings("unchecked"). */ - mapFeatures = (Collection>) element - .getObjectValue(); + mapFeatures = (Collection>) cached; } else { mapFeatures = doExtraction(layerName, variable, params); try { - featureCache.put(new Element(key, mapFeatures)); + featureCache.put(key, mapFeatures); } catch (Exception e) { log.error("Problem adding features to cache", e); /* @@ -514,7 +528,7 @@ private Dataset getDatasetFromLayerName(String layerName) { return getDatasetFromId(layerNameMapper.getDatasetIdFromLayerName(layerName)); } - private static class CacheKey implements Serializable { + public static class CacheKey implements Serializable { private static final long serialVersionUID = 1L; final String layerName; final PlottingDomainParams params; @@ -531,7 +545,7 @@ public int hashCode() { int result = 1; result = prime * result + ((layerName == null) ? 0 : layerName.hashCode()); result = prime * result + ((params == null) ? 0 : params.hashCode()); - return result; + return EdalCache.murmur3Finalize(result); } @Override