Feat/file

CMakeLists.txt

@@ -74,11 +74,27 @@ if(OPENMP_FOUND AND DSPACES_USE_OPENMP)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
     add_definitions(-DOPS_USE_OPENMP)
 endif()
+
 find_package(CURL)
 if(CURL_FOUND)
     add_definitions(-DDSPACES_HAVE_CURL)
 endif()
 
+find_package(HDF5)
+if(HDF5_FOUND)
+    add_definitions(-DDSPACES_HAVE_HDF5)
+endif()
+
+find_package(NetCDF)
+if(NetCDF_FOUND)
+    add_definitions(-DDSPACES_HAVE_NetCDF)
+endif()
+
+if(HDF5_FOUND OR NetCDF_FOUND)
+    set(ENABLE_FILE_STORAGE ON)
+    add_definitions(-DDSPACES_HAVE_FILE_STORAGE)
+endif()
+
 include(CheckLanguage)
 check_language(Fortran)
 if(CMAKE_Fortran_COMPILER)

cmake/FindNetCDF.cmake

@@ -0,0 +1,133 @@
+# Copied from VTK's FindNetCDF.cmake
+
+#[==[
+Provides the following variables:
+
+  * `NetCDF_FOUND`: Whether NetCDF was found or not.
+  * `NetCDF_INCLUDE_DIRS`: Include directories necessary to use NetCDF.
+  * `NetCDF_LIBRARIES`: Libraries necessary to use NetCDF.
+  * `NetCDF_VERSION`: The version of NetCDF found.
+  * `NetCDF::NetCDF`: A target to use with `target_link_libraries`.
+  * `NetCDF_HAS_PARALLEL`: Whether or not NetCDF was found with parallel IO support.
+#]==]
+
+function(FindNetCDF_get_is_parallel_aware include_dir)
+  file(STRINGS "${include_dir}/netcdf_meta.h" _netcdf_lines
+    REGEX "#define[ \t]+NC_HAS_PARALLEL[ \t]")
+  string(REGEX REPLACE ".*NC_HAS_PARALLEL[ \t]*([0-1]+).*" "\\1" _netcdf_has_parallel "${_netcdf_lines}")
+  if (_netcdf_has_parallel)
+    set(NetCDF_HAS_PARALLEL TRUE PARENT_SCOPE)
+  else()
+    set(NetCDF_HAS_PARALLEL FALSE PARENT_SCOPE)
+  endif()
+endfunction()
+
+# Try to find a CMake-built NetCDF.
+find_package(netCDF CONFIG QUIET)
+if (netCDF_FOUND)
+  # Forward the variables in a consistent way.
+  set(NetCDF_FOUND "${netCDF_FOUND}")
+  set(NetCDF_INCLUDE_DIRS "${netCDF_INCLUDE_DIR}")
+  set(NetCDF_LIBRARIES "${netCDF_LIBRARIES}")
+  set(NetCDF_VERSION "${NetCDFVersion}")
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(NetCDF
+    REQUIRED_VARS NetCDF_INCLUDE_DIRS NetCDF_LIBRARIES
+    VERSION_VAR NetCDF_VERSION)
+
+  if (NOT TARGET NetCDF::NetCDF)
+    add_library(NetCDF::NetCDF INTERFACE IMPORTED)
+    if (TARGET "netCDF::netcdf")
+      # 4.7.3
+      set_target_properties(NetCDF::NetCDF PROPERTIES
+        INTERFACE_LINK_LIBRARIES "netCDF::netcdf")
+    elseif (TARGET "netcdf")
+      set_target_properties(NetCDF::NetCDF PROPERTIES
+        INTERFACE_LINK_LIBRARIES "netcdf")
+    else ()
+      set_target_properties(NetCDF::NetCDF PROPERTIES
+        INTERFACE_LINK_LIBRARIES "${netCDF_LIBRARIES}")
+    endif ()
+  endif ()
+
+  FindNetCDF_get_is_parallel_aware("${NetCDF_INCLUDE_DIRS}")
+  # Skip the rest of the logic in this file.
+  return ()
+endif ()
+
+find_package(PkgConfig QUIET)
+if (PkgConfig_FOUND)
+  pkg_check_modules(_NetCDF QUIET netcdf IMPORTED_TARGET)
+  if (_NetCDF_FOUND)
+    # Forward the variables in a consistent way.
+    set(NetCDF_FOUND "${_NetCDF_FOUND}")
+    set(NetCDF_INCLUDE_DIRS "${_NetCDF_INCLUDE_DIRS}")
+    set(NetCDF_LIBRARIES "${_NetCDF_LIBRARIES}")
+    set(NetCDF_VERSION "${_NetCDF_VERSION}")
+
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(NetCDF
+      REQUIRED_VARS NetCDF_LIBRARIES
+      # This is not required because system-default include paths are not
+      # reported by `FindPkgConfig`, so this might be empty. Assume that if we
+      # have a library, the include directories are fine (if any) since
+      # PkgConfig reported that the package was found.
+      # NetCDF_INCLUDE_DIRS
+      VERSION_VAR NetCDF_VERSION)
+
+    if (NOT TARGET NetCDF::NetCDF)
+      add_library(NetCDF::NetCDF INTERFACE IMPORTED)
+      set_target_properties(NetCDF::NetCDF PROPERTIES
+        INTERFACE_LINK_LIBRARIES "PkgConfig::_NetCDF")
+    endif ()
+
+    FindNetCDF_get_is_parallel_aware("${_NetCDF_INCLUDEDIR}")
+    # Skip the rest of the logic in this file.
+    return ()
+  endif ()
+endif ()
+
+find_path(NetCDF_INCLUDE_DIR
+  NAMES netcdf.h
+  DOC "netcdf include directories")
+mark_as_advanced(NetCDF_INCLUDE_DIR)
+
+find_library(NetCDF_LIBRARY
+  NAMES netcdf
+  DOC "netcdf library")
+mark_as_advanced(NetCDF_LIBRARY)
+
+if (NetCDF_INCLUDE_DIR)
+  file(STRINGS "${NetCDF_INCLUDE_DIR}/netcdf_meta.h" _netcdf_version_lines
+    REGEX "#define[ \t]+NC_VERSION_(MAJOR|MINOR|PATCH|NOTE)")
+  string(REGEX REPLACE ".*NC_VERSION_MAJOR *\([0-9]*\).*" "\\1" _netcdf_version_major "${_netcdf_version_lines}")
+  string(REGEX REPLACE ".*NC_VERSION_MINOR *\([0-9]*\).*" "\\1" _netcdf_version_minor "${_netcdf_version_lines}")
+  string(REGEX REPLACE ".*NC_VERSION_PATCH *\([0-9]*\).*" "\\1" _netcdf_version_patch "${_netcdf_version_lines}")
+  string(REGEX REPLACE ".*NC_VERSION_NOTE *\"\([^\"]*\)\".*" "\\1" _netcdf_version_note "${_netcdf_version_lines}")
+  set(NetCDF_VERSION "${_netcdf_version_major}.${_netcdf_version_minor}.${_netcdf_version_patch}${_netcdf_version_note}")
+  unset(_netcdf_version_major)
+  unset(_netcdf_version_minor)
+  unset(_netcdf_version_patch)
+  unset(_netcdf_version_note)
+  unset(_netcdf_version_lines)
+
+  FindNetCDF_get_is_parallel_aware("${NetCDF_INCLUDE_DIR}")
+endif ()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NetCDF
+  REQUIRED_VARS NetCDF_LIBRARY NetCDF_INCLUDE_DIR
+  VERSION_VAR NetCDF_VERSION)
+
+if (NetCDF_FOUND)
+  set(NetCDF_INCLUDE_DIRS "${NetCDF_INCLUDE_DIR}")
+  set(NetCDF_LIBRARIES "${NetCDF_LIBRARY}")
+
+  if (NOT TARGET NetCDF::NetCDF)
+    add_library(NetCDF::NetCDF UNKNOWN IMPORTED)
+    set_target_properties(NetCDF::NetCDF PROPERTIES
+      IMPORTED_LOCATION "${NetCDF_LIBRARY}"
+      INTERFACE_INCLUDE_DIRECTORIES "${NetCDF_INCLUDE_DIR}")
+  endif ()
+endif ()

include/bbox.h

@@ -42,6 +42,11 @@ int bbox_include(const struct bbox *, const struct bbox *);
 int bbox_does_intersect(const struct bbox *, const struct bbox *);
 void bbox_intersect(const struct bbox *, const struct bbox *, struct bbox *);
 int bbox_equals(const struct bbox *, const struct bbox *);
+int bbox_include_ondim(const struct bbox *b0, const struct bbox *b1, int dim);
+int bbox_can_include(struct bbox* bbox0, struct bbox* bbox1);
+int bbox_can_union(const struct bbox* bbox0, const struct bbox* bbox1);
+void bbox_union_ondim(const struct bbox* bbox0, const struct bbox* bbox1, int dim, struct bbox* bbox2);
+int bbox_union(const struct bbox* bbox0, const struct bbox* bbox1, struct bbox* bbox2);
 
 uint64_t bbox_volume(const struct bbox *);
 void bbox_to_intv(const struct bbox *, uint64_t, int, struct intv **, int *);

include/dspaces-common.h

@@ -27,6 +27,8 @@ extern "C" {
 #define dspaces_ERR_UNKNOWN_PR -7  /* Could not find server */
 #define dspaces_ERR_UNKNOWN_OBJ -8 /* Could not find the object*/
 #define dspaces_ERR_END -9         /* End of range for valid error codes */
+#define dspaces_ERR_UTILS -12
+#define dspaces_ERR_HDF5 -13
 
 #define DS_MOD_EFAULT -1
 #define DS_MOD_ENODEF -2

include/dspaces-conf.h

@@ -5,6 +5,10 @@
 #include "dspaces-remote.h"
 #include "list.h"
 
+#ifdef DSPACES_HAVE_FILE_STORAGE
+#include "file_storage/policy.h"
+#endif
+
 static char *hash_strings[] = {"Dynamic", "Unitary", "SFC", "Bisection"};
 
 /* Server configuration parameters */
@@ -18,6 +22,9 @@ struct ds_conf {
     struct remote **remotes;
     int nremote;
     struct list_head *mods;
+#ifdef DSPACES_HAVE_FILE_STORAGE
+    struct swap_config swap;
+#endif
 };
 
 int parse_conf(const char *fname, struct ds_conf *conf);

include/file_storage/file.h

@@ -0,0 +1,10 @@
+#ifndef __DSPACES_FILE_STORAGE_H__
+#define __DSPACES_FILE_STORAGE_H__
+
+#include "ss_data.h"
+#include "file_storage/policy.h"
+
+int file_write_od(struct swap_config* swap_conf, struct obj_data *od);
+int file_read_od(struct swap_config* swap_conf, struct obj_data *od);
+
+#endif // __DSPACES_FILE_STORAGE_H__

include/file_storage/file_hdf5.h

@@ -0,0 +1,10 @@
+#ifndef __DSPACES_FILE_STORAGE_HDF5_H__
+#define __DSPACES_FILE_STORAGE_HDF5_H__
+
+#include "ss_data.h"
+#include "file_storage/policy.h"
+
+int hdf5_write_od(struct swap_config* swap_conf, struct obj_data *od);
+int hdf5_read_od(struct swap_config* swap_conf, struct obj_data *od);
+
+#endif // __DSPACES_FILE_STORAGE_HDF5_H__

include/file_storage/policy.h

@@ -0,0 +1,45 @@
+#ifndef __DSPACES_FILE_STORAGE_POLICY_H__
+#define __DSPACES_FILE_STORAGE_POLICY_H__
+
+#include "stdint.h"
+
+
+#include "ss_data.h"
+
+/* Server swap space configuration parameters */
+typedef enum mem_value_type {DS_MEM_BYTES, DS_MEM_PERCENT} ds_mem_val_t;
+
+typedef enum file_type {
+#ifdef DSPACES_HAVE_HDF5
+    DS_FILE_HDF5, 
+#endif
+#ifdef DSPACES_HAVE_NetCDF
+    DS_FILE_NetCDF
+#endif
+} ds_file_t;
+
+struct swap_config
+{
+    char *file_dir;
+    ds_mem_val_t mem_quota_type;
+    ds_file_t file_backend;
+    char *policy;
+    float disk_quota_MB;
+    union {
+        float MB;
+        float percent;
+    } mem_quota;
+};
+
+void free_ls_od_list(struct list_head* ls_od_list);
+
+void memory_quota_parser(char* str, struct swap_config* swap);
+void disk_quota_parser(char* str, struct swap_config* swap);
+
+int policy_str_check(const char*str);
+
+int need_swap_out(struct swap_config *swap, uint64_t size_MB);
+
+struct obj_data* which_swap_out(struct swap_config* swap, struct list_head* ls_od_list);
+
+#endif // __DSPACES_FILE_STORAGE_POLICY_H__

include/gspace.h

@@ -60,6 +60,11 @@ struct ds_gspace {
     /* Pending object descriptors for draining. */
     struct list_head obj_desc_drain_list;
 
+#ifdef DSPACES_HAVE_FILE_STORAGE
+    /* List of object data for swap out */
+    struct list_head ls_od_list;
+#endif // DSPACES_HAVE_FILE_STORAGE
+
     int rank;
     int size_sp;
     char **server_address;

include/ss_data.h

@@ -66,9 +66,20 @@ struct global_dimension {
     struct coord sizes;
 };
 
+#ifdef DSPACES_HAVE_FILE_STORAGE
+struct flat_od_list_info {
+    struct list_head entry;
+    int usecnt;
+};
+#endif // DSPACES_HAVE_FILE_STORAGE
+
 struct obj_data {
     struct list_head obj_entry;
 
+#ifdef DSPACES_HAVE_FILE_STORAGE
+    struct flat_od_list_info flat_list_entry;
+#endif // DSPACES_HAVE_FILE_STORAGE
+
     obj_descriptor obj_desc;
 
     struct global_dimension gdim;
@@ -439,6 +450,7 @@ struct obj_data *ls_lookup(ss_storage *, char *);
 void ls_remove(ss_storage *, struct obj_data *);
 void ls_try_remove_free(ss_storage *, struct obj_data *);
 struct obj_data *ls_find(ss_storage *, obj_descriptor *);
+struct obj_data *ls_find_include(ss_storage *, obj_descriptor *);
 struct obj_data *ls_find_od(ss_storage *, obj_descriptor *);
 int ls_find_ods(ss_storage *ls, obj_descriptor *odsc, obj_descriptor ***od_tab);
 struct obj_data *ls_find_no_version(ss_storage *, obj_descriptor *);
@@ -460,10 +472,11 @@ void obj_data_resize(obj_descriptor *obj_desc, uint64_t *new_dims);
 int obj_desc_equals(obj_descriptor *, obj_descriptor *);
 int obj_desc_equals_no_owner(const obj_descriptor *, const obj_descriptor *);
 
-int obj_desc_equals_intersect(obj_descriptor *odsc1, obj_descriptor *odsc2);
+int obj_desc_equals_intersect(const obj_descriptor *odsc1, const obj_descriptor *odsc2);
 
 int obj_desc_by_name_intersect(const obj_descriptor *odsc1,
                                const obj_descriptor *odsc2);
+int obj_desc_equals_include(const obj_descriptor *odsc1, const obj_descriptor *odsc2);
 
 // void copy_global_dimension(struct global_dimension *l, int ndim, const
 // uint64_t *gdim);

include/util.h

@@ -2,6 +2,7 @@
 #define __DS_UTIL_H_
 
 #include <stdlib.h>
+#include <stdint.h>
 
 size_t str_len(const char *str);
 char *str_append_const(char *, const char *);
@@ -28,4 +29,42 @@ struct name_value_pair *text_to_nv_pairs(const char *text);
 void free_nv_pairs(struct name_value_pair *pairs);
 char *alloc_sprintf(const char *fmt_str, ...);
 
+/*******************************************************
+   Memory Info
+**********************************************************/
+
+#define MEM_PATH_LEN 256
+typedef struct {
+    // Values from /proc/meminfo, in KiB or converted to MiB.
+    uint64_t MemTotalKiB;
+    uint64_t MemTotalMiB;
+    uint64_t MemAvailableMiB; // -1 means no data available
+    uint64_t SwapTotalMiB;
+    uint64_t SwapTotalKiB;
+    uint64_t SwapFreeMiB;
+    // Calculated percentages
+    double MemAvailablePercent; // percent of total memory that is available
+    double SwapFreePercent; // percent of total swap that is free
+} meminfo_t;
+
+typedef struct procinfo {
+    int pid;
+    int pidfd;
+    int uid;
+    int badness;
+    int oom_score_adj;
+    long long VmRSSkiB;
+    char name[MEM_PATH_LEN];
+} procinfo_t;
+
+meminfo_t parse_meminfo();
+
+/*******************************************************
+   Directory Opreations
+**********************************************************/
+int check_dir_exist(const char* dir_path);
+int check_dir_write_permission(const char* dir_path);
+void mkdir_all_owner_permission(const char* dir_path);
+int remove_dir_rf(const char *dir_path);
+
 #endif