Feat/file

CMakeLists.txt

@@ -74,11 +74,27 @@ if(OPENMP_FOUND AND DSPACES_USE_OPENMP)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
     add_definitions(-DOPS_USE_OPENMP)
 endif()
+
 find_package(CURL)
 if(CURL_FOUND)
     add_definitions(-DDSPACES_HAVE_CURL)
 endif()
 
+find_package(HDF5)
+if(HDF5_FOUND)
+    add_definitions(-DDSPACES_HAVE_HDF5)
+endif()
+
+find_package(NetCDF)
+if(NetCDF_FOUND)
+    add_definitions(-DDSPACES_HAVE_NetCDF)
+endif()
+
+if(HDF5_FOUND OR NetCDF_FOUND)
+    option(FILEBACKEND_FOUND "Option to Enable File Swap on DataSpaces Server" ON)
+    add_definitions(-DDSPACES_HAVE_FILEBACKEND)
+endif()
+
 include(CheckLanguage)
 check_language(Fortran)
 if(CMAKE_Fortran_COMPILER)

include/bbox.h

@@ -42,6 +42,11 @@ int bbox_include(const struct bbox *, const struct bbox *);
 int bbox_does_intersect(const struct bbox *, const struct bbox *);
 void bbox_intersect(const struct bbox *, const struct bbox *, struct bbox *);
 int bbox_equals(const struct bbox *, const struct bbox *);
+int bbox_include_ondim(const struct bbox *b0, const struct bbox *b1, int dim);
+int bbox_can_include(struct bbox* bbox0, struct bbox* bbox1);
+int bbox_can_union(const struct bbox* bbox0, const struct bbox* bbox1);
+void bbox_union_ondim(const struct bbox* bbox0, const struct bbox* bbox1, int dim, struct bbox* bbox2);
+int bbox_union(const struct bbox* bbox0, const struct bbox* bbox1, struct bbox* bbox2);
 
 uint64_t bbox_volume(const struct bbox *);
 void bbox_to_intv(const struct bbox *, uint64_t, int, struct intv **, int *);

include/dspaces-common.h

@@ -27,6 +27,8 @@ extern "C" {
 #define dspaces_ERR_UNKNOWN_PR -7  /* Could not find server */
 #define dspaces_ERR_UNKNOWN_OBJ -8 /* Could not find the object*/
 #define dspaces_ERR_END -9         /* End of range for valid error codes */
+#define dspaces_ERR_UTILS -12
+#define dspaces_ERR_HDF5 -13
 
 #define DS_MOD_EFAULT -1
 #define DS_MOD_ENODEF -2

include/dspaces-conf.h

@@ -4,6 +4,7 @@
 #include "bbox.h"
 #include "dspaces-remote.h"
 #include "list.h"
+#include "file_storage/policy.h"
 
 static char *hash_strings[] = {"Dynamic", "Unitary", "SFC", "Bisection"};
 
@@ -18,6 +19,7 @@ struct ds_conf {
     struct remote **remotes;
     int nremote;
     struct list_head *mods;
+    struct swap_config swap;
 };
 
 int parse_conf(const char *fname, struct ds_conf *conf);

include/file_storage/file_hdf5.h

@@ -0,0 +1,10 @@
+#ifndef __DSPACES_FILE_STORAGE_HDF5_H__
+#define __DSPACES_FILE_STORAGE_HDF5_H__
+
+#include "ss_data.h"
+#include "file_storage/policy.h"
+
+int hdf5_write_od(struct swap_config* swap_conf, struct obj_data *od);
+int hdf5_read_od(struct swap_config* swap_conf, struct obj_data *od);
+
+#endif // __DSPACES_FILE_STORAGE_HDF5_H__

include/file_storage/policy.h

@@ -0,0 +1,33 @@
+#ifndef __DSPACES_FILE_STORAGE_POLICY_H__
+#define __DSPACES_FILE_STORAGE_POLICY_H__
+
+#include "stdint.h"
+
+
+#include "ss_data.h"
+
+/* Server swap space configuration parameters */
+struct swap_config
+{
+    char *file_dir;
+    int mem_quota_type;
+    char *policy;
+    float disk_quota_MB;
+    union {
+        float MB;
+        float percent;
+    } mem_quota;
+};
+
+void free_ls_od_list(struct list_head* ls_od_list);
+
+void memory_quota_parser(char* str, struct swap_config* swap);
+void disk_quota_parser(char* str, struct swap_config* swap);
+
+int policy_str_check(const char*str);
+
+int need_swap_out(struct swap_config *swap, uint64_t size_MB);
+
+struct obj_data* which_swap_out(struct swap_config* swap, struct list_head* ls_od_list);
+
+#endif // __DSPACES_FILE_STORAGE_POLICY_H__

include/gspace.h

@@ -60,6 +60,9 @@ struct ds_gspace {
     /* Pending object descriptors for draining. */
     struct list_head obj_desc_drain_list;
 
+    /* List of object data for swap out */
+    struct list_head ls_od_list;
+
     int rank;
     int size_sp;
     char **server_address;

include/ss_data.h

@@ -66,9 +66,16 @@ struct global_dimension {
     struct coord sizes;
 };
 
+struct flat_od_list_info {
+    struct list_head entry;
+    int usecnt;
+};
+
 struct obj_data {
     struct list_head obj_entry;
 
+    struct flat_od_list_info flat_list_entry;
+
     obj_descriptor obj_desc;
 
     struct global_dimension gdim;
@@ -83,6 +90,9 @@ struct obj_data {
 
     /* Flag to mark if we should free this data object. */
     unsigned int f_free : 1;
+
+    /* Reference to the flat od list in local storage; Used for advanced swap policy */
+    struct obj_data_ptr_flat_list_entry* ls_od_entry;
 };
 
 struct gdim_list_entry {
@@ -439,6 +449,7 @@ struct obj_data *ls_lookup(ss_storage *, char *);
 void ls_remove(ss_storage *, struct obj_data *);
 void ls_try_remove_free(ss_storage *, struct obj_data *);
 struct obj_data *ls_find(ss_storage *, obj_descriptor *);
+struct obj_data *ls_find_include(ss_storage *, obj_descriptor *);
 struct obj_data *ls_find_od(ss_storage *, obj_descriptor *);
 int ls_find_ods(ss_storage *ls, obj_descriptor *odsc, obj_descriptor ***od_tab);
 struct obj_data *ls_find_no_version(ss_storage *, obj_descriptor *);
@@ -460,10 +471,11 @@ void obj_data_resize(obj_descriptor *obj_desc, uint64_t *new_dims);
 int obj_desc_equals(obj_descriptor *, obj_descriptor *);
 int obj_desc_equals_no_owner(const obj_descriptor *, const obj_descriptor *);
 
-int obj_desc_equals_intersect(obj_descriptor *odsc1, obj_descriptor *odsc2);
+int obj_desc_equals_intersect(const obj_descriptor *odsc1, const obj_descriptor *odsc2);
 
 int obj_desc_by_name_intersect(const obj_descriptor *odsc1,
                                const obj_descriptor *odsc2);
+int obj_desc_equals_include(const obj_descriptor *odsc1, const obj_descriptor *odsc2);
 
 // void copy_global_dimension(struct global_dimension *l, int ndim, const
 // uint64_t *gdim);

include/util.h

@@ -28,4 +28,42 @@ struct name_value_pair *text_to_nv_pairs(const char *text);
 void free_nv_pairs(struct name_value_pair *pairs);
 char *alloc_sprintf(const char *fmt_str, ...);
 
+/*******************************************************
+   Memory Info
+**********************************************************/
+
+#define MEM_PATH_LEN 256
+typedef struct {
+    // Values from /proc/meminfo, in KiB or converted to MiB.
+    long long MemTotalKiB;
+    long long MemTotalMiB;
+    long long MemAvailableMiB; // -1 means no data available
+    long long SwapTotalMiB;
+    long long SwapTotalKiB;
+    long long SwapFreeMiB;
+    // Calculated percentages
+    double MemAvailablePercent; // percent of total memory that is available
+    double SwapFreePercent; // percent of total swap that is free
+} meminfo_t;
+
+typedef struct procinfo {
+    int pid;
+    int pidfd;
+    int uid;
+    int badness;
+    int oom_score_adj;
+    long long VmRSSkiB;
+    char name[MEM_PATH_LEN];
+} procinfo_t;
+
+meminfo_t parse_meminfo();
+
+/*******************************************************
+   Directory Opreations
+**********************************************************/
+int check_dir_exist(const char* dir_path);
+int check_dir_write_permission(const char* dir_path);
+void mkdir_all_owner_permission(const char* dir_path);
+int remove_dir_rf(const char *dir_path);
+
 #endif

src/CMakeLists.txt

@@ -33,15 +33,16 @@ set_target_properties (dspaces
     PROPERTIES VERSION ${dspaces_VERSION}
     SOVERSION ${dspaces_VERSION_MAJOR})
 
-set(dspaces-server-src util.c bbox.c ss_data.c dspaces-server.c dspaces-conf.c dspaces-modules.c dspaces-logging.c toml.c)
+set(dspaces-server-src util.c bbox.c ss_data.c dspaces-server.c dspaces-conf.c dspaces-modules.c
+                        dspaces-logging.c toml.c file_storage/file_hdf5.c file_storage/policy.c)
 
 add_library(dspaces-server ${dspaces-server-src} ${dspaces-src})
 if(HAVE_DRC)
-    target_link_libraries (dspaces-server margo m pthread ${DRC_LIBRARIES} liblz4)
+    target_link_libraries (dspaces-server margo m pthread ${DRC_LIBRARIES} liblz4 ${CURL_LIBRARIES} HDF5::HDF5)
     target_include_directories (dspaces-server BEFORE PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> ${DRC_INCLUDE_DIRS})
 else()
-    target_link_libraries (dspaces-server margo m pthread liblz4)
+    target_link_libraries (dspaces-server margo m pthread liblz4 HDF5::HDF5)
     target_include_directories (dspaces-server BEFORE PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>)
 endif()

src/bbox.c

@@ -99,7 +99,7 @@ void bbox_divide(struct bbox *b0, struct bbox *b_tab)
 /*
    Test if bounding box b0 includes b1 along dimension dim.
 */
-static inline int bbox_include_ondim(const struct bbox *b0,
+int bbox_include_ondim(const struct bbox *b0,
                                      const struct bbox *b1, int dim)
 {
     if((b0->lb.c[dim] <= b1->lb.c[dim]) && (b0->ub.c[dim] >= b1->ub.c[dim]))
@@ -122,6 +122,46 @@ int bbox_include(const struct bbox *b0, const struct bbox *b1)
     return 1;
 }
 
+/* Test if bounding box bbox0 includes bbox1 or bbox1 includes bbox0
+ * along all dimensions. Return value could be:
+ * 0 - no include relationship
+ * 1 - bbox0 includes bbox1
+ * 2 - bbox1 includes bbox0
+ * */
+int bbox_can_include(struct bbox* bbox0, struct bbox* bbox1)
+{
+    int ret;
+    struct bbox* big, *small;
+
+    // Find the bigger bbox first
+    if(bbox_dist(bbox0, 0)<=bbox_dist(bbox1, 0)) {
+        big = bbox1;
+        small = bbox0;
+        ret = 2;
+    } else {
+        big = bbox0;
+        small = bbox1;
+        ret = 1;
+    }
+
+    // If we cannot find a bbox whose size on every dimension is
+    // bigger than the other one, then there is no chance for either
+    // 1 bbox to include the other 
+    for(int i=1; i<bbox0->num_dims; i++) {
+        if(bbox_dist(small, i)>bbox_dist(big, i)) {
+            return 0;
+        }
+    }
+
+    for(int i=0; i<bbox0->num_dims; i++) {
+        if(!bbox_include_ondim(big, small, i)) {
+            return 0;
+        }
+    }
+
+    return ret;
+}
+
 /*
   Test if bounding boxes b0 and b1 intersect along dimension dim.
 */
@@ -182,6 +222,102 @@ int bbox_equals(const struct bbox *bb0, const struct bbox *bb1)
     return 0;
 }
 
+/* Test if bounding box bbox0 and bbox1 can union.
+ * Return value is the dimension where they can union along,
+ * -1 when they cannot union.
+ * */
+int bbox_can_union(const struct bbox* bbox0, const struct bbox* bbox1)
+{
+    int share;
+    /* Two bbox can union means that they share a common
+     * (n-1) dim coordinates, and has 1 dimension extended. */
+    for(int i=0; i<bbox0->num_dims; i++) { // Choose 1 dimension
+        // check the coordinates of the other (n-1) dimension
+        share = 1;
+        for(int j=0; j<bbox0->num_dims; j++) {
+            if(j==i) continue;
+            if(bbox0->lb.c[j] != bbox1->lb.c[j] || bbox0->ub.c[j] != bbox1->ub.c[j]) {
+                share = 0;
+                break;
+            }
+        }
+
+        if(!share) continue; // choose the next dimension
+
+        if(share && bbox_intersect_ondim(bbox0, bbox1, i)) return i;
+    }
+    return -1;
+}
+
+/* Compute the union result of bounding box bbox0 and bbox1.
+ * Require to test if bbox0 & bbox1 can union before calling
+ * this function. Need the union dimension as the input.
+ * The result will be stored in bbox2.
+ * */
+void bbox_union_ondim(const struct bbox* bbox0, const struct bbox* bbox1,
+                                int dim, struct bbox* bbox2)
+{
+    bbox2->num_dims = bbox0->num_dims;
+    for(int i=0; i<bbox0->num_dims; i++) {
+        if(i==dim) {
+            bbox2->lb.c[i] = min(bbox0->lb.c[i], bbox1->lb.c[i]);
+            bbox2->ub.c[i] = min(bbox0->ub.c[i], bbox1->ub.c[i]);
+        } else {
+            bbox2->lb.c[i] = bbox0->lb.c[i];
+            bbox2->ub.c[i] = bbox0->ub.c[i];
+        }
+    }
+}
+
+/* Test if bounding box bbox0 and bbox1 can union and
+ * compute their union bbox. The result will be stored in bbox2.
+ * Return value is 1 when they can union, otherwise 0.
+ * */
+int bbox_union(const struct bbox* bbox0, const struct bbox* bbox1, struct bbox* bbox2)
+{
+    int found = 0, share;
+    int dim;
+    /* Two bbox can union means that they share a common
+        (n-1) dim coordinates, and has 1 dimension extended. */
+    for(int i=0; i<bbox0->num_dims; i++) { // Choose 1 dimension
+        // check the coordinates of the other (n-1) dimension
+        share = 1;
+        for(int j=0; j<bbox0->num_dims; j++) {
+            if(j==i) continue;
+            if(bbox0->lb.c[j] != bbox1->lb.c[j] || bbox0->ub.c[j] != bbox1->ub.c[j]) {
+                share = 0;
+                break;
+            }
+        }
+
+        if(!share) continue; // choose the next dimension
+
+        found = share && bbox_intersect_ondim(bbox0, bbox1, i);
+
+        if(found) {
+            dim = i;
+            break;
+        }
+    }
+
+    if(!found) {
+        return 0;
+    } else { // fill bbox2
+        bbox2->num_dims = bbox0->num_dims;
+        for(int i=0; i<bbox0->num_dims; i++) {
+            if(i==dim) {
+                bbox2->lb.c[i] = min(bbox0->lb.c[i], bbox1->lb.c[i]);
+                bbox2->ub.c[i] = min(bbox0->ub.c[i], bbox1->ub.c[i]);
+            } else {
+                bbox2->lb.c[i] = bbox0->lb.c[i];
+                bbox2->ub.c[i] = bbox0->ub.c[i];
+            }
+        }
+        return 1;
+    }
+
+}
+
 uint64_t bbox_volume(const struct bbox *bb)
 {
     uint64_t n = 1;