Skip to content
Open
Show file tree
Hide file tree
Changes from 53 commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
f221341
Add Dataset API for CAGRA (host vs device / owning vs view / standard…
HowardHuang1 Jul 27, 2026
badb837
Merge remote-tracking branch 'upstream/release/26.08' into HH-Dataset…
HowardHuang1 Jul 27, 2026
41a6f22
Fix failing hnsw host index serialize_to_hnswlib test case
HowardHuang1 Jul 27, 2026
b406041
squash all Tarang commits to remove 26.10 ancestry. Should be left wi…
HowardHuang1 Jul 28, 2026
23e4114
Merge pull request #1 from tarang-jain/hh-dataset-api
HowardHuang1 Jul 28, 2026
0882b6c
merge API updates
aamijar Jul 28, 2026
56cb7e1
c-serialize
divyegala Jul 28, 2026
e2d7b53
Merge remote-tracking branch 'upstream/release/26.08' into HH-Dataset…
HowardHuang1 Jul 28, 2026
b9dd705
rust: make CAGRA dataset ownership explicit
yan-zaretskiy Jul 28, 2026
2354487
record type and layout in serialization
divyegala Jul 28, 2026
b28cc72
merge and reconcile conflicts
divyegala Jul 28, 2026
07f91a1
rename make verb + noun
aamijar Jul 28, 2026
5e334b2
skip dataset when reading graph only
divyegala Jul 28, 2026
dab5c6b
Add CAGRA build heuristics independent of HNSW parameters (#2345)
achirkin Jul 28, 2026
195b04c
Merge remote-tracking branch 'upstream/release/26.08' into HH-Dataset…
HowardHuang1 Jul 28, 2026
11af0a6
Fix failing rocky clib tests on CI
HowardHuang1 Jul 28, 2026
a306dc9
remove cuvsCagraIndex_t from dataset.h
aamijar Jul 28, 2026
7e68c56
finish audit
divyegala Jul 28, 2026
5f7b90b
Fix regression introduced by upstream merge still using old build() f…
HowardHuang1 Jul 28, 2026
fdfab57
review bug
divyegala Jul 28, 2026
21481e0
unify dataset make and view factories
aamijar Jul 28, 2026
6d534e2
Revert "unify dataset make and view factories"
aamijar Jul 29, 2026
2b753ac
unify dataset make and view factories 2
aamijar Jul 29, 2026
b4f7d3e
Fix failing Java test cases. Java test call sites were outdated and m…
HowardHuang1 Jul 29, 2026
2ffe67b
Fix merge conflicts. Factory names were using old factory names prior…
HowardHuang1 Jul 29, 2026
ed911bf
conflicts
divyegala Jul 29, 2026
c503887
Merge remote-tracking branch 'upstream/release/26.08' into HH-Dataset…
HowardHuang1 Jul 29, 2026
38cf6fc
Have downstream wrappers mirror dataset factory consolidation in C AP…
HowardHuang1 Jul 29, 2026
80905ea
Fix merge conflict
HowardHuang1 Jul 29, 2026
96ff183
Merge remote-tracking branch 'upstream/release/26.08' into HH-Dataset…
HowardHuang1 Jul 29, 2026
a6dd4eb
Fix merge conflict that brought back compression param in a ann_cagra…
HowardHuang1 Jul 29, 2026
089808a
merge
divyegala Jul 29, 2026
f52a0a7
merge
divyegala Jul 29, 2026
501d066
undo java and python changes
divyegala Jul 29, 2026
469f912
Fix merge conflict where upstream introduced a new cpp test case that…
HowardHuang1 Jul 29, 2026
33ce2bd
Update get_dlpack.cmake
divyegala Jul 29, 2026
679553e
Merge pull request #2 from divyegala/c-serialize
HowardHuang1 Jul 29, 2026
8b20957
Fix styles. Fix copyright issue from merge
HowardHuang1 Jul 29, 2026
ed188de
target_mem_type argument in C API
aamijar Jul 29, 2026
598998c
restore java behavior
divyegala Jul 29, 2026
04649cb
Update get_dlpack.cmake
divyegala Jul 29, 2026
4f5ebd5
Update get_dlpack.cmake
divyegala Jul 29, 2026
fe7a121
add datasets back
divyegala Jul 30, 2026
f046dd2
Merge remote-tracking branch 'origin/java-serialize' into java-serialize
divyegala Jul 30, 2026
7f81391
update python api
tarang-jain Jul 30, 2026
c8df0d4
Add new extend API contract. Users no longer pass in 2 separate datas…
HowardHuang1 Jul 30, 2026
e916441
Merge remote-tracking branch 'upstream/release/26.08' into HH-Dataset…
HowardHuang1 Jul 30, 2026
5581ffd
Merge pull request #3 from divyegala/java-serialize
HowardHuang1 Jul 30, 2026
8140998
Fix merge conflicts. Upstream introduced new test cases that used old…
HowardHuang1 Jul 30, 2026
62a8da6
More test case call site fixes to resolve merge conflicts and port th…
HowardHuang1 Jul 30, 2026
06b02ad
remove explicit view type in C API
aamijar Jul 30, 2026
378ac4e
bool is_owning
aamijar Jul 30, 2026
a08714f
Revert "update python api"
tarang-jain Jul 30, 2026
55bc6e8
Merge branch 'HH-Dataset-API' of https://github.com/HowardHuang1/cuvs…
tarang-jain Jul 30, 2026
02963e1
remove unused header in dataset.h
aamijar Jul 30, 2026
1d23a34
fix python api
tarang-jain Jul 30, 2026
09b3334
Increase binary limits to pass CI
HowardHuang1 Jul 30, 2026
9f90009
new updates after pr reviews
tarang-jain Jul 30, 2026
640ab06
update the python tests
tarang-jain Jul 30, 2026
84eeb19
address pr review
tarang-jain Jul 30, 2026
3a3c116
Fix failing NEIGHBORS_MG_TEST which was calling MG with ace params. T…
HowardHuang1 Jul 30, 2026
15a50a7
new c getters
tarang-jain Jul 30, 2026
a7594d0
add c tests for dataset
tarang-jain Jul 30, 2026
1763e0b
update pydocs
tarang-jain Jul 30, 2026
09256be
Merge pull request #4 from tarang-jain/hh-dataset-api-2
HowardHuang1 Jul 30, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,12 @@ cuvsResourcesCreate(&res);
cuvsCagraIndexParamsCreate(&index_params);
cuvsCagraIndexCreate(&index);
cuvsCagraBuild(res, index_params, dataset, index);
cuvsDataset_t dataset_view;
cuvsDatasetMakeStandardView(res, dataset, &dataset_view);
cuvsCagraBuild(res, index_params, dataset_view, index);
cuvsDatasetDestroy(dataset_view);
cuvsCagraIndexDestroy(index);
cuvsCagraIndexParamsDestroy(index_params);
cuvsResourcesDestroy(res);
Expand Down
1 change: 1 addition & 0 deletions c/include/cuvs/core/all.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <cuvs/core/c_config.h>
#include <cuvs/core/export.h>
#include <cuvs/core/c_api.h>
#include <cuvs/core/dataset.h>

#include <cuvs/cluster/kmeans.h>

Expand Down
98 changes: 98 additions & 0 deletions c/include/cuvs/core/dataset.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

#pragma once

#include <cuvs/core/c_api.h>

#include <dlpack/dlpack.h>
#include <stdbool.h>
#include <stdint.h>

#ifdef __cplusplus
extern "C" {
#endif

/**
* @brief Generic dataset layout kind for C API dataset handles.
*/
typedef enum {
CUVS_DATASET_LAYOUT_STANDARD = 0,
CUVS_DATASET_LAYOUT_PADDED = 1

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will this be expanded later with compressed layouts?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. This enum is not final.

} cuvsDatasetLayout_t;

/**
* @brief Memory space holding a C API dataset handle's data.
*/
typedef enum {
CUVS_DATASET_MEM_TYPE_HOST = 0,
CUVS_DATASET_MEM_TYPE_DEVICE = 1
} cuvsDatasetMemType_t;

/**
* @brief Dataset handle representing owning storage or a non-owning view.
*
* `addr` points to C++ dataset storage or view metadata managed by the C API. `mem_type`
* identifies the memory space, `layout` identifies the data layout, and `is_owning` indicates
* whether the handle owns its backing data.
*/
typedef struct {
uintptr_t addr;
void (*destroy_addr)(void*);
DLDataType dtype;
cuvsDatasetMemType_t mem_type;
cuvsDatasetLayout_t layout;
bool is_owning;
} cuvsDataset;
typedef cuvsDataset* cuvsDataset_t;

/**
* @brief Create an empty owning dataset handle.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can improve the docs here slightly to also mention that the underlying data is not allocated, only the handle.

*
* The dataset storage, memory type, layout, and dtype are populated by the operation that fills
* this handle.
*/
CUVS_EXPORT cuvsError_t cuvsDatasetCreate(cuvsDataset_t* dataset);

/**
* @brief Create an owning padded dataset in the requested memory space.
*
* The source tensor may reside in host- or device-accessible memory. Its contents are copied into
* newly allocated padded storage in `target_mem_type`.
*
* @param[in] res cuVS resources
* @param[in] dataset source tensor
* @param[in] target_mem_type memory space in which to allocate the padded dataset
* @param[out] padded_dataset newly allocated owning padded dataset
*/
CUVS_EXPORT cuvsError_t cuvsDatasetMakePadded(cuvsResources_t res,
DLManagedTensor* dataset,
cuvsDatasetMemType_t target_mem_type,
cuvsDataset_t* padded_dataset);

/**
* @brief Create a non-owning padded dataset view from a host- or device-resident tensor.
*
* Memory residency is inferred from the tensor.
*/
CUVS_EXPORT cuvsError_t cuvsDatasetMakePaddedView(cuvsResources_t res,
DLManagedTensor* dataset,
cuvsDataset_t* padded_dataset);

/**
* @brief Create a non-owning standard dataset view from a host- or device-resident tensor.
*
* Memory residency is inferred from the tensor.
*/
CUVS_EXPORT cuvsError_t cuvsDatasetMakeStandardView(cuvsResources_t res,
DLManagedTensor* dataset,
cuvsDataset_t* standard_dataset);

/** @brief Destroy a dataset handle created by a `cuvsDatasetMake*` function. */
CUVS_EXPORT cuvsError_t cuvsDatasetDestroy(cuvsDataset_t dataset);

#ifdef __cplusplus
}
#endif
Loading
Loading