Repository navigation
482 lines (423 loc) · 19.3 KB
/
Copy pathci.yml
File metadata and controls
482 lines (423 loc) · 19.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
name: CI
on:
push:
branches: [main]
pull_request:
# The version matrix is a measurement rather than a gate, and wanting it again is not a reason to push
# a commit. Qdrant releasing a new minor is the usual reason.
workflow_dispatch:
permissions:
contents: read
# Run every JavaScript-based action on Node 24. The runner is phasing out Node 20
# (some actions still declare node20); this forces the newer runtime everywhere and
# drops the "Node 20 is being deprecated" warnings.
env:
ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node24
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
jobs:
validate-wrapper:
name: Validate Gradle wrapper
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: gradle/actions/wrapper-validation@v6
vendored-qdrant:
name: Vendored Qdrant files
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up JDK 21
uses: actions/setup-java@v6
with:
java-version: "21"
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
# The protobuf definitions and the OpenAPI document the contract test validates against are
# copies of upstream's, and a copy nobody diffs is a copy that drifts. This reaches the network,
# which is why it is a job of its own rather than part of `check`.
- name: Compare every vendored file with the pinned Qdrant tag
run: ./gradlew verifyVendoredQdrant --no-daemon --stacktrace
build:
name: Build (JDK ${{ matrix.java }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
java: ["17", "21"]
steps:
- uses: actions/checkout@v7
# `build` links the linuxX64 test binary, and the Linux engine is Ktor's Curl. Without the
# headers the link fails, which is a confusing way to learn that the transport went native.
- name: Install libcurl
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v6
with:
java-version: ${{ matrix.java }}
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
- name: Build, test, lint (ktlint + detekt) and verify public API
run: ./gradlew build --no-daemon --stacktrace
# The JMH sources are not part of `build`, so nothing compiled them and they rotted: the official
# client moved PointId into another generated class at 1.19 and the comparison harness stopped
# compiling without anything saying so. A harness that does not build publishes no numbers, which
# is a slower way to have no benchmark than not writing one.
- name: Compile the benchmark harness
if: matrix.java == '17'
run: ./gradlew :benchmarks:compileJmhKotlin --no-daemon --stacktrace
- name: Coverage (Kover)
if: matrix.java == '17'
run: ./gradlew koverXmlReport koverHtmlReport koverVerify --no-daemon --stacktrace
- name: Upload coverage report
if: matrix.java == '17'
uses: actions/upload-artifact@v7
with:
name: coverage
path: build/reports/kover/
retention-days: 7
- name: Upload reports
if: failure()
uses: actions/upload-artifact@v7
with:
name: reports-jdk-${{ matrix.java }}
path: |
**/build/reports/
**/build/test-results/
retention-days: 7
# The Linux build above compiles the multiplatform modules for the JVM, Linux and Windows, and skips
# the Apple targets because no Linux host can build them. Without this job an iOS or macOS break would
# first be seen at release time, on the one runner that does compile them.
apple-targets:
name: Apple targets
runs-on: macos-latest
steps:
- uses: actions/checkout@v7
- name: Set up JDK 17
uses: actions/setup-java@v6
with:
java-version: "17"
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
# The transport is here as well as the core from 2.1.0. Before that an iOS consumer got the
# models and no engine, so compiling the core alone was the whole claim.
- name: Compile and test the Apple targets
run: >
./gradlew
:kdrant-core:compileKotlinIosArm64
:kdrant-core:compileKotlinIosX64
:kdrant-core:iosSimulatorArm64Test
:kdrant-core:macosArm64Test
:kdrant-transport-rest:compileKotlinIosArm64
:kdrant-transport-rest:compileKotlinIosX64
:kdrant-transport-rest:compileKotlinIosSimulatorArm64
:kdrant-migrate:compileKotlinIosArm64
--no-daemon --stacktrace
# The klib dump is one merged file per module covering every native target, and only a Mac can
# build the Apple ones. Checking it anywhere else would pass on a dump with the iOS declarations
# missing, which is the exact failure the dump exists to catch.
- name: Verify the public API, klibs included
run: ./gradlew apiCheck --no-daemon --stacktrace
# M38's exit criterion. A klib that links has not been shown to work, so the same behavioural contract
# the JVM runs is run from a native binary against a real Qdrant — once on Linux, once on macOS.
native-contract-linux:
name: Client contract (Linux native)
runs-on: ubuntu-latest
services:
qdrant:
image: qdrant/qdrant:v1.19.1
ports:
- 6333:6333
steps:
- uses: actions/checkout@v7
# The Linux engine is Curl, which links against the system libcurl. Saying so here is the same
# statement the module's KDoc makes to a consumer building for a slim image.
- name: Install libcurl
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
- name: Set up JDK 17
uses: actions/setup-java@v6
with:
java-version: "17"
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
- name: Run the client contract from a linuxX64 binary
env:
KDRANT_QDRANT_HOST: 127.0.0.1
KDRANT_QDRANT_PORT: "6333"
KDRANT_QDRANT_REQUIRED: "1"
run: ./gradlew :kdrant-transport-rest:linuxX64Test --no-daemon --stacktrace
native-contract-macos:
name: Client contract (macOS native)
runs-on: macos-latest
steps:
- uses: actions/checkout@v7
# No Docker on a macOS runner, so Qdrant is the released binary rather than the image. It is the
# same server; Testcontainers is a JVM convenience, and this job has no JVM in the process under
# test.
- name: Start Qdrant
run: |
# The download is retried because the hosted runner's TLS to the release host fails now and
# then — "self signed certificate", on a certificate that is fine from everywhere else, and
# gone on the next attempt. Retrying is the honest answer to that; -k would turn a flake into
# a job that downloads and runs whatever answers the name.
downloaded=""
for attempt in 1 2 3; do
if curl -fsSL -o qdrant.tar.gz \
"https://github.com/qdrant/qdrant/releases/download/v${QDRANT_VERSION}/qdrant-aarch64-apple-darwin.tar.gz"
then
downloaded=yes
break
fi
sleep $((attempt * 5))
done
if [ -z "$downloaded" ]; then
echo "::error::could not download Qdrant ${QDRANT_VERSION} after three attempts"; exit 1
fi
tar -xzf qdrant.tar.gz
./qdrant &
for _ in $(seq 1 60); do
curl -fsS http://127.0.0.1:6333/readyz >/dev/null 2>&1 && exit 0
sleep 1
done
echo "::error::Qdrant did not become ready"; exit 1
env:
QDRANT_VERSION: "1.19.1"
- name: Set up JDK 17
uses: actions/setup-java@v6
with:
java-version: "17"
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
- name: Run the client contract from a macosArm64 binary
env:
KDRANT_QDRANT_HOST: 127.0.0.1
KDRANT_QDRANT_PORT: "6333"
KDRANT_QDRANT_REQUIRED: "1"
run: ./gradlew :kdrant-transport-rest:macosArm64Test --no-daemon --stacktrace
# M40's exit criterion. The README's comparison table claimed Kdrant was GraalVM native-image friendly
# and needed no reflection configuration, and nobody had ever built an image. This job is the claim:
# --no-fallback fails the build the day a dependency starts reflecting, and the binary then has to
# answer a real search against a real Qdrant.
native-image:
name: GraalVM native image
runs-on: ubuntu-latest
services:
qdrant:
image: qdrant/qdrant:v1.19.1
ports:
- 6333:6333
steps:
- uses: actions/checkout@v7
# Two JDKs on purpose: the library modules compile against a 17 toolchain, and native-image comes
# from the GraalVM distribution, which is a 21.
- name: Set up JDK 17
uses: actions/setup-java@v6
with:
java-version: "17"
distribution: temurin
- name: Set up GraalVM
uses: graalvm/setup-graalvm@v1
with:
java-version: "21"
distribution: graalvm
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
- name: Build the native image
run: >
./gradlew :example-native-image:nativeCompile
-Porg.gradle.java.installations.fromEnv=JAVA_HOME_17_X64,GRAALVM_HOME
--no-daemon --stacktrace
# Seeded here rather than by the binary, because the binary's clock starts at process start and a
# collection it created itself would be inside the measurement. What is being timed is a cold
# process answering one search, which is the claim the README makes.
- name: Seed the collection the timed search reads
run: |
curl -fsS -X DELETE http://127.0.0.1:6333/collections/native-image-smoke > /dev/null 2>&1 || true
curl -fsS -X PUT http://127.0.0.1:6333/collections/native-image-smoke \
-H 'content-type: application/json' \
-d '{"vectors":{"size":8,"distance":"Cosine"}}' > /dev/null
curl -fsS -X PUT "http://127.0.0.1:6333/collections/native-image-smoke/points?wait=true" \
-H 'content-type: application/json' \
-d '{"points":[
{"id":1,"vector":[1,0,0,0,0,0,0,0],"payload":{"lang":"kotlin"}},
{"id":2,"vector":[0,1,0,0,0,0,0,0],"payload":{"lang":"rust"}},
{"id":3,"vector":[0,0,1,0,0,0,0,0],"payload":{"lang":"go"}}]}' > /dev/null
- name: Search from the native image, and measure what it cost
env:
QDRANT_HOST: 127.0.0.1
QDRANT_PORT: "6333"
run: |
BINARY=example-native-image/build/native/nativeCompile/kdrant-native-smoke
SIZE_MB=$(du -m "$BINARY" | cut -f1)
OUTPUT=$("$BINARY")
echo "$OUTPUT"
FIRST_SEARCH_MS=$(printf '%s\n' "$OUTPUT" | sed -n 's/^KDRANT_TIME_TO_FIRST_SEARCH_MS=//p')
if [ -z "$FIRST_SEARCH_MS" ]; then
echo "::error::the native image did not report a time to first search"; exit 1
fi
# The README publishes this number, so something has to defend it: before this the job measured
# the time, printed it, and passed whatever it said.
#
# The ceiling is loose on purpose, and the first attempt at it taught why. It was 75, on a
# measurement that then spanned a collection creation and an `upsert` with `wait = true`, and a
# run came back at 290 because the server was slow rather than the client. The timed span is
# now one search from a cold process, which is what the claim is about and has a far shorter
# tail, but a shared runner still has bad days. 150 catches a regression that quadruples the
# usual figure without going red over a noisy neighbour. Tighten it once there are enough runs
# to know the distribution rather than four samples of it.
CEILING_MS=150
if [ "$FIRST_SEARCH_MS" -gt "$CEILING_MS" ]; then
echo "::error::first search took ${FIRST_SEARCH_MS} ms, over the ${CEILING_MS} ms ceiling " \
"the README's claim rests on. Either something regressed or the claim needs rewriting."
exit 1
fi
{
echo "### GraalVM native image"
echo
echo "| | |"
echo "| --- | --- |"
echo "| Binary size | ${SIZE_MB} MB |"
echo "| Time to first search, cold | ${FIRST_SEARCH_MS} ms |"
echo "| Reflection metadata | generated from the classpath, shipped in kdrant-transport-rest |"
} >> "$GITHUB_STEP_SUMMARY"
cli:
name: The binaries against a real Qdrant
runs-on: ubuntu-latest
# Linux only, and on purpose. This job exists to catch a defect before a tag, and every defect that
# stopped 2.2.0 three times was in the tool rather than in a platform: the Linux runner has Docker,
# so it is the cheap one, and the release still builds and proves all three binaries. A macOS or
# Windows job here would triple the wall clock to re-prove what the release re-proves anyway.
services:
qdrant:
image: qdrant/qdrant:v1.19.1
ports:
- 6333:6333
steps:
- uses: actions/checkout@v7
- name: Install libcurl
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
- name: Set up JDK 17
uses: actions/setup-java@v6
with:
java-version: "17"
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
- name: Build the binaries
run: >
./gradlew :kdrant-cli:linkKdrantReleaseExecutableLinuxX64
:kdrant-mcp:linkKdrant-mcpReleaseExecutableLinuxX64
--no-daemon --no-configuration-cache --stacktrace
- name: Wait for Qdrant
run: |
for _ in $(seq 1 60); do
curl -fsS http://127.0.0.1:6333/readyz >/dev/null 2>&1 && exit 0
sleep 1
done
echo "::error::Qdrant did not become ready"; exit 1
# The same scripts the release runs, so the release is the second time these execute.
- name: Run every subcommand
run: bash .github/scripts/prove-cli.sh linuxX64
# The MCP server is a protocol rather than a command line, so it is proven by speaking to it: a
# client initializes, lists the tools and completes a search. It also checks that nothing but
# JSON-RPC reaches stdout, which is how this nearly shipped broken: the SDK's logging library
# prints a banner there, before the handshake, and a client sees a parse error and disconnects.
- name: Speak MCP to the server
run: bash .github/scripts/prove-mcp.sh linuxX64
qdrant-compat:
name: Integration (${{ matrix.qdrant }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
qdrant:
- "qdrant/qdrant:v1.19.1"
- "qdrant/qdrant:latest"
steps:
- uses: actions/checkout@v7
- name: Set up JDK 17
uses: actions/setup-java@v6
with:
java-version: "17"
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
# Both engines run the same client contract from kdrant-testkit, against the same image. The otel,
# micrometer and migrate modules join them: none can be proven against a mock — two compare what
# two real engines emit, the third kills a copy halfway and restarts it.
- name: Integration tests against ${{ matrix.qdrant }}
env:
QDRANT_IMAGE: ${{ matrix.qdrant }}
run: >
./gradlew :kdrant-transport-rest:jvmTest :kdrant-transport-grpc:test
:kdrant-otel:test :kdrant-micrometer:test :kdrant-migrate:jvmTest
--tests "*IntegrationTest*" --no-daemon --stacktrace
# M50. The blocking compatibility check is the job above, against the pinned image and `latest`. This
# one runs the same contract against the four most recent minors and publishes the result, including
# where it is red: a reader on an older server needs to know which part is affected, which a range
# like "1.16 and later" cannot say. It does not gate the build, deliberately — a matrix that fails a
# release for being honest is a matrix that gets deleted.
qdrant-version-matrix:
name: Qdrant version matrix
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@v7
- name: Set up JDK 17
uses: actions/setup-java@v6
with:
java-version: "17"
distribution: temurin
- name: Set up Gradle
uses: gradle/actions/setup-gradle@v6
# `--rerun` because this job's product is a measurement, not a verdict, and Gradle cannot tell the
# difference. The task was restored FROM-CACHE on every run after the one that first produced the
# table, so the matrix stopped executing, wrote no report, uploaded no artifact, and reported
# success the whole time. The README's table went three releases stale that way: it named Qdrant
# v1.19.0 and 39 contract cases when the pin was v1.19.1 and the contract had 46.
- name: Run the client contract against every version
run: >
./gradlew :kdrant-transport-rest:jvmTest --rerun
--tests "*QdrantVersionMatrixIntegrationTest*" --no-daemon --stacktrace
# The table itself, not a pointer to it. A summary that only says how to regenerate the table is a
# summary that makes somebody run it again to read what this run already produced.
- name: Publish the matrix to the run summary
if: always()
run: |
TABLE=kdrant-transport-rest/build/reports/qdrant-matrix.md
{
echo "### Qdrant version matrix"
echo
if [ -s "$TABLE" ]; then cat "$TABLE"; else echo "The matrix produced no table."; fi
echo
echo "Paste it between the compatibility markers in README.md, or regenerate it there with:"
echo
echo '```bash'
echo "KDRANT_UPDATE_COMPAT=1 ./gradlew :kdrant-transport-rest:jvmTest --tests '*QdrantVersionMatrix*'"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload the matrix
if: always()
uses: actions/upload-artifact@v7
with:
name: qdrant-version-matrix
path: kdrant-transport-rest/build/reports/qdrant-matrix.md
retention-days: 30
# An error rather than ignore. A cell that fails against an older server is what this job
# exists to publish and must not fail it; a run that produced no table at all measured
# nothing, which is the one outcome here that is a defect rather than a result.
if-no-files-found: error
dependency-review:
name: Dependency review
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/dependency-review-action@v5