Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions .github/scripts/test_behavior/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@

INTEGRATIONS = ["object_store"]

SHARED_SERVICE_SCHEMES = {
"s3": {"s3", "minio", "r2"},
}


def provided_cases() -> list[dict[str, str]]:
root_dir = f"{GITHUB_DIR}/services"
Expand Down Expand Up @@ -116,9 +120,10 @@ def mark_service_affected(service: str) -> None:
for integration in INTEGRATIONS:
setattr(hint, f"integration_{integration}", True)

hint.services.add(service)
hint.services.add(service.replace("-", "_"))
hint.services.add(service.replace("_", "-"))
for scheme in SHARED_SERVICE_SCHEMES.get(service, {service}):
hint.services.add(scheme)
hint.services.add(scheme.replace("-", "_"))
hint.services.add(scheme.replace("_", "-"))

for p in changed_files:
# workflow behavior tests affected
Expand Down Expand Up @@ -311,6 +316,10 @@ def generate_language_binding_cases(
# Remove invalid cases for go.
if language == "go":
cases = [v for v in cases if v["service"] not in [
# Provider presets share the Rust S3 crate and don't have separate
# opendal-go-services packages.
"minio",
"r2",
# opendal-go-services doesn't provide TOS yet.
"tos",
]]
Expand All @@ -330,6 +339,8 @@ def generate_language_binding_cases(
"memory",
"obs",
"oss",
"minio",
"r2",
"s3",
"webdav",
"webhdfs",
Expand Down
30 changes: 30 additions & 0 deletions .github/scripts/test_behavior/test_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,36 @@ def test_core_services_hdfs_native_mapping(self):
self.assertTrue("hdfs_native" in cases)
self.assertFalse("fs" in cases)

@patch.dict("os.environ", {"GITHUB_HAS_SECRETS": "true"})
def test_s3_crate_schedules_provider_cases(self):
result = plan(["core/services/s3/src/lib.rs"])
core_cases = {
(case["service"], case["feature"])
for target in result["core"]
for case in target["cases"]
}
self.assertEqual(
core_cases,
{
("s3", "services-s3"),
("minio", "services-minio"),
("r2", "services-r2"),
},
)

go_services = {
case["service"]
for target in result["binding_go"]
for case in target["cases"]
}
ruby_services = {
case["service"]
for target in result["binding_ruby"]
for case in target["cases"]
}
self.assertTrue({"minio", "r2"}.isdisjoint(go_services))
self.assertTrue({"minio", "r2"} <= ruby_services)

def test_group_core_cases_by_service(self):
cases = [
{
Expand Down
46 changes: 46 additions & 0 deletions .github/services/minio/0_minio_s3/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: minio
description: 'Behavior test for the MinIO provider preset.'

runs:
using: "composite"
steps:
- name: Setup MinIO Server
shell: bash
working-directory: fixtures/s3
run: docker compose -f docker-compose-minio.yml up -d --wait
- name: Setup test bucket
shell: bash
env:
AWS_ACCESS_KEY_ID: "minioadmin"
AWS_SECRET_ACCESS_KEY: "minioadmin"
AWS_EC2_METADATA_DISABLED: "true"
run: |
aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test

- name: Setup
shell: bash
run: |
cat << EOF >> $GITHUB_ENV
OPENDAL_MINIO_BUCKET=test
OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000
OPENDAL_MINIO_ACCESS_KEY_ID=minioadmin
OPENDAL_MINIO_SECRET_ACCESS_KEY=minioadmin
OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,restore=false,restore_with_version=false,restore_with_if_not_exists=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false,delete_with_if_match=false
EOF
51 changes: 51 additions & 0 deletions .github/services/minio/minio_s3_with_anonymous/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: minio_with_anonymous
description: 'Behavior test for the MinIO provider preset with anonymous access.'

runs:
using: "composite"
steps:
- name: Setup MinIO Server
shell: bash
working-directory: fixtures/s3
run: |
docker compose -f docker-compose-minio.yml up -d --wait
- name: Setup test bucket
shell: bash
env:
AWS_ACCESS_KEY_ID: "minioadmin"
AWS_SECRET_ACCESS_KEY: "minioadmin"
AWS_EC2_METADATA_DISABLED: "true"
run: |
set -e
aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test

# MinIO archived community client downloads, so dl.min.io now returns
# HTTP 410 for `mc`. Apply the same readwrite bucket policy that
# `mc anonymous set public` used to set.
aws --endpoint-url http://127.0.0.1:9000/ s3api put-bucket-policy --bucket test --policy '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::test"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:AbortMultipartUpload","s3:ListMultipartUploadParts"],"Resource":["arn:aws:s3:::test/*"]}]}'
- name: Setup
shell: bash
run: |
cat << EOF >> $GITHUB_ENV
OPENDAL_MINIO_BUCKET=test
OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000
OPENDAL_MINIO_SKIP_SIGNATURE=true
OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,restore=false,restore_with_version=false,restore_with_if_not_exists=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false,delete_with_if_match=false
EOF
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# under the License.

name: r2
description: "Behavior test for Cloudflare R2. This service is sponsored by @Xuanwo."
description: "Behavior test for the Cloudflare R2 provider preset. This service is sponsored by @Xuanwo."

runs:
using: "composite"
Expand All @@ -26,17 +26,15 @@ runs:
with:
export-env: true
env:
OPENDAL_S3_BUCKET: op://services/r2/bucket
OPENDAL_S3_ENDPOINT: op://services/r2/endpoint
OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
OPENDAL_R2_BUCKET: op://services/r2/bucket
OPENDAL_R2_ENDPOINT: op://services/r2/endpoint
OPENDAL_R2_ACCESS_KEY_ID: op://services/r2/access_key_id
OPENDAL_R2_SECRET_ACCESS_KEY: op://services/r2/secret_access_key

# R2 has a lower delete batch limit and doesn't support stat override response queries.
# Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
- name: Add extra settings
- name: Add test overrides
shell: bash
run: |
# R2 does not enforce destination copy conditions or delete If-Match.
cat << EOF >> $GITHUB_ENV
OPENDAL_S3_REGION=auto
OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,restore=false,restore_with_version=false,restore_with_if_not_exists=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false
OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,copy_with_if_not_exists=false,copy_with_if_match=false,delete_with_if_match=false,restore=false,restore_with_version=false,restore_with_if_not_exists=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false
EOF
2 changes: 2 additions & 0 deletions bindings/dart/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@ opendal = { path = "../../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
"services-minio",
"services-obs",
"services-oss",
"services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
Expand Down
2 changes: 2 additions & 0 deletions bindings/dotnet/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
"services-minio",
"services-obs",
"services-oss",
"services-r2",
"services-s3",
"services-tos",
"services-webdav",
Expand Down
105 changes: 105 additions & 0 deletions bindings/dotnet/OpenDAL/ServiceConfig/MinioServiceConfig.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs.

using OpenDAL.ServiceConfig.Abstractions;

namespace OpenDAL.ServiceConfig
{
/// <summary>
/// Configuration for service minio.
/// </summary>
public sealed class MinioServiceConfig : IServiceConfig
{
/// <summary>
/// Access key ID. Set this field together with `secret_access_key`.
/// </summary>
public string? AccessKeyId { get; init; }
/// <summary>
/// Bucket name. This field is required.
/// </summary>
public string? Bucket { get; init; }
/// <summary>
/// MinIO endpoint. This field is required because MinIO deployments do not share a universal endpoint.
/// </summary>
public string? Endpoint { get; init; }
/// <summary>
/// Signing region. The default is `auto`. Set this field when the deployment requires a configured region.
/// </summary>
public string? Region { get; init; }
/// <summary>
/// Root within the bucket. All operations happen under this root. The default is `/`.
/// </summary>
public string? Root { get; init; }
/// <summary>
/// Secret access key. Set this field together with `access_key_id`.
/// </summary>
public string? SecretAccessKey { get; init; }
/// <summary>
/// Session token for temporary credentials. This field requires `access_key_id` and `secret_access_key`.
/// </summary>
public string? SessionToken { get; init; }
/// <summary>
/// Send requests without signing them. This option cannot be combined with direct credentials.
/// </summary>
public bool? SkipSignature { get; init; }

public string Scheme => "minio";

public IReadOnlyDictionary<string, string> ToOptions()
{
var map = new Dictionary<string, string>();
if (AccessKeyId is not null)
{
map["access_key_id"] = Utilities.ToOptionString(AccessKeyId);
}
if (Bucket is not null)
{
map["bucket"] = Utilities.ToOptionString(Bucket);
}
if (Endpoint is not null)
{
map["endpoint"] = Utilities.ToOptionString(Endpoint);
}
if (Region is not null)
{
map["region"] = Utilities.ToOptionString(Region);
}
if (Root is not null)
{
map["root"] = Utilities.ToOptionString(Root);
}
if (SecretAccessKey is not null)
{
map["secret_access_key"] = Utilities.ToOptionString(SecretAccessKey);
}
if (SessionToken is not null)
{
map["session_token"] = Utilities.ToOptionString(SessionToken);
}
if (SkipSignature is not null)
{
map["skip_signature"] = Utilities.ToOptionString(SkipSignature);
}
return map;
}
}

}
Loading
Loading