diff --git a/src/viam/api/CMakeLists.txt b/src/viam/api/CMakeLists.txt index c61854587..747831a39 100644 --- a/src/viam/api/CMakeLists.txt +++ b/src/viam/api/CMakeLists.txt @@ -261,6 +261,10 @@ if (VIAMCPPSDK_USE_DYNAMIC_PROTOS) ${PROTO_GEN_DIR}/service/slam/v1/slam.grpc.pb.h ${PROTO_GEN_DIR}/service/slam/v1/slam.pb.cc ${PROTO_GEN_DIR}/service/slam/v1/slam.pb.h + ${PROTO_GEN_DIR}/service/vision/v1/vision.grpc.pb.cc + ${PROTO_GEN_DIR}/service/vision/v1/vision.grpc.pb.h + ${PROTO_GEN_DIR}/service/vision/v1/vision.pb.cc + ${PROTO_GEN_DIR}/service/vision/v1/vision.pb.h ${PROTO_GEN_DIR}/tagger/v1/tagger.grpc.pb.cc ${PROTO_GEN_DIR}/tagger/v1/tagger.grpc.pb.h ${PROTO_GEN_DIR}/tagger/v1/tagger.pb.cc @@ -381,6 +385,8 @@ target_sources(viamapi ${PROTO_GEN_DIR}/service/navigation/v1/navigation.pb.cc ${PROTO_GEN_DIR}/service/slam/v1/slam.grpc.pb.cc ${PROTO_GEN_DIR}/service/slam/v1/slam.pb.cc + ${PROTO_GEN_DIR}/service/vision/v1/vision.grpc.pb.cc + ${PROTO_GEN_DIR}/service/vision/v1/vision.pb.cc ${PROTO_GEN_DIR}/tagger/v1/tagger.grpc.pb.cc ${PROTO_GEN_DIR}/tagger/v1/tagger.pb.cc PUBLIC FILE_SET viamapi_includes TYPE HEADERS @@ -454,6 +460,8 @@ target_sources(viamapi ${PROTO_GEN_DIR}/../../viam/api/service/navigation/v1/navigation.pb.h ${PROTO_GEN_DIR}/../../viam/api/service/slam/v1/slam.grpc.pb.h ${PROTO_GEN_DIR}/../../viam/api/service/slam/v1/slam.pb.h + ${PROTO_GEN_DIR}/../../viam/api/service/vision/v1/vision.grpc.pb.h + ${PROTO_GEN_DIR}/../../viam/api/service/vision/v1/vision.pb.h ${PROTO_GEN_DIR}/../../viam/api/tagger/v1/tagger.pb.h ) diff --git a/src/viam/sdk/CMakeLists.txt b/src/viam/sdk/CMakeLists.txt index 5874e4c62..bf91dcb71 100644 --- a/src/viam/sdk/CMakeLists.txt +++ b/src/viam/sdk/CMakeLists.txt @@ -70,6 +70,7 @@ target_sources(viamsdk common/utils.cpp common/version_metadata.cpp common/world_state.cpp + common/private/raw_image.cpp common/private/service_helper.cpp tracing/private/span_guard.cpp tracing/private/tracer.cpp @@ -154,6 +155,7 @@ target_sources(viamsdk services/mlmodel.cpp services/motion.cpp services/navigation.cpp + services/vision.cpp services/private/discovery_client.cpp services/private/discovery_server.cpp services/private/generic_client.cpp @@ -165,6 +167,9 @@ target_sources(viamsdk services/private/motion_server.cpp services/private/navigation_client.cpp services/private/navigation_server.cpp + services/private/vision.cpp + services/private/vision_client.cpp + services/private/vision_server.cpp services/service.cpp spatialmath/geometry.cpp spatialmath/orientation.cpp @@ -234,6 +239,7 @@ target_sources(viamsdk ../../viam/sdk/services/mlmodel.hpp ../../viam/sdk/services/motion.hpp ../../viam/sdk/services/navigation.hpp + ../../viam/sdk/services/vision.hpp ../../viam/sdk/services/service.hpp ../../viam/sdk/spatialmath/geometry.hpp ../../viam/sdk/spatialmath/orientation.hpp diff --git a/src/viam/sdk/common/private/raw_image.cpp b/src/viam/sdk/common/private/raw_image.cpp new file mode 100644 index 000000000..d19e379ee --- /dev/null +++ b/src/viam/sdk/common/private/raw_image.cpp @@ -0,0 +1,28 @@ +#include + +#include + +namespace viam { +namespace sdk { +namespace impl { + +Camera::raw_image from_proto(const ::viam::component::camera::v1::Image& proto) { + Camera::raw_image raw_image; + std::string img_string = proto.image(); + const std::vector bytes(img_string.begin(), img_string.end()); + raw_image.bytes = bytes; + raw_image.mime_type = proto.mime_type(); + raw_image.source_name = proto.source_name(); + return raw_image; +} + +void to_proto(const Camera::raw_image& image, ::viam::component::camera::v1::Image* out) { + const std::string img_string = bytes_to_string(image.bytes); + out->set_source_name(image.source_name); + out->set_mime_type(image.mime_type); + out->set_image(img_string); +} + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/common/private/raw_image.hpp b/src/viam/sdk/common/private/raw_image.hpp new file mode 100644 index 000000000..1abd283af --- /dev/null +++ b/src/viam/sdk/common/private/raw_image.hpp @@ -0,0 +1,21 @@ +/// @file common/private/raw_image.hpp +/// +/// @brief Proto conversion helpers for Camera::raw_image and Image proto. +#pragma once + +#include +#include + +namespace viam { +namespace sdk { +namespace impl { + +/// @brief Convert a proto Image to Camera::raw_image. +Camera::raw_image from_proto(const ::viam::component::camera::v1::Image& proto); + +/// @brief Convert a Camera::raw_image to proto Image. +void to_proto(const Camera::raw_image& image, ::viam::component::camera::v1::Image* out); + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/components/private/camera_client.cpp b/src/viam/sdk/components/private/camera_client.cpp index b3ff6e56b..0765d1d57 100644 --- a/src/viam/sdk/components/private/camera_client.cpp +++ b/src/viam/sdk/components/private/camera_client.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -27,13 +28,7 @@ Camera::image_collection from_proto(const viam::component::camera::v1::GetImages Camera::image_collection image_collection; std::vector images; for (const auto& img : proto.images()) { - Camera::raw_image raw_image; - std::string img_string = img.image(); - const std::vector bytes(img_string.begin(), img_string.end()); - raw_image.bytes = bytes; - raw_image.mime_type = img.mime_type(); - raw_image.source_name = img.source_name(); - images.push_back(raw_image); + images.push_back(impl::from_proto(img)); } image_collection.images = std::move(images); image_collection.metadata = from_proto(proto.response_metadata()); diff --git a/src/viam/sdk/components/private/camera_server.cpp b/src/viam/sdk/components/private/camera_server.cpp index 6a80cbf4e..4013c99a7 100644 --- a/src/viam/sdk/components/private/camera_server.cpp +++ b/src/viam/sdk/components/private/camera_server.cpp @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -55,8 +56,8 @@ ::grpc::Status CameraServer::DoCommand(::grpc::ServerContext* context, ::viam::common::v1::DoCommandResponse* response) noexcept { return make_service_helper( "CameraServer::DoCommand", this, context, request)([&](auto&, auto& camera) { - const ProtoStruct result = camera->do_command(from_proto(request->command())); - *response->mutable_result() = to_proto(result); + const ProtoStruct result = camera->do_command(sdk::from_proto(request->command())); + *response->mutable_result() = sdk::to_proto(result); }); } @@ -70,14 +71,9 @@ ::grpc::Status CameraServer::GetImages( {request->filter_source_names().begin(), request->filter_source_names().end()}, helper.getExtra()); for (const auto& img : image_coll.images) { - ::viam::component::camera::v1::Image proto_image; - const std::string img_string = bytes_to_string(img.bytes); - proto_image.set_source_name(img.source_name); - proto_image.set_mime_type(img.mime_type); - proto_image.set_image(img_string); - *response->mutable_images()->Add() = std::move(proto_image); + impl::to_proto(img, response->mutable_images()->Add()); } - *response->mutable_response_metadata() = to_proto(image_coll.metadata); + *response->mutable_response_metadata() = sdk::to_proto(image_coll.metadata); }); } @@ -102,7 +98,7 @@ ::grpc::Status CameraServer::GetGeometries( "CameraServer::GetGeometries", this, context, request)([&](auto& helper, auto& camera) { const std::vector geometries = camera->get_geometries(helper.getExtra()); for (const auto& geometry : geometries) { - *response->mutable_geometries()->Add() = to_proto(geometry); + *response->mutable_geometries()->Add() = sdk::to_proto(geometry); } }); } @@ -129,7 +125,7 @@ ::grpc::Status CameraServer::GetStatus(::grpc::ServerContext* context, return make_service_helper( "CameraServer::GetStatus", this, context, request)([&](auto&, auto& camera) { const ProtoStruct result = camera->get_status(); - *response->mutable_result() = to_proto(result); + *response->mutable_result() = sdk::to_proto(result); }); } diff --git a/src/viam/sdk/registry/registry.cpp b/src/viam/sdk/registry/registry.cpp index 498fe3ad4..7e4bf63ab 100644 --- a/src/viam/sdk/registry/registry.cpp +++ b/src/viam/sdk/registry/registry.cpp @@ -60,6 +60,8 @@ #include #include #include +#include +#include #include namespace viam { @@ -229,6 +231,7 @@ void Registry::register_resources() { register_resource(); register_resource(); register_resource(); + register_resource(); } void Registry::initialize() { diff --git a/src/viam/sdk/services/private/vision.cpp b/src/viam/sdk/services/private/vision.cpp new file mode 100644 index 000000000..4256aaf7b --- /dev/null +++ b/src/viam/sdk/services/private/vision.cpp @@ -0,0 +1,130 @@ +// Copyright 2024 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include + +namespace viam { +namespace sdk { +namespace impl { +namespace vision { + +namespace vpb = ::viam::service::vision::v1; + +vpb::Detection to_proto(const Vision::detection& d) { + vpb::Detection out; + if (d.x_min) { + out.set_x_min(*d.x_min); + } + if (d.y_min) { + out.set_y_min(*d.y_min); + } + if (d.x_max) { + out.set_x_max(*d.x_max); + } + if (d.y_max) { + out.set_y_max(*d.y_max); + } + if (d.x_min_normalized) { + out.set_x_min_normalized(*d.x_min_normalized); + } + if (d.y_min_normalized) { + out.set_y_min_normalized(*d.y_min_normalized); + } + if (d.x_max_normalized) { + out.set_x_max_normalized(*d.x_max_normalized); + } + if (d.y_max_normalized) { + out.set_y_max_normalized(*d.y_max_normalized); + } + out.set_class_name(d.class_name); + out.set_confidence(d.confidence); + return out; +} + +Vision::detection from_proto(const vpb::Detection& p) { + Vision::detection out; + if (p.has_x_min()) { + out.x_min = p.x_min(); + } + if (p.has_y_min()) { + out.y_min = p.y_min(); + } + if (p.has_x_max()) { + out.x_max = p.x_max(); + } + if (p.has_y_max()) { + out.y_max = p.y_max(); + } + if (p.has_x_min_normalized()) { + out.x_min_normalized = p.x_min_normalized(); + } + if (p.has_y_min_normalized()) { + out.y_min_normalized = p.y_min_normalized(); + } + if (p.has_x_max_normalized()) { + out.x_max_normalized = p.x_max_normalized(); + } + if (p.has_y_max_normalized()) { + out.y_max_normalized = p.y_max_normalized(); + } + out.class_name = p.class_name(); + out.confidence = p.confidence(); + return out; +} + +vpb::Classification to_proto(const Vision::classification& c) { + vpb::Classification out; + out.set_class_name(c.class_name); + out.set_confidence(c.confidence); + return out; +} + +Vision::classification from_proto(const vpb::Classification& p) { + return Vision::classification{p.class_name(), p.confidence()}; +} + +void to_proto(const Vision::point_cloud_object& o, ::viam::common::v1::PointCloudObject* out) { + out->set_point_cloud( + std::string(reinterpret_cast(o.cloud.pc.data()), o.cloud.pc.size())); + *(out->mutable_geometries()->mutable_geometries()) = impl::to_repeated_field(o.geometries); +} + +Vision::point_cloud_object from_proto(const ::viam::common::v1::PointCloudObject& p) { + Vision::point_cloud_object out; + const auto& bytes = p.point_cloud(); + out.cloud.pc.assign(reinterpret_cast(bytes.data()), + reinterpret_cast(bytes.data()) + bytes.size()); + out.geometries = impl::from_repeated_field(p.geometries().geometries()); + return out; +} + +void to_proto(const Vision::properties& props, vpb::GetPropertiesResponse* out) { + out->set_classifications_supported(props.classifications_supported); + out->set_detections_supported(props.detections_supported); + out->set_object_point_clouds_supported(props.object_point_clouds_supported); +} + +Vision::properties from_proto(const vpb::GetPropertiesResponse& p) { + return Vision::properties{ + p.classifications_supported(), p.detections_supported(), p.object_point_clouds_supported()}; +} + +} // namespace vision +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/services/private/vision.hpp b/src/viam/sdk/services/private/vision.hpp new file mode 100644 index 000000000..bdea980b9 --- /dev/null +++ b/src/viam/sdk/services/private/vision.hpp @@ -0,0 +1,41 @@ +// Copyright 2024 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace viam { +namespace sdk { +namespace impl { +namespace vision { + +::viam::service::vision::v1::Detection to_proto(const Vision::detection&); +Vision::detection from_proto(const ::viam::service::vision::v1::Detection&); + +::viam::service::vision::v1::Classification to_proto(const Vision::classification&); +Vision::classification from_proto(const ::viam::service::vision::v1::Classification&); + +void to_proto(const Vision::point_cloud_object&, ::viam::common::v1::PointCloudObject* out); +Vision::point_cloud_object from_proto(const ::viam::common::v1::PointCloudObject&); + +void to_proto(const Vision::properties&, ::viam::service::vision::v1::GetPropertiesResponse* out); +Vision::properties from_proto(const ::viam::service::vision::v1::GetPropertiesResponse&); + +} // namespace vision +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/services/private/vision_client.cpp b/src/viam/sdk/services/private/vision_client.cpp new file mode 100644 index 000000000..dfec8f1ff --- /dev/null +++ b/src/viam/sdk/services/private/vision_client.cpp @@ -0,0 +1,179 @@ +// Copyright 2023 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include + +#include +#include +#include + +namespace viam { +namespace sdk { +namespace impl { + +VisionClient::VisionClient(std::string name, const ViamChannel& channel) + : Vision(std::move(name)), + stub_(service_type::NewStub(channel.channel())), + channel_(&channel) {} + +std::vector VisionClient::get_detections_from_camera( + const std::string& camera_name, const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &service_type::StubInterface::GetDetectionsFromCamera) + .with(extra, [&](auto& req) { req.set_camera_name(camera_name); }) + .invoke([](auto& response) { + std::vector out; + out.reserve(response.detections_size()); + for (const auto& d : response.detections()) { + out.push_back(impl::vision::from_proto(d)); + } + return out; + }); +} + +std::vector VisionClient::get_detections(const Vision::image& img, + const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &service_type::StubInterface::GetDetections) + .with(extra, + [&](auto& req) { + req.set_image(std::string(img.bytes.begin(), img.bytes.end())); + req.set_mime_type(img.mime_type); + }) + .invoke([](auto& response) { + std::vector out; + out.reserve(response.detections_size()); + for (const auto& d : response.detections()) { + out.push_back(impl::vision::from_proto(d)); + } + return out; + }); +} + +std::vector VisionClient::get_classifications_from_camera( + const std::string& camera_name, int count, const ProtoStruct& extra) { + return make_client_helper( + this, *stub_, &service_type::StubInterface::GetClassificationsFromCamera) + .with(extra, + [&](auto& req) { + req.set_camera_name(camera_name); + req.set_n(count); + }) + .invoke([](auto& response) { + std::vector out; + out.reserve(response.classifications_size()); + for (const auto& c : response.classifications()) { + out.push_back(impl::vision::from_proto(c)); + } + return out; + }); +} + +std::vector VisionClient::get_classifications(const Vision::image& img, + int count, + const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &service_type::StubInterface::GetClassifications) + .with(extra, + [&](auto& req) { + req.set_image(std::string(img.bytes.begin(), img.bytes.end())); + req.set_mime_type(img.mime_type); + req.set_n(count); + }) + .invoke([](auto& response) { + std::vector out; + out.reserve(response.classifications_size()); + for (const auto& c : response.classifications()) { + out.push_back(impl::vision::from_proto(c)); + } + return out; + }); +} + +std::vector VisionClient::get_object_point_clouds( + const std::string& camera_name, const std::string& mime_type, const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &service_type::StubInterface::GetObjectPointClouds) + .with(extra, + [&](auto& req) { + req.set_camera_name(camera_name); + req.set_mime_type(mime_type); + }) + .invoke([](auto& response) { + std::vector out; + out.reserve(response.objects_size()); + // The wire format carries one mime_type for all objects; copy it + // onto each returned object. May differ from the requested mime — + // see GetObjectPointCloudsResponse.mime_type in vision.proto. + for (const auto& proto_obj : response.objects()) { + auto pco = impl::vision::from_proto(proto_obj); + pco.cloud.mime_type = response.mime_type(); + out.push_back(std::move(pco)); + } + return out; + }); +} + +Vision::properties VisionClient::get_properties(const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &service_type::StubInterface::GetProperties) + .with(extra) + .invoke([](auto& response) { return impl::vision::from_proto(response); }); +} + +Vision::capture_all_result VisionClient::capture_all_from_camera( + const std::string& camera_name, const Vision::capture_options& opts, const ProtoStruct& extra) { + return make_client_helper(this, *stub_, &service_type::StubInterface::CaptureAllFromCamera) + .with(extra, + [&](auto& req) { + req.set_camera_name(camera_name); + req.set_return_image(opts.return_image); + req.set_return_detections(opts.return_detections); + req.set_return_classifications(opts.return_classifications); + req.set_return_object_point_clouds(opts.return_object_point_clouds); + }) + .invoke([](auto& response) { + Vision::capture_all_result out; + if (response.has_image()) { + out.image = impl::from_proto(response.image()); + } + for (const auto& d : response.detections()) { + out.detections.push_back(impl::vision::from_proto(d)); + } + for (const auto& c : response.classifications()) { + out.classifications.push_back(impl::vision::from_proto(c)); + } + for (const auto& o : response.objects()) { + out.objects.push_back(impl::vision::from_proto(o)); + } + if (response.has_extra()) { + out.extra = sdk::from_proto(response.extra()); + } + return out; + }); +} + +ProtoStruct VisionClient::do_command(const ProtoStruct& command) { + return make_client_helper(this, *stub_, &service_type::StubInterface::DoCommand) + .with([&](auto& request) { *request.mutable_command() = to_proto(command); }) + .invoke([](auto& response) { return sdk::from_proto(response.result()); }); +} + +ProtoStruct VisionClient::get_status() { + return make_client_helper(this, *stub_, &service_type::StubInterface::GetStatus) + .invoke([](auto& response) { return sdk::from_proto(response.result()); }); +} + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/services/private/vision_client.hpp b/src/viam/sdk/services/private/vision_client.hpp new file mode 100644 index 000000000..7a5a2ad08 --- /dev/null +++ b/src/viam/sdk/services/private/vision_client.hpp @@ -0,0 +1,72 @@ +// Copyright 2023 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include + +namespace viam { +namespace sdk { +namespace impl { + +class VisionClient : public Vision { + public: + using interface_type = Vision; + using service_type = ::viam::service::vision::v1::VisionService; + + VisionClient(std::string name, const ViamChannel& channel); + + const ViamChannel& channel() const { + return *channel_; + } + + std::vector get_detections_from_camera(const std::string& camera_name, + const ProtoStruct& extra) override; + std::vector get_detections(const Vision::image& img, + const ProtoStruct& extra) override; + std::vector get_classifications_from_camera( + const std::string& camera_name, int count, const ProtoStruct& extra) override; + std::vector get_classifications(const Vision::image& img, + int count, + const ProtoStruct& extra) override; + std::vector get_object_point_clouds( + const std::string& camera_name, + const std::string& mime_type, + const ProtoStruct& extra) override; + Vision::properties get_properties(const ProtoStruct& extra) override; + Vision::capture_all_result capture_all_from_camera(const std::string& camera_name, + const Vision::capture_options& opts, + const ProtoStruct& extra) override; + ProtoStruct do_command(const ProtoStruct& command) override; + ProtoStruct get_status() override; + + using Vision::capture_all_from_camera; + using Vision::get_classifications; + using Vision::get_classifications_from_camera; + using Vision::get_detections; + using Vision::get_detections_from_camera; + using Vision::get_object_point_clouds; + using Vision::get_properties; + + private: + std::unique_ptr stub_; + const ViamChannel* channel_; +}; + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/services/private/vision_server.cpp b/src/viam/sdk/services/private/vision_server.cpp new file mode 100644 index 000000000..c55394416 --- /dev/null +++ b/src/viam/sdk/services/private/vision_server.cpp @@ -0,0 +1,176 @@ +// Copyright 2023 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include + +namespace viam { +namespace sdk { +namespace impl { + +VisionServer::VisionServer(std::shared_ptr manager) + : ResourceServer(std::move(manager)) {} + +::grpc::Status VisionServer::GetDetectionsFromCamera( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetDetectionsFromCameraRequest* request, + ::viam::service::vision::v1::GetDetectionsFromCameraResponse* response) noexcept { + return make_service_helper( + "VisionServer::GetDetectionsFromCamera", this, context, request)( + [&](auto& helper, auto& vs) { + const auto results = + vs->get_detections_from_camera(request->camera_name(), helper.getExtra()); + for (const auto& d : results) { + *response->add_detections() = impl::vision::to_proto(d); + } + }); +} + +::grpc::Status VisionServer::GetDetections( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetDetectionsRequest* request, + ::viam::service::vision::v1::GetDetectionsResponse* response) noexcept { + return make_service_helper( + "VisionServer::GetDetections", this, context, request)([&](auto& helper, auto& vs) { + Vision::image img; + img.bytes.assign(request->image().begin(), request->image().end()); + img.mime_type = request->mime_type(); + const auto results = vs->get_detections(img, helper.getExtra()); + for (const auto& d : results) { + *response->add_detections() = impl::vision::to_proto(d); + } + }); +} + +::grpc::Status VisionServer::GetClassificationsFromCamera( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetClassificationsFromCameraRequest* request, + ::viam::service::vision::v1::GetClassificationsFromCameraResponse* response) noexcept { + return make_service_helper( + "VisionServer::GetClassificationsFromCamera", this, context, request)( + [&](auto& helper, auto& vs) { + const auto results = vs->get_classifications_from_camera( + request->camera_name(), request->n(), helper.getExtra()); + for (const auto& c : results) { + *response->add_classifications() = impl::vision::to_proto(c); + } + }); +} + +::grpc::Status VisionServer::GetClassifications( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetClassificationsRequest* request, + ::viam::service::vision::v1::GetClassificationsResponse* response) noexcept { + return make_service_helper( + "VisionServer::GetClassifications", this, context, request)([&](auto& helper, auto& vs) { + Vision::image img; + img.bytes.assign(request->image().begin(), request->image().end()); + img.mime_type = request->mime_type(); + const auto results = vs->get_classifications(img, request->n(), helper.getExtra()); + for (const auto& c : results) { + *response->add_classifications() = impl::vision::to_proto(c); + } + }); +} + +::grpc::Status VisionServer::GetObjectPointClouds( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetObjectPointCloudsRequest* request, + ::viam::service::vision::v1::GetObjectPointCloudsResponse* response) noexcept { + return make_service_helper( + "VisionServer::GetObjectPointClouds", this, context, request)([&](auto& helper, auto& vs) { + const auto results = vs->get_object_point_clouds( + request->camera_name(), request->mime_type(), helper.getExtra()); + for (const auto& obj : results) { + impl::vision::to_proto(obj, response->add_objects()); + } + // The wire format carries a single mime_type for all objects; take it + // from the first result (the server may legitimately convert formats + // and return something different from request->mime_type()). + if (!results.empty()) { + response->set_mime_type(results.front().cloud.mime_type); + } + }); +} + +::grpc::Status VisionServer::GetProperties( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetPropertiesRequest* request, + ::viam::service::vision::v1::GetPropertiesResponse* response) noexcept { + return make_service_helper( + "VisionServer::GetProperties", this, context, request)([&](auto& helper, auto& vs) { + const auto props = vs->get_properties(helper.getExtra()); + impl::vision::to_proto(props, response); + }); +} + +::grpc::Status VisionServer::CaptureAllFromCamera( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::CaptureAllFromCameraRequest* request, + ::viam::service::vision::v1::CaptureAllFromCameraResponse* response) noexcept { + return make_service_helper( + "VisionServer::CaptureAllFromCamera", this, context, request)([&](auto& helper, auto& vs) { + Vision::capture_options opts; + opts.return_image = request->return_image(); + opts.return_detections = request->return_detections(); + opts.return_classifications = request->return_classifications(); + opts.return_object_point_clouds = request->return_object_point_clouds(); + + const auto result = + vs->capture_all_from_camera(request->camera_name(), opts, helper.getExtra()); + + if (result.image) { + impl::to_proto(*result.image, response->mutable_image()); + } + for (const auto& d : result.detections) { + *response->add_detections() = impl::vision::to_proto(d); + } + for (const auto& c : result.classifications) { + *response->add_classifications() = impl::vision::to_proto(c); + } + for (const auto& o : result.objects) { + impl::vision::to_proto(o, response->add_objects()); + } + if (!result.extra.empty()) { + *response->mutable_extra() = to_proto(result.extra); + } + }); +} + +::grpc::Status VisionServer::DoCommand(::grpc::ServerContext* context, + const ::viam::common::v1::DoCommandRequest* request, + ::viam::common::v1::DoCommandResponse* response) noexcept { + return make_service_helper( + "VisionServer::DoCommand", this, context, request)([&](auto&, auto& vs) { + const ProtoStruct result = vs->do_command(sdk::from_proto(request->command())); + *response->mutable_result() = to_proto(result); + }); +} + +::grpc::Status VisionServer::GetStatus(::grpc::ServerContext* context, + const ::viam::common::v1::GetStatusRequest* request, + ::viam::common::v1::GetStatusResponse* response) noexcept { + return make_service_helper( + "VisionServer::GetStatus", this, context, request)([&](auto&, auto& vs) { + const ProtoStruct result = vs->get_status(); + *response->mutable_result() = to_proto(result); + }); +} + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/services/private/vision_server.hpp b/src/viam/sdk/services/private/vision_server.hpp new file mode 100644 index 000000000..ec3b462ff --- /dev/null +++ b/src/viam/sdk/services/private/vision_server.hpp @@ -0,0 +1,80 @@ +// Copyright 2023 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include + +namespace viam { +namespace sdk { +namespace impl { + +class VisionServer : public ResourceServer, + public ::viam::service::vision::v1::VisionService::Service { + public: + using interface_type = Vision; + using service_type = service::vision::v1::VisionService; + explicit VisionServer(std::shared_ptr manager); + + ::grpc::Status GetDetectionsFromCamera( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetDetectionsFromCameraRequest* request, + ::viam::service::vision::v1::GetDetectionsFromCameraResponse* response) noexcept override; + + ::grpc::Status GetDetections( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetDetectionsRequest* request, + ::viam::service::vision::v1::GetDetectionsResponse* response) noexcept override; + + ::grpc::Status GetClassificationsFromCamera( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetClassificationsFromCameraRequest* request, + ::viam::service::vision::v1::GetClassificationsFromCameraResponse* response) noexcept + override; + + ::grpc::Status GetClassifications( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetClassificationsRequest* request, + ::viam::service::vision::v1::GetClassificationsResponse* response) noexcept override; + + ::grpc::Status GetObjectPointClouds( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetObjectPointCloudsRequest* request, + ::viam::service::vision::v1::GetObjectPointCloudsResponse* response) noexcept override; + + ::grpc::Status GetProperties( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::GetPropertiesRequest* request, + ::viam::service::vision::v1::GetPropertiesResponse* response) noexcept override; + + ::grpc::Status CaptureAllFromCamera( + ::grpc::ServerContext* context, + const ::viam::service::vision::v1::CaptureAllFromCameraRequest* request, + ::viam::service::vision::v1::CaptureAllFromCameraResponse* response) noexcept override; + + ::grpc::Status DoCommand(::grpc::ServerContext* context, + const ::viam::common::v1::DoCommandRequest* request, + ::viam::common::v1::DoCommandResponse* response) noexcept override; + + ::grpc::Status GetStatus(::grpc::ServerContext* context, + const ::viam::common::v1::GetStatusRequest* request, + ::viam::common::v1::GetStatusResponse* response) noexcept override; +}; + +} // namespace impl +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/services/vision.cpp b/src/viam/sdk/services/vision.cpp new file mode 100644 index 000000000..9f317763a --- /dev/null +++ b/src/viam/sdk/services/vision.cpp @@ -0,0 +1,56 @@ +// Copyright 2024 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +namespace viam { +namespace sdk { + +API Vision::api() const { + return API::get(); +} + +API API::traits::api() { + return API(kRDK, kService, "vision"); +} + +Vision::Vision(std::string name) : Service(std::move(name)) {} + +bool operator==(const Vision::detection& lhs, const Vision::detection& rhs) { + return lhs.x_min == rhs.x_min && lhs.y_min == rhs.y_min && lhs.x_max == rhs.x_max && + lhs.y_max == rhs.y_max && lhs.class_name == rhs.class_name && + lhs.confidence == rhs.confidence && lhs.x_min_normalized == rhs.x_min_normalized && + lhs.y_min_normalized == rhs.y_min_normalized && + lhs.x_max_normalized == rhs.x_max_normalized && + lhs.y_max_normalized == rhs.y_max_normalized; +} + +bool operator==(const Vision::classification& lhs, const Vision::classification& rhs) { + return lhs.class_name == rhs.class_name && lhs.confidence == rhs.confidence; +} + +bool operator==(const Vision::point_cloud_object& lhs, const Vision::point_cloud_object& rhs) { + return lhs.cloud == rhs.cloud && lhs.geometries == rhs.geometries; +} + +bool operator==(const Vision::properties& lhs, const Vision::properties& rhs) { + return lhs.classifications_supported == rhs.classifications_supported && + lhs.detections_supported == rhs.detections_supported && + lhs.object_point_clouds_supported == rhs.object_point_clouds_supported; +} + +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/services/vision.hpp b/src/viam/sdk/services/vision.hpp new file mode 100644 index 000000000..75ae76984 --- /dev/null +++ b/src/viam/sdk/services/vision.hpp @@ -0,0 +1,211 @@ +/// @file services/vision.hpp +/// +/// @brief Defines a `Vision` service. +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include + +namespace viam { +namespace sdk { + +/// @class Vision vision.hpp "services/vision.hpp" +/// @brief A vision service that runs detectors / classifiers / segmenters. +/// +/// This acts as an abstract base class to be inherited from by any drivers representing +/// specific vision implementations. This class cannot be used on its own. +class Vision : public Service { + public: + API api() const override; + + /// @struct detection + /// @brief Result of a single object detection. + /// Pixel bbox fields are optional int64; normalized bbox fields are optional double. + struct detection { + boost::optional x_min; + boost::optional y_min; + boost::optional x_max; + boost::optional y_max; + boost::optional x_min_normalized; + boost::optional y_min_normalized; + boost::optional x_max_normalized; + boost::optional y_max_normalized; + std::string class_name; + double confidence = 0.0; + }; + + /// @struct classification + /// @brief Result of a single image classification. + struct classification { + std::string class_name; + double confidence = 0.0; + }; + + using point_cloud = Camera::point_cloud; + using raw_image = Camera::raw_image; + + /// @struct image + /// @brief Image input to `get_detections` / `get_classifications`. + /// + /// Distinct from `raw_image` (a.k.a. `Camera::raw_image`) because the Vision request + /// messages on the wire only carry bytes + mime_type — they do not carry a `source_name`. + /// See the proto definitions for `GetDetectionsRequest` and `GetClassificationsRequest`. + /// The `raw_image` alias remains for `capture_all_result.image`, whose wire type is the + /// full Camera Image proto and does carry a source name. + struct image { + std::string mime_type; + std::vector bytes; + }; + + /// @struct point_cloud_object + /// @brief A point cloud and its associated geometry. + /// @note The proto field is named `point_cloud`; we use `cloud` here to avoid shadowing + /// the `point_cloud` type alias (GCC rejects the shadowing under -fpermissive). + struct point_cloud_object { + point_cloud cloud; + std::vector geometries; + }; + + /// @struct properties + /// @brief Describes the vision service's supported capabilities. + struct properties { + bool classifications_supported = false; + bool detections_supported = false; + bool object_point_clouds_supported = false; + }; + + /// @struct capture_options + /// @brief Controls which data types are returned by `capture_all_from_camera`. + struct capture_options { + bool return_image = false; + bool return_detections = false; + bool return_classifications = false; + bool return_object_point_clouds = false; + }; + + /// @struct capture_all_result + /// @brief Aggregate result returned by `capture_all_from_camera`. + struct capture_all_result { + boost::optional image; + std::vector detections; + std::vector classifications; + std::vector objects; + ProtoStruct extra; + }; + + /// @brief Get detections from a named camera's next image. + inline std::vector get_detections_from_camera(const std::string& camera_name) { + return get_detections_from_camera(camera_name, {}); + } + + /// @brief Get detections from a named camera's next image. + /// @param extra Any additional arguments to the method. + virtual std::vector get_detections_from_camera(const std::string& camera_name, + const ProtoStruct& extra) = 0; + + /// @brief Get detections from an image. + inline std::vector get_detections(const image& img) { + return get_detections(img, {}); + } + + /// @brief Get detections from an image. + /// @param extra Any additional arguments to the method. + virtual std::vector get_detections(const image& img, const ProtoStruct& extra) = 0; + + /// @brief Get classifications from a named camera's next image. + /// @param count The number of classifications to return. + inline std::vector get_classifications_from_camera( + const std::string& camera_name, int count) { + return get_classifications_from_camera(camera_name, count, {}); + } + + /// @brief Get classifications from a named camera's next image. + /// @param count The number of classifications to return. + /// @param extra Any additional arguments to the method. + virtual std::vector get_classifications_from_camera( + const std::string& camera_name, int count, const ProtoStruct& extra) = 0; + + /// @brief Get classifications from an image. + /// @param count The number of classifications to return. + inline std::vector get_classifications(const image& img, int count) { + return get_classifications(img, count, {}); + } + + /// @brief Get classifications from an image. + /// @param count The number of classifications to return. + /// @param extra Any additional arguments to the method. + virtual std::vector get_classifications(const image& img, + int count, + const ProtoStruct& extra) = 0; + + /// @brief Get point cloud objects detected by a named camera. + /// @param mime_type The desired mime type of the point cloud (does not guarantee output type). + inline std::vector get_object_point_clouds(const std::string& camera_name, + const std::string& mime_type) { + return get_object_point_clouds(camera_name, mime_type, {}); + } + + /// @brief Get point cloud objects detected by a named camera. + /// @param mime_type The desired mime type of the point cloud (does not guarantee output type). + /// @param extra Any additional arguments to the method. + virtual std::vector get_object_point_clouds(const std::string& camera_name, + const std::string& mime_type, + const ProtoStruct& extra) = 0; + + /// @brief Get the properties (capabilities) of this vision service. + inline struct properties get_properties() { + return get_properties({}); + } + + /// @brief Get the properties (capabilities) of this vision service. + /// @param extra Any additional arguments to the method. + virtual struct properties get_properties(const ProtoStruct& extra) = 0; + + /// @brief Capture image, detections, classifications, and/or point clouds in a single call. + /// @param camera_name The name of the camera to capture from. + /// @param opts Controls which data types to include in the result. + inline capture_all_result capture_all_from_camera(const std::string& camera_name, + const capture_options& opts) { + return capture_all_from_camera(camera_name, opts, {}); + } + + /// @brief Capture image, detections, classifications, and/or point clouds in a single call. + /// @param camera_name The name of the camera to capture from. + /// @param opts Controls which data types to include in the result. + /// @param extra Any additional arguments to the method. + virtual capture_all_result capture_all_from_camera(const std::string& camera_name, + const capture_options& opts, + const ProtoStruct& extra) = 0; + + /// @brief Send/receive arbitrary commands to the resource. + /// @param command The command to execute. + /// @return The result of the executed command. + virtual ProtoStruct do_command(const ProtoStruct& command) = 0; + + /// @brief Get the status of the vision service. + /// @return A `ProtoStruct` containing the status of the vision service. + virtual ProtoStruct get_status() = 0; + + protected: + explicit Vision(std::string name); +}; + +template <> +struct API::traits { + static API api(); +}; + +bool operator==(const Vision::detection& lhs, const Vision::detection& rhs); +bool operator==(const Vision::classification& lhs, const Vision::classification& rhs); +bool operator==(const Vision::point_cloud_object& lhs, const Vision::point_cloud_object& rhs); +bool operator==(const Vision::properties& lhs, const Vision::properties& rhs); + +} // namespace sdk +} // namespace viam diff --git a/src/viam/sdk/tests/CMakeLists.txt b/src/viam/sdk/tests/CMakeLists.txt index fd0f27006..74ec76e3e 100644 --- a/src/viam/sdk/tests/CMakeLists.txt +++ b/src/viam/sdk/tests/CMakeLists.txt @@ -40,6 +40,7 @@ target_sources(viamsdk_test mocks/mock_sensor.cpp mocks/mock_servo.cpp mocks/mock_switch.cpp + mocks/mock_vision.cpp mocks/mock_robot.cpp test_utils.cpp ) @@ -84,4 +85,5 @@ viamcppsdk_add_boost_test(test_resource.cpp) viamcppsdk_add_boost_test(test_sensor.cpp) viamcppsdk_add_boost_test(test_servo.cpp) viamcppsdk_add_boost_test(test_switch.cpp) +viamcppsdk_add_boost_test(test_vision.cpp) viamcppsdk_add_boost_test(test_robot.cpp) diff --git a/src/viam/sdk/tests/mocks/mock_vision.cpp b/src/viam/sdk/tests/mocks/mock_vision.cpp new file mode 100644 index 000000000..678cab104 --- /dev/null +++ b/src/viam/sdk/tests/mocks/mock_vision.cpp @@ -0,0 +1,113 @@ +#include "mock_vision.hpp" + +#include + +namespace viam { +namespace sdktests { +namespace vision { + +using namespace viam::sdk; + +MockVision::MockVision(std::string name) : sdk::Vision(std::move(name)) {} + +std::vector MockVision::get_detections_from_camera( + const std::string& camera_name, const ProtoStruct& extra) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_camera_name = camera_name; + last_extra = extra; + return canned_detections; +} + +std::vector MockVision::get_detections(const Vision::image& img, + const ProtoStruct& extra) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_image = img; + last_extra = extra; + return canned_detections; +} + +std::vector MockVision::get_classifications_from_camera( + const std::string& camera_name, int count, const ProtoStruct& extra) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_camera_name = camera_name; + last_count = count; + last_extra = extra; + return canned_classifications; +} + +std::vector MockVision::get_classifications(const Vision::image& img, + int count, + const ProtoStruct& extra) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_image = img; + last_count = count; + last_extra = extra; + return canned_classifications; +} + +std::vector MockVision::get_object_point_clouds( + const std::string& camera_name, const std::string& mime_type, const ProtoStruct& extra) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_camera_name = camera_name; + last_mime_type = mime_type; + last_extra = extra; + return canned_objects; +} + +Vision::properties MockVision::get_properties(const ProtoStruct& extra) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_extra = extra; + return canned_properties; +} + +Vision::capture_all_result MockVision::capture_all_from_camera(const std::string& camera_name, + const capture_options& opts, + const ProtoStruct& extra) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_camera_name = camera_name; + last_capture_options = opts; + last_extra = extra; + return canned_capture_all; +} + +ProtoStruct MockVision::do_command(const ProtoStruct& command) { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + last_command = command; + return command; +} + +ProtoStruct MockVision::get_status() { + if (throw_on_next_call) { + throw_on_next_call = false; + throw sdk::Exception("mock failure"); + } + return canned_status; +} + +} // namespace vision +} // namespace sdktests +} // namespace viam diff --git a/src/viam/sdk/tests/mocks/mock_vision.hpp b/src/viam/sdk/tests/mocks/mock_vision.hpp new file mode 100644 index 000000000..ccec79d7a --- /dev/null +++ b/src/viam/sdk/tests/mocks/mock_vision.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include +#include + +#include + +namespace viam { +namespace sdktests { +namespace vision { + +class MockVision : public sdk::Vision { + public: + explicit MockVision(std::string name = "mock-vision"); + + // Canned responses (set per-test). + std::vector canned_detections; + std::vector canned_classifications; + std::vector canned_objects; + sdk::Vision::properties canned_properties{false, false, false}; + sdk::Vision::capture_all_result canned_capture_all; + sdk::ProtoStruct canned_status; + + // Last-call records (used to assert request fields round-trip). + std::string last_camera_name; + int last_count = 0; + std::string last_mime_type; + sdk::Vision::image last_image; + sdk::Vision::capture_options last_capture_options; + sdk::ProtoStruct last_extra; + sdk::ProtoStruct last_command; + + // Pluggable hook for the exception-mapping test. + bool throw_on_next_call = false; + + std::vector get_detections_from_camera( + const std::string& camera_name, const sdk::ProtoStruct& extra) override; + std::vector get_detections(const sdk::Vision::image& img, + const sdk::ProtoStruct& extra) override; + std::vector get_classifications_from_camera( + const std::string& camera_name, int count, const sdk::ProtoStruct& extra) override; + std::vector get_classifications( + const sdk::Vision::image& img, int count, const sdk::ProtoStruct& extra) override; + std::vector get_object_point_clouds( + const std::string& camera_name, + const std::string& mime_type, + const sdk::ProtoStruct& extra) override; + sdk::Vision::properties get_properties(const sdk::ProtoStruct& extra) override; + sdk::Vision::capture_all_result capture_all_from_camera( + const std::string& camera_name, + const sdk::Vision::capture_options& opts, + const sdk::ProtoStruct& extra) override; + sdk::ProtoStruct do_command(const sdk::ProtoStruct& command) override; + sdk::ProtoStruct get_status() override; +}; + +} // namespace vision +} // namespace sdktests +} // namespace viam diff --git a/src/viam/sdk/tests/test_vision.cpp b/src/viam/sdk/tests/test_vision.cpp new file mode 100644 index 000000000..45497c65f --- /dev/null +++ b/src/viam/sdk/tests/test_vision.cpp @@ -0,0 +1,467 @@ +// Copyright 2024 Viam Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define BOOST_TEST_MODULE vision + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vimpl = viam::sdk::impl::vision; +namespace vpb = viam::service::vision::v1; +using viam::sdk::Vision; + +namespace viam { +namespace sdktests { +namespace vision { + +// vision_fixture stands up an in-process gRPC server backed by MockVision and a +// VisionClient connected to it over an InProcessChannel. Per-RPC test suites +// (tasks 8-14) declare this as their fixture so they can reach both the canned +// mock state and the client interface under test. +// Custom struct (rather than test_utils.hpp's client_to_mock_pipeline<>) +// so per-RPC tests can introspect the mock between calls. +struct vision_fixture { + std::shared_ptr mock; + + // Order: channel_ must outlive client (VisionClient holds a raw channel ptr). + std::shared_ptr server_; + TestServer test_server_; + std::unique_ptr channel_; + + std::shared_ptr client; + + vision_fixture() + : mock(std::make_shared("mock-vision")), + server_(std::make_shared()), + test_server_(server_) { + server_->add_resource(mock); + server_->start(); + + channel_ = std::make_unique(test_server_.grpc_in_process_channel()); + + client = std::dynamic_pointer_cast( + sdk::Registry::get() + .lookup_resource_client(sdk::API::get()) + ->create_rpc_client(mock->name(), *channel_)); + + BOOST_REQUIRE(client != nullptr); + } + + ~vision_fixture() { + server_->shutdown(); + } +}; + +} // namespace vision +} // namespace sdktests +} // namespace viam + +BOOST_TEST_DONT_PRINT_LOG_VALUE(Vision::detection) +BOOST_TEST_DONT_PRINT_LOG_VALUE(Vision::classification) +BOOST_TEST_DONT_PRINT_LOG_VALUE(Vision::properties) + +BOOST_AUTO_TEST_SUITE(vision_proto_conv) + +BOOST_AUTO_TEST_CASE(detection_round_trip_pixel_only) { + Vision::detection d; + d.x_min = 1; + d.y_min = 2; + d.x_max = 3; + d.y_max = 4; + d.class_name = "cat"; + d.confidence = 0.9; + + const vpb::Detection proto = vimpl::to_proto(d); + const Vision::detection back = vimpl::from_proto(proto); + + BOOST_TEST(back == d); + BOOST_TEST(!back.x_min_normalized.has_value()); +} + +BOOST_AUTO_TEST_CASE(detection_round_trip_normalized_only) { + Vision::detection d; + d.x_min_normalized = 0.1; + d.y_min_normalized = 0.2; + d.x_max_normalized = 0.3; + d.y_max_normalized = 0.4; + d.class_name = "dog"; + d.confidence = 0.7; + + const vpb::Detection proto = vimpl::to_proto(d); + const Vision::detection back = vimpl::from_proto(proto); + + BOOST_TEST(back == d); + BOOST_TEST(!back.x_min.has_value()); +} + +BOOST_AUTO_TEST_CASE(detection_round_trip_no_bbox) { + Vision::detection d; + d.class_name = "unknown"; + d.confidence = 0.1; + + const vpb::Detection proto = vimpl::to_proto(d); + const Vision::detection back = vimpl::from_proto(proto); + + BOOST_TEST(back == d); + BOOST_TEST(!back.x_min.has_value()); + BOOST_TEST(!back.x_min_normalized.has_value()); +} + +BOOST_AUTO_TEST_CASE(classification_round_trip) { + Vision::classification c{"cat", 0.95}; + const vpb::Classification proto = vimpl::to_proto(c); + const Vision::classification back = vimpl::from_proto(proto); + BOOST_TEST(back == c); +} + +BOOST_AUTO_TEST_CASE(properties_round_trip) { + Vision::properties p{true, false, true}; + vpb::GetPropertiesResponse proto; + vimpl::to_proto(p, &proto); + BOOST_TEST(vimpl::from_proto(proto) == p); +} + +BOOST_AUTO_TEST_CASE(point_cloud_object_round_trip_no_geometries) { + Vision::point_cloud_object o; + o.cloud.mime_type = "application/pcd"; + o.cloud.pc = {0xDE, 0xAD, 0xBE, 0xEF}; + // geometries empty + ::viam::common::v1::PointCloudObject proto; + vimpl::to_proto(o, &proto); + auto back = vimpl::from_proto(proto); + // mime_type round-trip is the caller's responsibility; assert only bytes + geometries. + BOOST_CHECK_EQUAL_COLLECTIONS( + back.cloud.pc.begin(), back.cloud.pc.end(), o.cloud.pc.begin(), o.cloud.pc.end()); + BOOST_TEST(back.geometries.empty()); +} + +BOOST_AUTO_TEST_SUITE_END() + +namespace viam { +namespace sdktests { +namespace vision { + +BOOST_AUTO_TEST_SUITE(vision_rpc) + +BOOST_AUTO_TEST_CASE(get_properties_round_trip) { + vision_fixture f; + f.mock->canned_properties = {true, false, true}; + auto got = f.client->get_properties(); + BOOST_TEST(got == f.mock->canned_properties); + BOOST_TEST(f.mock->last_extra.empty()); +} + +BOOST_AUTO_TEST_CASE(get_properties_passes_extra) { + vision_fixture f; + f.mock->canned_properties = {false, true, false}; + sdk::ProtoStruct extra; + extra["k"] = sdk::ProtoValue("v"); + auto got = f.client->get_properties(extra); + BOOST_TEST(got == f.mock->canned_properties); +} + +BOOST_AUTO_TEST_CASE(get_status_round_trip) { + vision_fixture f; + sdk::ProtoStruct status; + status["ready"] = sdk::ProtoValue(true); + status["model"] = sdk::ProtoValue(std::string("yolov8")); + f.mock->canned_status = status; + + auto got = f.client->get_status(); + BOOST_TEST(got == f.mock->canned_status); +} + +BOOST_AUTO_TEST_CASE(do_command_round_trip_simple) { + vision_fixture f; + sdk::ProtoStruct cmd; + cmd["op"] = sdk::ProtoValue(std::string("inspect")); + cmd["limit"] = sdk::ProtoValue(static_cast(42)); + + auto echoed = f.client->do_command(cmd); + + // MockVision::do_command echoes its input back; verify echo round-trips. + BOOST_TEST(echoed == cmd); + // Also verify the mock recorded what came in. + BOOST_TEST(f.mock->last_command == cmd); +} + +BOOST_AUTO_TEST_CASE(do_command_round_trip_nested) { + vision_fixture f; + sdk::ProtoStruct nested; + nested["enabled"] = sdk::ProtoValue(true); + nested["weight"] = sdk::ProtoValue(0.75); + + sdk::ProtoStruct cmd; + cmd["mode"] = sdk::ProtoValue(std::string("calibrate")); + cmd["params"] = sdk::ProtoValue(nested); + + auto echoed = f.client->do_command(cmd); + BOOST_TEST(echoed == cmd); + BOOST_TEST(f.mock->last_command == cmd); +} + +BOOST_AUTO_TEST_CASE(get_classifications_from_camera_round_trip) { + vision_fixture f; + f.mock->canned_classifications = {{"cat", 0.9}, {"dog", 0.1}}; + + auto got = f.client->get_classifications_from_camera("cam0", 5); + + BOOST_TEST_REQUIRE(got.size() == 2u); + BOOST_TEST(got[0] == f.mock->canned_classifications[0]); + BOOST_TEST(got[1] == f.mock->canned_classifications[1]); + BOOST_TEST(f.mock->last_camera_name == "cam0"); + BOOST_TEST(f.mock->last_count == 5); +} + +BOOST_AUTO_TEST_CASE(get_classifications_round_trip) { + vision_fixture f; + f.mock->canned_classifications = {{"box", 0.7}}; + sdk::Vision::image img; + img.mime_type = "image/jpeg"; + img.bytes = {1, 2, 3, 4}; + + auto got = f.client->get_classifications(img, 3); + + BOOST_TEST_REQUIRE(got.size() == 1u); + BOOST_TEST(got[0] == f.mock->canned_classifications[0]); + BOOST_TEST(f.mock->last_image.bytes == img.bytes); + BOOST_TEST(f.mock->last_image.mime_type == img.mime_type); + BOOST_TEST(f.mock->last_count == 3); +} + +BOOST_AUTO_TEST_CASE(get_detections_from_camera_round_trip) { + vision_fixture f; + sdk::Vision::detection d1; + d1.x_min = 10; + d1.y_min = 20; + d1.x_max = 30; + d1.y_max = 40; + d1.class_name = "car"; + d1.confidence = 0.95; + sdk::Vision::detection d2; + d2.x_min_normalized = 0.1; + d2.y_min_normalized = 0.2; + d2.x_max_normalized = 0.3; + d2.y_max_normalized = 0.4; + d2.class_name = "tree"; + d2.confidence = 0.4; + f.mock->canned_detections = {d1, d2}; + + auto got = f.client->get_detections_from_camera("front"); + + BOOST_TEST_REQUIRE(got.size() == 2u); + BOOST_TEST(got[0] == d1); + BOOST_TEST(got[1] == d2); + BOOST_TEST(f.mock->last_camera_name == "front"); +} + +BOOST_AUTO_TEST_CASE(get_detections_round_trip) { + vision_fixture f; + sdk::Vision::detection d; + d.x_min = 5; + d.y_min = 6; + d.x_max = 7; + d.y_max = 8; + d.class_name = "ball"; + d.confidence = 0.5; + f.mock->canned_detections = {d}; + + sdk::Vision::image img; + img.mime_type = "image/png"; + img.bytes = {9, 8, 7, 6, 5}; + + auto got = f.client->get_detections(img); + + BOOST_TEST_REQUIRE(got.size() == 1u); + BOOST_TEST(got[0] == d); + BOOST_TEST(f.mock->last_image.bytes == img.bytes); + BOOST_TEST(f.mock->last_image.mime_type == img.mime_type); +} + +BOOST_AUTO_TEST_CASE(get_detections_no_bbox_round_trip_over_wire) { + // Edge case: a Detection with neither pixel nor normalized bbox engaged + // must round-trip with both groups remaining unset on the response side. + vision_fixture f; + sdk::Vision::detection d; + d.class_name = "unknown"; + d.confidence = 0.1; + // Intentionally no bbox fields engaged. + f.mock->canned_detections = {d}; + + auto got = f.client->get_detections_from_camera("any"); + + BOOST_TEST_REQUIRE(got.size() == 1u); + BOOST_TEST(got[0] == d); + BOOST_TEST(!got[0].x_min.has_value()); + BOOST_TEST(!got[0].x_min_normalized.has_value()); + BOOST_TEST(got[0].class_name == "unknown"); + BOOST_TEST(got[0].confidence == 0.1); +} + +BOOST_AUTO_TEST_CASE(get_object_point_clouds_round_trip_no_geometries) { + vision_fixture f; + sdk::Vision::point_cloud_object o; + o.cloud.mime_type = "application/pcd"; + o.cloud.pc = {0xDE, 0xAD, 0xBE, 0xEF}; + f.mock->canned_objects = {o}; + + auto got = f.client->get_object_point_clouds("rear_cam", "application/pcd"); + + BOOST_TEST_REQUIRE(got.size() == 1u); + BOOST_TEST(got[0].cloud.pc == o.cloud.pc); + BOOST_TEST(got[0].cloud.mime_type == "application/pcd"); + BOOST_TEST(got[0].geometries.empty()); + BOOST_TEST(f.mock->last_camera_name == "rear_cam"); + BOOST_TEST(f.mock->last_mime_type == "application/pcd"); +} + +BOOST_AUTO_TEST_CASE(get_object_point_clouds_uses_server_mime_type) { + // The Vision proto allows the server to return a different mime_type than + // the one requested ("A specific MIME type can be requested but may not + // necessarily be the same one returned"). Verify the client propagates the + // server-returned mime, not the request mime. + vision_fixture f; + sdk::Vision::point_cloud_object o; + o.cloud.mime_type = "application/pcd"; // server-returned mime + o.cloud.pc = {0x01, 0x02, 0x03}; + f.mock->canned_objects = {o}; + + // Request a different mime than the mock will return. + auto got = f.client->get_object_point_clouds("cam", "pointcloud/octet-stream"); + + BOOST_TEST_REQUIRE(got.size() == 1u); + BOOST_TEST(got[0].cloud.mime_type == "application/pcd"); +} + +BOOST_AUTO_TEST_CASE(capture_all_only_image) { + vision_fixture f; + sdk::Vision::raw_image img; + img.mime_type = "image/jpeg"; + img.bytes = {0xFF, 0xD8, 0xFF, 0xE0}; + img.source_name = "front_cam"; + f.mock->canned_capture_all.image = img; + + sdk::Vision::capture_options opts; + opts.return_image = true; + + auto got = f.client->capture_all_from_camera("cam", opts); + + BOOST_TEST_REQUIRE(got.image.has_value()); + BOOST_TEST(got.image->mime_type == "image/jpeg"); + BOOST_TEST(got.image->bytes == img.bytes); + BOOST_TEST(got.detections.empty()); + BOOST_TEST(got.classifications.empty()); + BOOST_TEST(got.objects.empty()); + BOOST_TEST(f.mock->last_camera_name == "cam"); + BOOST_TEST(f.mock->last_capture_options.return_image == true); +} + +BOOST_AUTO_TEST_CASE(capture_all_only_detections) { + vision_fixture f; + sdk::Vision::detection d; + d.x_min = 1; + d.y_min = 2; + d.x_max = 3; + d.y_max = 4; + d.class_name = "person"; + d.confidence = 0.99; + f.mock->canned_capture_all.detections = {d}; + + sdk::Vision::capture_options opts; + opts.return_detections = true; + + auto got = f.client->capture_all_from_camera("cam", opts); + + BOOST_TEST(!got.image.has_value()); + BOOST_TEST_REQUIRE(got.detections.size() == 1u); + BOOST_TEST(got.detections[0] == d); + BOOST_TEST(got.classifications.empty()); + BOOST_TEST(got.objects.empty()); + BOOST_TEST(f.mock->last_capture_options.return_detections == true); +} + +BOOST_AUTO_TEST_CASE(capture_all_full_payload) { + vision_fixture f; + sdk::Vision::raw_image img; + img.mime_type = "image/png"; + img.bytes = {1, 2, 3}; + f.mock->canned_capture_all.image = img; + f.mock->canned_capture_all.classifications = {{"thing", 0.5}}; + sdk::Vision::detection d; + d.x_min = 0; + d.y_min = 0; + d.x_max = 100; + d.y_max = 100; + d.class_name = "obj"; + d.confidence = 0.8; + f.mock->canned_capture_all.detections = {d}; + + sdk::Vision::capture_options opts; + opts.return_image = true; + opts.return_detections = true; + opts.return_classifications = true; + + auto got = f.client->capture_all_from_camera("cam", opts); + + BOOST_TEST_REQUIRE(got.image.has_value()); + BOOST_TEST(got.image->bytes == img.bytes); + BOOST_TEST(got.detections.size() == 1u); + BOOST_TEST(got.detections[0] == d); + BOOST_TEST(got.classifications.size() == 1u); + BOOST_TEST(got.classifications[0].class_name == "thing"); +} + +BOOST_AUTO_TEST_CASE(server_exception_maps_to_grpc_error) { + vision_fixture f; + f.mock->canned_properties = {true, true, true}; + f.mock->throw_on_next_call = true; + + // The mock throws sdk::Exception on the next call. The server's + // make_service_helper catches std::exception and converts it to a + // non-OK grpc::Status; the client's make_client_helper then throws + // GRPCException. + BOOST_CHECK_THROW(f.client->get_properties(), sdk::GRPCException); + + // Confirm the flag was reset (so a follow-up call would succeed). + BOOST_TEST(f.mock->throw_on_next_call == false); +} + +BOOST_AUTO_TEST_CASE(server_exception_recovery) { + vision_fixture f; + f.mock->canned_properties = {true, false, true}; + f.mock->throw_on_next_call = true; + + // First call throws. + BOOST_CHECK_THROW(f.client->get_properties(), sdk::GRPCException); + + // Second call should succeed now that the mock flag was reset. + auto got = f.client->get_properties(); + BOOST_TEST(got == f.mock->canned_properties); +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace vision +} // namespace sdktests +} // namespace viam