fix: Export binary-tagged UTF-8 attributes over OTLP

common/lib/opentelemetry/common/utilities.rb

@@ -54,9 +54,9 @@ def time_in_nanoseconds(timestamp = Time.now)
       #
       # @param [String] string The string to be utf8 encoded
       # @param [optional boolean] binary This option is for displaying binary data
-      # @param [optional String] placeholder The fallback string to be used if encoding fails
+      # @param [String, nil] placeholder The fallback value to be used if encoding fails
       #
-      # @return [String]
+      # @return [String, nil]
       def utf8_encode(string, binary: false, placeholder: STRING_PLACEHOLDER)
         string = string.to_s
 
@@ -66,6 +66,11 @@ def utf8_encode(string, binary: false, placeholder: STRING_PLACEHOLDER)
           string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
         elsif string.encoding == ::Encoding::UTF_8
           string
+        elsif string.encoding == ::Encoding::ASCII_8BIT
+          utf8_string = string.dup.force_encoding(::Encoding::UTF_8)
+          raise Encoding::InvalidByteSequenceError, 'binary string is not valid UTF-8' unless utf8_string.valid_encoding?
+
+          utf8_string
         else
           string.encode(::Encoding::UTF_8)
         end

common/test/opentelemetry/common/utilities_test.rb

@@ -76,6 +76,22 @@ def shutdown(timeout: nil); end
       assert_equal('?', common_utils.utf8_encode(time_bomb, placeholder: '?'))
     end
 
+    it 'preserves valid UTF-8 bytes from a binary-encoded string' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      encoded = common_utils.utf8_encode(city)
+
+      assert_equal('Montréal', encoded)
+      assert_equal(::Encoding::UTF_8, encoded.encoding)
+      assert_equal(::Encoding::ASCII_8BIT, city.encoding)
+    end
+
+    it 'does not validate an already UTF-8-tagged string' do
+      invalid = "\xC3".dup.force_encoding(::Encoding::UTF_8)
+
+      assert_same(invalid, common_utils.utf8_encode(invalid, placeholder: '?'))
+    end
+
     it 'with binary data' do
       byte_array = (+"keep what\xC2 is valid").force_encoding(::Encoding::ASCII_8BIT)
 

exporter/otlp-common/lib/opentelemetry/exporter/otlp/common.rb

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 require 'opentelemetry'
+require 'opentelemetry/common'
 require 'opentelemetry/exporter/otlp/common/version'
 
 require 'google/rpc/status_pb'
@@ -130,9 +131,10 @@ def as_otlp_span_kind(kind)
         end
 
         def as_otlp_key_value(key, value)
+          key = OpenTelemetry::Common::Utilities.utf8_encode(key, placeholder: 'Encoding Error')
           Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value))
         rescue Encoding::UndefinedConversionError => e
-          encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
+          encoded_value = value.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
           OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}")
           Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error'))
         end
@@ -141,7 +143,7 @@ def as_otlp_any_value(value)
           result = Opentelemetry::Proto::Common::V1::AnyValue.new
           case value
           when String
-            result.string_value = value
+            result.string_value = OpenTelemetry::Common::Utilities.utf8_encode(value, placeholder: value)
           when Integer
             result.int_value = value
           when Float

exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'googleapis-common-protos-types', '~> 1.3'
   spec.add_dependency 'google-protobuf', '~> 3.19'
   spec.add_dependency 'opentelemetry-api', '~> 1.1'
+  spec.add_dependency 'opentelemetry-common', '~> 0.20'
 
   if spec.respond_to?(:metadata)
     spec.metadata['changelog_uri'] = "https://rubydoc.info/gems/#{spec.name}/#{spec.version}/file/CHANGELOG.md"

exporter/otlp-common/test/opentelemetry/exporter/otlp/common/common_test.rb

@@ -61,6 +61,46 @@
       _(result.resource_spans).must_be_empty
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded attribute strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'city' => city }
+      )
+
+      etsr = OpenTelemetry::Exporter::OTLP::Common.as_etsr([span_data])
+      exported_span = etsr.resource_spans.first.scope_spans.first.spans.first
+
+      _(exported_span.attributes.first.value.string_value).must_equal('Montréal')
+    end
+
+    it 'safely exports attributes with invalid UTF-8 keys' do
+      invalid_key = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { invalid_key => 'value' }
+      )
+
+      etsr = OpenTelemetry::Exporter::OTLP::Common.as_etsr([span_data])
+      exported_attribute = etsr.resource_spans.first.scope_spans.first.spans.first.attributes.first
+
+      _(exported_attribute.key).must_equal('Encoding Error')
+      _(exported_attribute.value.string_value).must_equal('value')
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'values' => [invalid_value] }
+      )
+
+      etsr = OpenTelemetry::Exporter::OTLP::Common.as_etsr([span_data])
+      exported_value = etsr.resource_spans.first.scope_spans.first.spans.first.attributes.first.value
+
+      _(exported_value.string_value).must_equal('Encoding Error')
+    end
+
     it 'batches per resource and instrumentation scope' do
       # Test resource batching
       resource_one = OpenTelemetry::SDK::Resources::Resource.create('k1' => 'v1')

exporter/otlp-logs/lib/opentelemetry/exporter/otlp/logs/logs_exporter.rb

@@ -318,9 +318,10 @@ def as_otlp_log_record(log_record_data)
           end
 
           def as_otlp_key_value(key, value)
+            key = OpenTelemetry::Common::Utilities.utf8_encode(key, placeholder: 'Encoding Error')
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value))
           rescue Encoding::UndefinedConversionError => e
-            encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
+            encoded_value = value.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
             OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}")
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error'))
           end
@@ -329,7 +330,7 @@ def as_otlp_any_value(value) # rubocop:disable Metrics/CyclomaticComplexity
             result = Opentelemetry::Proto::Common::V1::AnyValue.new
             case value
             when String
-              result.string_value = value
+              result.string_value = OpenTelemetry::Common::Utilities.utf8_encode(value, placeholder: value)
             when Integer
               result.int_value = value
             when Float

exporter/otlp-logs/test/opentelemetry/exporter/otlp/logs_exporter_test.rb

@@ -649,6 +649,23 @@
       OpenTelemetry.logger = logger
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      value = exporter.send(:as_otlp_any_value, city)
+
+      _(value.string_value).must_equal('Montréal')
+      _(value.string_value.encoding).must_equal(::Encoding::UTF_8)
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      attribute = exporter.send(:as_otlp_key_value, 'values', [invalid_value])
+
+      _(attribute.value.string_value).must_equal('Encoding Error')
+    end
+
     it 'logs rpc.Status on bad request' do
       log_stream = StringIO.new
       logger = OpenTelemetry.logger

exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb

@@ -31,9 +31,10 @@ def around_request
           end
 
           def as_otlp_key_value(key, value)
+            key = OpenTelemetry::Common::Utilities.utf8_encode(key, placeholder: 'Encoding Error')
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value))
           rescue Encoding::UndefinedConversionError => e
-            encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
+            encoded_value = value.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
             OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}")
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error'))
           end
@@ -42,7 +43,7 @@ def as_otlp_any_value(value)
             result = Opentelemetry::Proto::Common::V1::AnyValue.new
             case value
             when String
-              result.string_value = value
+              result.string_value = OpenTelemetry::Common::Utilities.utf8_encode(value, placeholder: value)
             when Integer
               result.int_value = value
             when Float

exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb

@@ -587,6 +587,23 @@
       OpenTelemetry.logger = logger
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded attribute strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      value = exporter.send(:as_otlp_any_value, city)
+
+      _(value.string_value).must_equal('Montréal')
+      _(value.string_value.encoding).must_equal(::Encoding::UTF_8)
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      attribute = exporter.send(:as_otlp_key_value, 'values', [invalid_value])
+
+      _(attribute.value.string_value).must_equal('Encoding Error')
+    end
+
     it 'is able to encode NumberDataPoint with Integer or Float value' do
       stub_request(:post, 'http://localhost:4318/v1/metrics').to_return(status: 200)
 

exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb

@@ -391,9 +391,10 @@ def as_otlp_span_kind(kind)
         end
 
         def as_otlp_key_value(key, value)
+          key = OpenTelemetry::Common::Utilities.utf8_encode(key, placeholder: 'Encoding Error')
           Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value))
         rescue Encoding::UndefinedConversionError => e
-          encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
+          encoded_value = value.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
           OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}")
           Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error'))
         end
@@ -402,7 +403,7 @@ def as_otlp_any_value(value)
           result = Opentelemetry::Proto::Common::V1::AnyValue.new
           case value
           when String
-            result.string_value = value
+            result.string_value = OpenTelemetry::Common::Utilities.utf8_encode(value, placeholder: value)
           when Integer
             result.int_value = value
           when Float

exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb

@@ -627,6 +627,34 @@
       OpenTelemetry.logger = logger
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded attribute strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'city' => city }
+      )
+
+      encoded_data = exporter.send(:encode, [span_data])
+      decoded = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(encoded_data)
+      exported_span = decoded.resource_spans.first.scope_spans.first.spans.first
+
+      _(exported_span.attributes.first.value.string_value).must_equal('Montréal')
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'values' => [invalid_value] }
+      )
+
+      encoded_data = exporter.send(:encode, [span_data])
+      decoded = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(encoded_data)
+      exported_value = decoded.resource_spans.first.scope_spans.first.spans.first.attributes.first.value
+
+      _(exported_value.string_value).must_equal('Encoding Error')
+    end
+
     it 'logs rpc.Status on bad request' do
       log_stream = StringIO.new
       logger = OpenTelemetry.logger

logs_sdk/Gemfile

@@ -20,6 +20,7 @@ group :test, :development do
   gem 'yard', '~> 0.9.0'
   gem 'yard-doctest', '~> 0.1.17'
   gem 'opentelemetry-api', path: '../api', require: false
+  gem 'opentelemetry-common', path: '../common', require: false
   gem 'opentelemetry-exporter-otlp-logs', path: '../exporter/otlp-logs', require: false
   gem 'opentelemetry-logs-api', path: '../logs_api', require: false
   gem 'opentelemetry-sdk', path: '../sdk', require: false

logs_sdk/lib/opentelemetry/sdk/logs/log_record.rb

@@ -81,7 +81,7 @@ def initialize(
           @severity_text = severity_text
           @severity_number = severity_number
           @body = body
-          @attributes = attributes&.to_h # We need a mutable copy of attributes
+          @attributes = Internal.normalize_attribute_encodings(body, 'log record', attributes&.to_h)
           @event_name = event_name
           @trace_id = trace_id
           @span_id = span_id

logs_sdk/test/opentelemetry/sdk/logs/log_record_test.rb

@@ -136,6 +136,39 @@
         end
       end
 
+      it 'normalizes valid UTF-8 bytes from a binary-encoded attribute string' do
+        city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+        log_record = Logs::LogRecord.new(attributes: { 'city' => city })
+        value = log_record.attributes['city']
+
+        assert_equal('Montréal', value)
+        assert_equal(::Encoding::UTF_8, value.encoding)
+        assert_equal(::Encoding::ASCII_8BIT, city.encoding)
+      end
+
+      it 'normalizes valid UTF-8 bytes in attribute keys' do
+        key = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+        log_record = Logs::LogRecord.new(attributes: { key => 'city' })
+        normalized_key = log_record.attributes.keys.first
+
+        assert_equal('Montréal', normalized_key)
+        assert_equal(::Encoding::UTF_8, normalized_key.encoding)
+        assert_equal(::Encoding::ASCII_8BIT, key.encoding)
+      end
+
+      it 'drops binary-encoded attribute strings that are not valid UTF-8' do
+        invalid = "\xC3".dup.force_encoding(::Encoding::ASCII_8BIT)
+
+        OpenTelemetry::TestHelpers.with_test_logger do |log_stream|
+          log_record = Logs::LogRecord.new(attributes: { 'invalid' => invalid })
+
+          assert_empty(log_record.attributes)
+          assert_match(/invalid UTF-8 encoding.*invalid.*Dropping attribute/, log_stream.string)
+        end
+      end
+
       it 'uses the default limits if none provided' do
         log_record = Logs::LogRecord.new
         default = Logs::LogRecordLimits::DEFAULT

metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/asynchronous_instrument.rb

@@ -82,14 +82,15 @@ def timeout(timeout)
           end
 
           def add_attributes(attributes)
-            @attributes.merge!(attributes) if attributes.instance_of?(Hash)
+            @attributes.merge!(Internal.normalize_attributes(@name, 'metric', attributes)) if attributes.instance_of?(Hash)
           end
 
           private
 
           # update the observed value (after calling observe)
           # invoke callback will execute callback and export metric_data that is observed
           def update(timeout, attributes)
+            attributes = Internal.normalize_attributes(@name, 'metric', attributes)
             @metric_streams.each { |ms| ms.invoke_callback(timeout, attributes) }
           end
 

metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/synchronous_instrument.rb

@@ -44,6 +44,7 @@ def register_with_new_metric_store(metric_store, aggregation: default_aggregatio
           private
 
           def update(value, attributes)
+            attributes = Internal.normalize_attributes(@name, 'metric', attributes)
             @metric_streams.each { |ms| ms.update(value, attributes) }
           end
         end

metrics_sdk/test/opentelemetry/sdk/metrics/instrument/counter_test.rb

@@ -30,4 +30,28 @@
     _(last_snapshot[0].data_points[0].attributes).must_equal('foo' => 'bar')
     _(last_snapshot[0].aggregation_temporality).must_equal(:cumulative)
   end
+
+  it 'normalizes valid UTF-8 bytes in attributes' do
+    city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+    counter.add(1, attributes: { 'city' => city })
+    metric_exporter.pull
+    value = metric_exporter.metric_snapshots[0].data_points[0].attributes['city']
+
+    _(value).must_equal('Montréal')
+    _(value.encoding).must_equal(::Encoding::UTF_8)
+    _(city.encoding).must_equal(::Encoding::ASCII_8BIT)
+  end
+
+  it 'drops attributes that are not valid UTF-8' do
+    invalid = "\xC3".dup.force_encoding(::Encoding::ASCII_8BIT)
+
+    OpenTelemetry::TestHelpers.with_test_logger do |log_stream|
+      counter.add(1, attributes: { 'invalid' => invalid })
+      metric_exporter.pull
+
+      _(metric_exporter.metric_snapshots[0].data_points[0].attributes).must_be_empty
+      _(log_stream.string).must_match(/invalid UTF-8 encoding.*invalid.*Dropping attribute/)
+    end
+  end
 end