[ntuple] Add kTypeS3 locator type for S3 storage backend

tree/ntuple/inc/ROOT/RNTupleTypes.hxx

@@ -236,6 +236,7 @@ public:
       // if the size of the referenced data block is >2GB
       kTypeFile = 0x00,
       kTypeDAOS = 0x02,
+      kTypeS3 = 0x03,
 
       kLastSerializableType = 0x7f,
       kTypePageZero = kLastSerializableType + 1,

tree/ntuple/src/RNTupleSerialize.cxx

@@ -506,7 +506,7 @@
       locator.SetNBytesOnStorage(nBytesOnStorage);
       RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint64_t), location);
    } else {
-      return R__FAIL("invalid DAOS locator payload size: " + std::to_string(sizeofLocatorPayload));
+      return R__FAIL("invalid Object64 locator payload size: " + std::to_string(sizeofLocatorPayload));
    }
    locator.SetPosition(ROOT::RNTupleLocatorObject64{location});
    return ROOT::RResult<void>::Success();
@@ -1091,6 +1091,10 @@
       size += SerializeLocatorPayloadObject64(locator, payloadp);
       locatorType = 0x02;
       break;
+   case RNTupleLocator::kTypeS3:
+      size += SerializeLocatorPayloadObject64(locator, payloadp);
+      locatorType = 0x03;
+      break;
    default:
       if (locator.GetType() == ROOT::Internal::kTestLocatorType) {
          // For the testing locator, use the same payload format as Object64. We won't read it back anyway.
@@ -1139,6 +1143,10 @@
          locator.SetType(RNTupleLocator::kTypeDAOS);
          DeserializeLocatorPayloadObject64(bytes, payloadSize, locator);
          break;
+      case 0x03:
+         locator.SetType(RNTupleLocator::kTypeS3);
+         DeserializeLocatorPayloadObject64(bytes, payloadSize, locator);
+         break;
       default: locator.SetType(RNTupleLocator::kTypeUnknown);
       }
       bytes += payloadSize;
@@ -1871,7 +1879,7 @@
       if (!featureFlags[i])
          continue;
       // NOTE: this assumes all valid feature flags are consecutive, thus we can just check the highest one set.
       unsigned int highestBitSet = 64 * i + (63 - ROOT::Internal::LeadingZeroes(featureFlags[i]));
       if (highestBitSet >= ROOT::RNTupleDescriptor::kFeatureFlag_COUNT)
          return R__FAIL("unsupported format feature: " + std::to_string(highestBitSet));
    }

tree/ntuple/src/RNTupleTypes.cxx

@@ -45,7 +45,8 @@ ROOT::RNTupleLocator::ELocatorType ROOT::RNTupleLocator::GetType() const
    case 1: return kTypeDAOS;
    case 2: return kTypePageZero;
    case 3: return kTypeUnknown;
-   case 4: return Internal::kTestLocatorType;
+   case 4: return kTypeS3;
+   case 5: return Internal::kTestLocatorType;
    default: break;
    }
    R__ASSERT(false);
@@ -60,9 +61,10 @@ void ROOT::RNTupleLocator::SetType(ELocatorType type)
    case kTypeDAOS: compactType = 1; break;
    case kTypePageZero: compactType = 2; break;
    case kTypeUnknown: compactType = 3; break;
+   case kTypeS3: compactType = 4; break;
    default:
       if (type == Internal::kTestLocatorType)
-         compactType = 4;
+         compactType = 5;
       else
          throw RException(R__FAIL("invalid locator type: " + std::to_string(type)));
    }
@@ -79,7 +81,7 @@ void ROOT::RNTupleLocator::SetPosition(std::uint64_t position)
 
 void ROOT::RNTupleLocator::SetPosition(RNTupleLocatorObject64 position)
 {
-   if (GetType() != kTypeDAOS)
+   if (GetType() != kTypeDAOS && GetType() != kTypeS3)
       throw RException(R__FAIL("cannot set position as 64bit object for type " + std::to_string(GetType())));
    fPosition = position.GetLocation();
 }
@@ -94,7 +96,7 @@ std::uint64_t ROOT::Internal::RNTupleLocatorHelper<std::uint64_t>::Get(const RNT
 ROOT::RNTupleLocatorObject64
 ROOT::Internal::RNTupleLocatorHelper<ROOT::RNTupleLocatorObject64>::Get(const RNTupleLocator &loc)
 {
-   if (loc.GetType() != ROOT::RNTupleLocator::kTypeDAOS)
+   if (loc.GetType() != ROOT::RNTupleLocator::kTypeDAOS && loc.GetType() != ROOT::RNTupleLocator::kTypeS3)
       throw RException(R__FAIL("cannot retrieve position as 64bit object for type " + std::to_string(loc.GetType())));
    return RNTupleLocatorObject64{loc.fPosition};
 }

tree/ntuple/test/ntuple_serialize.cxx

@@ -399,6 +399,31 @@ TEST(RNTuple, SerializeLocator)
    EXPECT_EQ(locator.GetReserved(), 1);
    EXPECT_EQ(1337U, locator.GetPosition<RNTupleLocatorObject64>().GetLocation());
 
+   // S3 locator round-trip with 32-bit nBytesOnStorage
+   locator = RNTupleLocator{};
+   locator.SetType(RNTupleLocator::kTypeS3);
+   locator.SetPosition(RNTupleLocatorObject64{42U});
+   locator.SetNBytesOnStorage(1024U);
+   locator.SetReserved(0);
+   EXPECT_EQ(16u, RNTupleSerializer::SerializeLocator(locator, buffer).Unwrap());
+   locator = RNTupleLocator{};
+   EXPECT_EQ(16u, RNTupleSerializer::DeserializeLocator(buffer, 16, locator).Unwrap());
+   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeS3);
+   EXPECT_EQ(locator.GetNBytesOnStorage(), 1024U);
+   EXPECT_EQ(locator.GetReserved(), 0);
+   EXPECT_EQ(42U, locator.GetPosition<RNTupleLocatorObject64>().GetLocation());
+
+   // S3 locator round-trip with 64-bit nBytesOnStorage and reserved bit
+   locator.SetNBytesOnStorage(static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1);
+   locator.SetReserved(1);
+   EXPECT_EQ(20u, RNTupleSerializer::SerializeLocator(locator, buffer).Unwrap());
+   locator = RNTupleLocator{};
+   EXPECT_EQ(20u, RNTupleSerializer::DeserializeLocator(buffer, 20, locator).Unwrap());
+   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeS3);
+   EXPECT_EQ(locator.GetNBytesOnStorage(), static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1);
+   EXPECT_EQ(locator.GetReserved(), 1);
+   EXPECT_EQ(42U, locator.GetPosition<RNTupleLocatorObject64>().GetLocation());
+
    std::int32_t *head = reinterpret_cast<std::int32_t *>(buffer);
 #ifndef R__BYTESWAP
    // on big endian system