Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions lib/routes/routeBackbeat.js
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,10 @@ function createCipherBundle(bucketInfo, isV2Request, log, cb) {
}

function putData(request, response, bucketInfo, objMd, log, callback) {
if (request.serverAccessLog) {
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.replication = true;
}
let errMessage;
const canonicalID = request.headers['x-scal-canonical-id'];
if (canonicalID === undefined) {
Expand Down Expand Up @@ -532,6 +536,52 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {

const { headers, bucketName, objectKey } = request;

// Destination-side delete-marker replication.
// We need the REPLICA status to distinquish from
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo: "distinquish" should be "distinguish".

```suggestion
// We need the REPLICA status to distinguish from

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo: "distinquish" should be "distinguish".

```suggestion
// We need the REPLICA status to distinguish from

// source-side replication status updates that also carry isDeleteMarker=true.
if (omVal.isDeleteMarker
&& omVal.replicationInfo
&& omVal.replicationInfo.status === 'REPLICA'
&& request.serverAccessLog) {
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.replication = true;
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.deleteMarker = true;
}

// Destination-side tag-only replication.
// AWS uses REST.PUT.OBJECT_TAGGING for both - a tag-delete
// is replicated as a PUT of an empty tag set with the same
// URI shape.
// The REPLICA status excludes source-side replication-status updates.
if (omVal.replicationInfo
&& omVal.replicationInfo.status === 'REPLICA'
&& (omVal.originOp === 's3:ObjectTagging:Put'
|| omVal.originOp === 's3:ObjectTagging:Delete')
&& request.serverAccessLog) {
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.replication = true;
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.tagging = true;
}

// Destination-side ACL-only replication.
// AWS uses REST.PUT.ACL on the destination with URI shape
// PUT /<bucket>/<key>?acl&versionId=<srcVersionId> and
// populates the aclRequired field.
// The REPLICA status excludes source-side replication-status updates.
if (omVal.replicationInfo
&& omVal.replicationInfo.status === 'REPLICA'
&& omVal.originOp === 's3:ObjectAcl:Put'
&& request.serverAccessLog) {
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.replication = true;
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.acl = true;
// eslint-disable-next-line no-param-reassign
request.serverAccessLog.aclRequired = 'Yes';
}

if (headers['x-scal-replication-content'] === 'METADATA') {
if (!objMd) {
return callback(errors.ObjNotFound);
Expand Down
46 changes: 43 additions & 3 deletions lib/utilities/serverAccessLogger.js
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,18 @@ function getOperation(req) {
if (req.serverAccessLog.expiration) {
return 'S3.EXPIRE.OBJECT';
}
if (req.serverAccessLog.replication) {
if (req.serverAccessLog.deleteMarker) {
return 'REST.DELETE.OBJECT';
}
if (req.serverAccessLog.tagging) {
return 'REST.PUT.OBJECT_TAGGING';
}
if (req.serverAccessLog.acl) {
return 'REST.PUT.ACL';
}
return 'REST.PUT.OBJECT';
}
return `REST.${req.method}.BACKBEAT`;
}

Expand Down Expand Up @@ -519,9 +531,10 @@ function buildLogEntry(req, params, options) {

// Scality server access logs extra fields
logFormatVersion: SERVER_ACCESS_LOG_FORMAT_VERSION,
// For non-expiration backbeat requests, force loggingEnabled
// to false to prevent delivery to log courier.
loggingEnabled: (params.backbeat && !params.expiration) ? false : (params.enabled ?? undefined),
// For backbeat requests other than expiration and replication,
// force loggingEnabled to false to prevent delivery to log courier.
loggingEnabled: (params.backbeat && !params.expiration && !params.replication)
? false : (params.enabled ?? undefined),
loggingTargetBucket: params.loggingEnabled?.TargetBucket ?? undefined,
loggingTargetPrefix: params.loggingEnabled?.TargetPrefix ?? undefined,
awsAccessKeyID: authInfo?.getAccessKey() ?? undefined,
Expand Down Expand Up @@ -628,6 +641,33 @@ function logServerAccess(req, res) {
logEntry.awsAccessKeyID = undefined;
}

// Match AWS log shape for replication entries: blank clientIP, userAgent
// and referer, and rewrite requestURI from the internal /_/backbeat/{data,metadata}
// endpoint to a standard verb on the destination bucket. Delete-marker
// replication arrives as a putMetadata (PUT) but is logged as a DELETE.
// Tag-only replication keeps PUT but appends ?tagging&versionId=<destVid>
// to mirror the AWS PutObjectTagging URI shape. ACL-only replication
// keeps PUT but appends ?acl&versionId=<destVid> to mirror the AWS
// PutObjectAcl URI shape.
if (params.replication) {
logEntry.clientIP = undefined;
logEntry.userAgent = undefined;
logEntry.referer = undefined;
let method = req.method;
let query = '';
if (params.deleteMarker) {
method = 'DELETE';
} else if (params.tagging) {
const versionId = req.query?.versionId;
query = versionId ? `?tagging&versionId=${versionId}` : '?tagging';
} else if (params.acl) {
const versionId = req.query?.versionId;
query = versionId ? `?acl&versionId=${versionId}` : '?acl';
}
logEntry.requestURI =
`${method} /${params.bucketName}/${params.objectKey}${query} HTTP/${req.httpVersion ?? '1.1'}`;
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

params.objectKey is URI-decoded (via _decodeURI at line 79 of routeBackbeat.js), so keys with special characters (spaces, unicode) will appear unencoded here (e.g. my file.txt), while normal S3 request URIs logged via getURI(req) preserve the raw HTTP encoding (my%20file.txt). This could cause inconsistent parsing for log consumers. Consider re-encoding the key to match non-replication entries.

— Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The synthesized requestURI uses the decoded objectKey directly, but for non-replication requests getURI() uses the raw request.url which is URL-encoded. An object key like my file.txt would appear as PUT /bucket/my%20file.txt HTTP/1.1 in normal logs but PUT /bucket/my file.txt HTTP/1.1 here. Consider URL-encoding bucketName and objectKey to stay consistent with the non-replication log format.

— Claude Code

}

if (params.internalLogRequestQueue && params.internalLogRequestQueue.length > 0) {
if (logEntry.operation === 'REST.POST.MULTI_OBJECT_DELETE') {
for (const entry of params.internalLogRequestQueue) {
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@zenko/cloudserver",
"version": "9.2.39",
"version": "9.2.40",
"description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
"main": "index.js",
"engines": {
Expand Down
Loading
Loading