Skip to content

perf: optimize ParserSQL digest and classification overhead #6137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Snehil-Shah wants to merge 1 commit into
base: v3.0
Choose a base branch
from
Open

perf: optimize ParserSQL digest and classification overhead #6137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions include/Query_Processor_ParserSQL.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,25 @@ enum MYSQL_COM_QUERY_command parsersql_command_type_mysql(const char* query, int
*/
enum PGSQL_QUERY_command parsersql_command_type_pgsql(const char* query, int query_length);

/**
* @brief Maps a ParserSQL StmtType to a `MYSQL_COM_QUERY_command`.
*
* @param stmt_type A `StmtType` value (see sql_parser/common.h), as an int.
* @return The corresponding command type.
*/
enum MYSQL_COM_QUERY_command parsersql_stmt_type_to_mysql_command(int stmt_type);

/**
* @brief Maps a ParserSQL StmtType to a `PGSQL_QUERY_command`.
*
* Identical to parsersql_stmt_type_to_mysql_command() but targets the
* PostgreSQL command enum.
*
* @param stmt_type A `StmtType` value (see sql_parser/common.h), as an int.
* @return The corresponding command type.
*/
enum PGSQL_QUERY_command parsersql_stmt_type_to_pgsql_command(int stmt_type);

/**
* @brief Parses a MySQL SET statement into variable-value pairs.
*
Expand Down
1 change: 1 addition & 0 deletions include/proxysql_structs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -899,6 +899,7 @@ struct __SQP_query_parser_t {
char *digest_text;
char *first_comment;
char *query_prefix;
int parsersql_stmt_type;

@cubic-dev-ai cubic-dev-ai Bot Aug 29, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: When a zero-initialized SQP_par_t reaches command lookup with ParserSQL enabled, parsersql_stmt_type is 0, so the lookup treats it as a cached type and never parses digest_text. Initialize this field to -1 for every manually constructed parser state, or make the lookup distinguish an absent cache from a valid type.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At include/proxysql_structs.h, line 902:

<comment>When a zero-initialized `SQP_par_t` reaches command lookup with ParserSQL enabled, `parsersql_stmt_type` is `0`, so the lookup treats it as a cached type and never parses `digest_text`. Initialize this field to `-1` for every manually constructed parser state, or make the lookup distinguish an absent cache from a valid type.</comment>

<file context>
@@ -899,6 +899,7 @@ struct __SQP_query_parser_t {
 	char *digest_text;
 	char *first_comment;
 	char *query_prefix;
+	int parsersql_stmt_type;
 };
 
</file context>

};

struct _PtrSize_t {
Expand Down
4 changes: 4 additions & 0 deletions lib/MySQL_Query_Processor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ enum MYSQL_COM_QUERY_command MySQL_Query_Processor::query_parser_command_type(SQ
enum MYSQL_COM_QUERY_command ret = MYSQL_COM_QUERY_UNKNOWN;

if (mysql_thread___query_processor_parser == 1) {
if (qp->parsersql_stmt_type >= 0) {
// already classified during the digest parse
return parsersql_stmt_type_to_mysql_command(qp->parsersql_stmt_type);
}
if (text) {
return parsersql_command_type_mysql(text, strlen(text)); // NOSONAR
}
Expand Down
4 changes: 4 additions & 0 deletions lib/PgSQL_Query_Processor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,10 @@ enum PGSQL_QUERY_command PgSQL_Query_Processor::query_parser_command_type(SQP_pa
enum PGSQL_QUERY_command ret = PGSQL_QUERY_UNKNOWN;

if (pgsql_thread___query_processor_parser == 1) {
if (qp->parsersql_stmt_type >= 0) {
// already classified during the digest parse
return parsersql_stmt_type_to_pgsql_command(qp->parsersql_stmt_type);
}
if (text) {
return parsersql_command_type_pgsql(text, strlen(text)); // NOSONAR
}
Expand Down
1 change: 1 addition & 0 deletions lib/Query_Processor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2328,6 +2328,7 @@ void Query_Processor<QP_DERIVED>::query_parser_init(SQP_par_t *qp, const char *q
qp->digest_text=NULL;
qp->first_comment=NULL;
qp->query_prefix=NULL;
qp->parsersql_stmt_type=-1;
if (GET_THREAD_VARIABLE(query_digests)) {
if (GET_THREAD_VARIABLE(query_processor_parser) == 1) {
qp_digest_parsersql<QP_DERIVED>(qp, query, query_length);
Expand Down
52 changes: 42 additions & 10 deletions lib/Query_Processor_ParserSQL.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,26 +413,39 @@ void parsersql_digest_init_mysql(SQP_par_t* qp, const char* query, int query_len
qp->first_comment = NULL;
qp->query_prefix = NULL;
qp->digest = 0;
qp->parsersql_stmt_type = -1;

auto result = tl_mysql_parser.parse(query, query_length);

if (result.status == ParseResult::OK || result.status == ParseResult::PARTIAL) {
std::string normalized;
// The arena holds 'normalized' until reset() below, so it can be hashed and copied out directly.
const char* normalized = NULL;
size_t normalized_len = 0;
if (result.ast) {
// Tier 1: full AST available — use Emitter in DIGEST mode
Emitter<Dialect::MySQL> emitter(tl_mysql_parser.arena(), EmitMode::DIGEST);
emitter.emit(result.ast);
StringRef ref = emitter.result();
normalized.assign(ref.ptr, ref.len);
normalized = ref.ptr;
normalized_len = ref.len;
} else {
// Tier 2: token-level fallback for statements without full AST support
Digest<Dialect::MySQL> digest(tl_mysql_parser.arena());
DigestResult dr = digest.compute(query, query_length);
normalized.assign(dr.normalized.ptr, dr.normalized.len);
normalized = dr.normalized.ptr;
normalized_len = dr.normalized.len;
}
qp->digest_text = strdup(normalized.c_str());
// SpookyHash is preserved for backward compatibility with existing digest stats
qp->digest = SpookyHash::Hash64(normalized.c_str(), normalized.size(), 0);
qp->digest = SpookyHash::Hash64(normalized, normalized_len, 0);
qp->parsersql_stmt_type = static_cast<int>(result.stmt_type);
// Reuse qp->buf when it fits, as query_parser_free() expects for the legacy tokenizer.
if (normalized_len < QUERY_DIGEST_BUF) {
memcpy(qp->buf, normalized, normalized_len);
qp->buf[normalized_len] = '\0';
qp->digest_text = qp->buf;

@cubic-dev-ai cubic-dev-ai Bot Aug 29, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1: When callers use either ParserSQL digest adapter directly, short normalized output now makes digest_text point to inline storage, but existing callers free that field and abort. Keep digest_text heap-allocated for this public adapter contract, or update the API and every caller to use the ownership-aware cleanup path in both MySQL and PostgreSQL branches.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At lib/Query_Processor_ParserSQL.cpp, line 445:

<comment>When callers use either ParserSQL digest adapter directly, short normalized output now makes `digest_text` point to inline storage, but existing callers free that field and abort. Keep `digest_text` heap-allocated for this public adapter contract, or update the API and every caller to use the ownership-aware cleanup path in both MySQL and PostgreSQL branches.</comment>

<file context>
@@ -413,26 +413,39 @@ void parsersql_digest_init_mysql(SQP_par_t* qp, const char* query, int query_len
+        if (normalized_len < QUERY_DIGEST_BUF) {
+            memcpy(qp->buf, normalized, normalized_len);
+            qp->buf[normalized_len] = '\0';
+            qp->digest_text = qp->buf;
+        } else {
+            qp->digest_text = strndup(normalized, normalized_len);
</file context>

} else {
qp->digest_text = strndup(normalized, normalized_len);
}
}

tl_mysql_parser.reset();
Expand All @@ -444,23 +457,34 @@ void parsersql_digest_init_pgsql(SQP_par_t* qp, const char* query, int query_len
qp->first_comment = NULL;
qp->query_prefix = NULL;
qp->digest = 0;
qp->parsersql_stmt_type = -1;

auto result = tl_pgsql_parser.parse(query, query_length);

if (result.status == ParseResult::OK || result.status == ParseResult::PARTIAL) {
std::string normalized;
const char* normalized = NULL;
size_t normalized_len = 0;
if (result.ast) {
Emitter<Dialect::PostgreSQL> emitter(tl_pgsql_parser.arena(), EmitMode::DIGEST);
emitter.emit(result.ast);
StringRef ref = emitter.result();
normalized.assign(ref.ptr, ref.len);
normalized = ref.ptr;
normalized_len = ref.len;
} else {
Digest<Dialect::PostgreSQL> digest(tl_pgsql_parser.arena());
DigestResult dr = digest.compute(query, query_length);
normalized.assign(dr.normalized.ptr, dr.normalized.len);
normalized = dr.normalized.ptr;
normalized_len = dr.normalized.len;
}
qp->digest = SpookyHash::Hash64(normalized, normalized_len, 0);
qp->parsersql_stmt_type = static_cast<int>(result.stmt_type);
if (normalized_len < QUERY_DIGEST_BUF) {
memcpy(qp->buf, normalized, normalized_len);
qp->buf[normalized_len] = '\0';
qp->digest_text = qp->buf;
} else {
qp->digest_text = strndup(normalized, normalized_len);
}
qp->digest_text = strdup(normalized.c_str());
qp->digest = SpookyHash::Hash64(normalized.c_str(), normalized.size(), 0);
}

tl_pgsql_parser.reset();
Expand Down Expand Up @@ -565,6 +589,14 @@ enum MYSQL_COM_QUERY_command parsersql_command_type_mysql(const char* query, int
return MYSQL_COM_QUERY_UNKNOWN;
}

enum MYSQL_COM_QUERY_command parsersql_stmt_type_to_mysql_command(int stmt_type) {
return stmt_type_to_mysql_command(static_cast<StmtType>(stmt_type));
}

enum PGSQL_QUERY_command parsersql_stmt_type_to_pgsql_command(int stmt_type) {
return stmt_type_to_pgsql_command(static_cast<StmtType>(stmt_type));
}

enum PGSQL_QUERY_command parsersql_command_type_pgsql(const char* query, int query_length) {
auto result = tl_pgsql_parser.parse(query, query_length);
tl_pgsql_parser.reset();
Expand Down
Loading