From baed4bdf7599c586c1b40f60d00a2af730a412f6 Mon Sep 17 00:00:00 2001 From: reeb Date: Tue, 2 Jun 2026 23:51:05 +0300 Subject: [PATCH 1/2] RDoc-2506 Explain in the SQL ETL page that the target SQL table's document ID column needs a case-insensitive collation --- docs/server/ongoing-tasks/etl/sql.mdx | 17 +++++++++++++++++ .../tasks/import-data/import-from-sql.mdx | 4 ++-- .../server/ongoing-tasks/etl/sql.mdx | 17 +++++++++++++++++ .../tasks/import-data/import-from-sql.mdx | 4 ++-- .../server/ongoing-tasks/etl/sql.mdx | 17 +++++++++++++++++ .../tasks/import-data/import-from-sql.mdx | 4 ++-- .../server/ongoing-tasks/etl/sql.mdx | 17 +++++++++++++++++ .../tasks/import-data/import-from-sql.mdx | 4 ++-- 8 files changed, 76 insertions(+), 8 deletions(-) diff --git a/docs/server/ongoing-tasks/etl/sql.mdx b/docs/server/ongoing-tasks/etl/sql.mdx index 0227bab6bb..9582f4b0a4 100644 --- a/docs/server/ongoing-tasks/etl/sql.mdx +++ b/docs/server/ongoing-tasks/etl/sql.mdx @@ -75,6 +75,23 @@ Define the target tables where the SQL ETL task will load data. * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. + + + The document ID column should use a **case-insensitive collation** in the destination database. + + When a document is deleted, RavenDB lowercases its ID in the generated DELETE + statement, for example: + `DELETE FROM "Orders" WHERE "Id" IN ('orders/1-a')` + + If the document ID column uses a case-sensitive collation, this lowercase ID will not + match the row that was stored with its original casing (such as `Orders/1-A`), so the + DELETE removes nothing and the row remains in the table as an orphaned record. + + This affects deletions only: when a document is created or updated, RavenDB sends the + ID in its original casing, so the row matches and is removed or replaced correctly. + + + * Note that the specified column does not need to be the primary key of the table. * For performance reasons, you should define indexes on the SQL tables on the relational database side, diff --git a/docs/studio/database/tasks/import-data/import-from-sql.mdx b/docs/studio/database/tasks/import-data/import-from-sql.mdx index c0ea1b447c..890cff3335 100644 --- a/docs/studio/database/tasks/import-data/import-from-sql.mdx +++ b/docs/studio/database/tasks/import-data/import-from-sql.mdx @@ -69,11 +69,11 @@ The configuration can then be exported and [reused](../../../../studio/database/ 1. **SQL database driver** Select one of the available database drivers: - * Microsoft SQL Server (System.Data.SqlClient) + * Microsoft SQL Server (Microsoft.Data.SqlClient) * MySQL Server (MySqlConnector.MySqlConnectorFactory) - * MySqlConnector.MySqlConnectorFactory * PostgreSQL (Npgsql) * Oracle Database (Oracle.ManagedDataAccess.Client) + * DEPRECATED: MySQL Server (MySql.Data.MySqlClient) 2. **Connection string** Provide a connection string to the data origin server. E.g., the connection string for a local mySQL database can be: diff --git a/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx b/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx index 94a8e04372..6271bdc5a8 100644 --- a/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx +++ b/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx @@ -76,6 +76,23 @@ Define the target tables where the SQL ETL task will load data. * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. + + + The document ID column should use a **case-insensitive collation** in the destination database. + + When a document is deleted, RavenDB lowercases its ID in the generated DELETE + statement, for example: + `DELETE FROM "Orders" WHERE "Id" IN ('orders/1-a')` + + If the document ID column uses a case-sensitive collation, this lowercase ID will not + match the row that was stored with its original casing (such as `Orders/1-A`), so the + DELETE removes nothing and the row remains in the table as an orphaned record. + + This affects deletions only: when a document is created or updated, RavenDB sends the + ID in its original casing, so the row matches and is removed or replaced correctly. + + + * Note that the specified column does not need to be the primary key of the table. * For performance reasons, you should define indexes on the SQL tables on the relational database side, diff --git a/versioned_docs/version-6.2/studio/database/tasks/import-data/import-from-sql.mdx b/versioned_docs/version-6.2/studio/database/tasks/import-data/import-from-sql.mdx index 01356a2bba..9ac241e99b 100644 --- a/versioned_docs/version-6.2/studio/database/tasks/import-data/import-from-sql.mdx +++ b/versioned_docs/version-6.2/studio/database/tasks/import-data/import-from-sql.mdx @@ -68,11 +68,11 @@ The configuration can then be exported and [reused](../../../../studio/database/ 1. **SQL database driver** Select one of the available database drivers: - * Microsoft SQL Server (System.Data.SqlClient) + * Microsoft SQL Server (Microsoft.Data.SqlClient) * MySQL Server (MySqlConnector.MySqlConnectorFactory) - * MySqlConnector.MySqlConnectorFactory * PostgreSQL (Npgsql) * Oracle Database (Oracle.ManagedDataAccess.Client) + * DEPRECATED: MySQL Server (MySql.Data.MySqlClient) 2. **Connection string** Provide a connection string to the data origin server. E.g., the connection string for a local mySQL database can be: diff --git a/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx b/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx index 94a8e04372..6271bdc5a8 100644 --- a/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx +++ b/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx @@ -76,6 +76,23 @@ Define the target tables where the SQL ETL task will load data. * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. + + + The document ID column should use a **case-insensitive collation** in the destination database. + + When a document is deleted, RavenDB lowercases its ID in the generated DELETE + statement, for example: + `DELETE FROM "Orders" WHERE "Id" IN ('orders/1-a')` + + If the document ID column uses a case-sensitive collation, this lowercase ID will not + match the row that was stored with its original casing (such as `Orders/1-A`), so the + DELETE removes nothing and the row remains in the table as an orphaned record. + + This affects deletions only: when a document is created or updated, RavenDB sends the + ID in its original casing, so the row matches and is removed or replaced correctly. + + + * Note that the specified column does not need to be the primary key of the table. * For performance reasons, you should define indexes on the SQL tables on the relational database side, diff --git a/versioned_docs/version-7.0/studio/database/tasks/import-data/import-from-sql.mdx b/versioned_docs/version-7.0/studio/database/tasks/import-data/import-from-sql.mdx index 01356a2bba..9ac241e99b 100644 --- a/versioned_docs/version-7.0/studio/database/tasks/import-data/import-from-sql.mdx +++ b/versioned_docs/version-7.0/studio/database/tasks/import-data/import-from-sql.mdx @@ -68,11 +68,11 @@ The configuration can then be exported and [reused](../../../../studio/database/ 1. **SQL database driver** Select one of the available database drivers: - * Microsoft SQL Server (System.Data.SqlClient) + * Microsoft SQL Server (Microsoft.Data.SqlClient) * MySQL Server (MySqlConnector.MySqlConnectorFactory) - * MySqlConnector.MySqlConnectorFactory * PostgreSQL (Npgsql) * Oracle Database (Oracle.ManagedDataAccess.Client) + * DEPRECATED: MySQL Server (MySql.Data.MySqlClient) 2. **Connection string** Provide a connection string to the data origin server. E.g., the connection string for a local mySQL database can be: diff --git a/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx b/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx index 7dc61f1cc9..bfb367ff37 100644 --- a/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx +++ b/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx @@ -76,6 +76,23 @@ Define the target tables where the SQL ETL task will load data. * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. + + + The document ID column should use a **case-insensitive collation** in the destination database. + + When a document is deleted, RavenDB lowercases its ID in the generated DELETE + statement, for example: + `DELETE FROM "Orders" WHERE "Id" IN ('orders/1-a')` + + If the document ID column uses a case-sensitive collation, this lowercase ID will not + match the row that was stored with its original casing (such as `Orders/1-A`), so the + DELETE removes nothing and the row remains in the table as an orphaned record. + + This affects deletions only: when a document is created or updated, RavenDB sends the + ID in its original casing, so the row matches and is removed or replaced correctly. + + + * Note that the specified column does not need to be the primary key of the table. * For performance reasons, you should define indexes on the SQL tables on the relational database side, diff --git a/versioned_docs/version-7.1/studio/database/tasks/import-data/import-from-sql.mdx b/versioned_docs/version-7.1/studio/database/tasks/import-data/import-from-sql.mdx index 01356a2bba..9ac241e99b 100644 --- a/versioned_docs/version-7.1/studio/database/tasks/import-data/import-from-sql.mdx +++ b/versioned_docs/version-7.1/studio/database/tasks/import-data/import-from-sql.mdx @@ -68,11 +68,11 @@ The configuration can then be exported and [reused](../../../../studio/database/ 1. **SQL database driver** Select one of the available database drivers: - * Microsoft SQL Server (System.Data.SqlClient) + * Microsoft SQL Server (Microsoft.Data.SqlClient) * MySQL Server (MySqlConnector.MySqlConnectorFactory) - * MySqlConnector.MySqlConnectorFactory * PostgreSQL (Npgsql) * Oracle Database (Oracle.ManagedDataAccess.Client) + * DEPRECATED: MySQL Server (MySql.Data.MySqlClient) 2. **Connection string** Provide a connection string to the data origin server. E.g., the connection string for a local mySQL database can be: From b24e17ce3ccc03c71c72f7641c64769a20a5ea69 Mon Sep 17 00:00:00 2001 From: reeb Date: Wed, 3 Jun 2026 00:20:31 +0300 Subject: [PATCH 2/2] RDoc-2506 - update Studio SQL ETL page layout --- docs/server/ongoing-tasks/etl/sql.mdx | 359 +++++++++-------- .../server/ongoing-tasks/etl/sql.mdx | 362 ++++++++++-------- .../server/ongoing-tasks/etl/sql.mdx | 362 ++++++++++-------- .../server/ongoing-tasks/etl/sql.mdx | 360 +++++++++-------- 4 files changed, 806 insertions(+), 637 deletions(-) diff --git a/docs/server/ongoing-tasks/etl/sql.mdx b/docs/server/ongoing-tasks/etl/sql.mdx index 9582f4b0a4..50d08c6960 100644 --- a/docs/server/ongoing-tasks/etl/sql.mdx +++ b/docs/server/ongoing-tasks/etl/sql.mdx @@ -6,37 +6,37 @@ sidebar_position: 2 --- import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; # Ongoing Tasks: SQL ETL * **SQL ETL** is a task that creates an [ETL process](../../../server/ongoing-tasks/etl/basics.mdx) where data from a RavenDB database is extracted, transformed, and loaded into a relational database as the destination. -* In this page: - * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) - * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) - * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) - * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) - * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#themethod) - * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) - * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) - * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) - * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) - * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) - * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) - * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) - * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) - * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) - * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) - * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) +* In this article: + * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) + * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) + * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) + * [Document ID column](../../../server/ongoing-tasks/etl/sql.mdx#document-id-column) + * [Insert only](../../../server/ongoing-tasks/etl/sql.mdx#insert-only) + * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) + * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#the-loadto-method) + * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) + * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) + * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) + * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) + * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) + * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) + * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) + * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) + * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) + * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) + * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) -## Supported relational databases + + * RavenDB supports ETL processes to the following relational databases: * Microsoft SQL Server @@ -55,22 +55,26 @@ import LanguageContent from "@site/src/components/LanguageContent"; * Before starting with SQL ETL, you need to create tables in the target relational database. These tables will serve as the destinations for records generated by the ETL scripts. -## Creating the SQL ETL task + + + To create an SQL ETL task using the Client API, see [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api). -To create an SQL ETL task from the Studio open `Tasks -> Ongoing Tasks`. +To create an SQL ETL task from Studio, open `Tasks` > `Ongoing Tasks`. ![Configure SQL ETL task](./assets/sql-etl-setup.png) + - -## Configuring the SQL tables + Define the target tables where the SQL ETL task will load data. ![Define SQL tables](./assets/sql-etl-tables.png) -#### Document ID Column + + +### Document ID column * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. @@ -97,16 +101,22 @@ Define the target tables where the SQL ETL task will load data. * For performance reasons, you should define indexes on the SQL tables on the relational database side, at least on the column used to store the document ID. -#### Insert only + + + + +### Insert only * The SQL ETL process updates documents in the relational database using DELETE and INSERT statements. * If your system is _append-only_, you can enable the "Insert Only" toggle to instruct RavenDB to insert data without executing DELETE statements beforehand. This can provide a significant performance boost for systems of this kind. + + -## Transformation scripts + The [basic characteristics](../../../server/ongoing-tasks/etl/basics.mdx) of an SQL ETL script are similar to those of other ETL types. The script defines what data to **extract** from the source document, how to **transform** this data, @@ -115,14 +125,17 @@ and which SQL table to **load** it to. A single SQL ETL task can have multiple transformation scripts. The script is defined per collection, and it cannot be empty. The script is executed per document from the source collection once the document is created or modified. -### The `loadTo` method + + + +### The `loadTo` method To specify which SQL table to load the data into, use either of the following methods in your script. The two methods are equivalent, offering alternative syntax: * **`loadTo(obj)`** * Here the target table is specified as part of the function name. - * The target _<TableName>_ in this syntax is Not a variable and cannot be used as one, + * The target _<TableName>_ in this syntax is not a variable and cannot be used as one, it is simply a string literal of the target's name. * **`loadTo('TableName', obj)`** @@ -130,6 +143,8 @@ The two methods are equivalent, offering alternative syntax: * Separating the table name from the `loadTo` command makes it possible to include symbols like `'-'` and `'.'` in table names. This is not possible when the `loadTo` syntax is used because including special characters in the name of a JavaScript function makes it invalid. +
+ | Parameter | Type | Description | |-----------------|--------|----------------------------------------------------------------------------------------------------------------------------------| | **TableName** | string | The name of the target SQL table | @@ -139,6 +154,11 @@ For example, the following two calls, which load data to "OrdersTable", are equi * `loadToOrdersTable(obj)` * `loadTo('OrdersTable', obj)` + + + + + ### Loading to multiple tables The `loadTo` method can be called multiple times in a single script. @@ -146,91 +166,97 @@ That allows you to split a single `Order` document having `Lines` collection int The following is a sample script that processes documents from the Orders collection: - - -{`// Create an orderData object +```javascript +// Create an orderData object // ========================== -var orderData = \{ +var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 -\}; +}; // Update the orderData's TotalCost field // ====================================== -for (var i = 0; i < this.Lines.length; i++) \{ +for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - // Load the object to SQL table 'OrdersTable' + // Load the object to SQL table 'OrderLines' // ========================================== - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); -\} + }); +} orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; // Load to SQL table 'Orders' // ========================== loadToOrders(orderData); -`} - - +``` + + + + + ### Loading related documents Use the `load` method to load a related document with the specified ID during script execution. - - -{`var company = load(this.Company); -`} - - -### Loading Attachments +```javascript +var company = load(this.Company); +``` + + + + + +### Loading attachments You can store binary data that is kept as attachments in RavenDB using the `loadAttachment()` method. For example, if you have the following _Attachments_ table: - - -{`CREATE TABLE [dbo].[Attachments] +```sql +CREATE TABLE [dbo].[Attachments] ( [Id] int identity primary key, [OrderId] [nvarchar](50) NOT NULL, [AttachmentName] [nvarchar](50) NULL, [Data] [varbinary](max) NULL ) -`} - - +``` + +
then you can define the following script that loads the document's attachments: - - -{`var attachments = this['@metadata']['@attachments']; +```javascript +var attachments = this['@metadata']['@attachments']; -for (var i = 0; i < attachments.length; i++) \{ - var attachment = \{ +for (var i = 0; i < attachments.length; i++) { + var attachment = { OrderId: id(this), AttachmentName: attachments[i].Name, Data: loadAttachment(attachments[i].Name) - \}; + }; loadToAttachments(attachment); -\} -`} - - +} +``` + +
-* Attachments can be also accessed using the `getAttachments()` helper function +* Attachments can be also accessed using the `getAttachments()` helper function (instead of grabbing them from metadata). * The existence of an attachment can be checked by the `hasAttachment(name)` function. + + + + ### Loading to VARCHAR and NVARCHAR columns Two additional functions are designed specifically for working with VARCHAR and NVARCHAR types: @@ -240,96 +266,108 @@ Two additional functions are designed specifically for working with VARCHAR and | `varchar(value, size = 50)` | Defines the parameter type as VARCHAR, with the option to specify its size
(default is 50 if not provided). | | `nvarchar(value, size = 50)` | Defines the parameter type as NVARCHAR, with the option to specify its size
(default is 50 if not specified). | - - -{`var names = this.Name.split(' '); +```javascript +var names = this.Name.split(' '); loadToUsers( -\{ +{ FirstName: varchar(names[0], 30), LastName: nvarchar(names[1]), -\}); -`} - - +}); +``` + + + + + ### Loading to specific column types The SQL type of the target column can be explicitly specified in the SQL ETL script. This is done by defining the `Type` and the `Value` properties for the data being loaded. - * **Type**: - The type specifies the SQL column type the value is loaded to. - The type should correspond to the data types used in the target relational database. +* **Type**: + The type specifies the SQL column type the value is loaded to. + The type should correspond to the data types used in the target relational database. + + Supported enums for `Type` include: + * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) + * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) + * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) + * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - Supported enums for `Type` include: - * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) - * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) - * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) - * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - - Some databases allow combining enum values using `|`. - For example, using `Array | Double` for the Type is valid for PostgreSQL. + Some databases allow combining enum values using `|`. + For example, using `Array | Double` for the Type is valid for PostgreSQL. - If no type is specified, the column type will be detected automatically. + If no type is specified, the column type will be detected automatically. - * **Value**: - The value contains the actual data to be loaded into the column. +* **Value**: + The value contains the actual data to be loaded into the column. - - -{`var orderData = \{ +In the following script, the `Type` and `Value` are specified for the `Quantities` and `Products` columns: + +```javascript +var orderData = { Id: id(this), OrderLinesCount: this.OrderLines.length, - Quantities: \{ + Quantities: { // Specify the Type and Value for 'Quantities': // ============================================ Type: 'Array | Double', - Value: this.OrderLines.map(function(l) \{return l.Quantity;\}) - \}, - Products: \{ + Value: this.OrderLines.map(function(l) {return l.Quantity;}) + }, + Products: { // Specify the Type and Value for 'Products': // ========================================== Type: 'Array | Text', - Value: this.OrderLines.map(function(l) \{return l.Product;\}) - \}, -\}; + Value: this.OrderLines.map(function(l) {return l.Product;}) + }, +}; // Load the data into the 'Orders' table loadToOrders(orderData); -`} - - +``` + + + + + ### Filtering To filter some documents out from the ETL, simply omit the `loadTo` call: - - -{`if (this.ShipTo.Country === 'USA') \{ +```javascript +if (this.ShipTo.Country === 'USA') { // Load only orders shipped to USA - loadToOrders(\{ ... \}); -\} -`} - - + loadToOrders({ ... }); +} +``` + + + + + ### Accessing the metadata You can access the metadata in the following way: - - -{`var value = this['@metadata']['custom-metadata-key']; -`} - - +```javascript +var value = this['@metadata']['custom-metadata-key']; +``` + + + + + ### Document extensions The SQL ETL task does not support sending [Counters](../../../document-extensions/counters/overview.mdx), [Time series](../../../document-extensions/timeseries/overview.mdx), or [Revisions](../../../document-extensions/revisions/overview.mdx). + + -## Advanced options + * **Command timeout** Number of seconds after which SQL command will timeout. @@ -346,22 +384,25 @@ The SQL ETL task does not support sending [Counters](../../../document-extension Control whether to force the SQL Server to recompile the query statement using (`OPTION(RECOMPILE)`). Default: `false`. + - -## Transaction processing + All records created in a single ETL run, one for each `loadTo` call, are sent in a single batch and processed within the same transaction. + + -## Creating the SQL ETL task from the Client API +Creating an SQL ETL task from the Client API takes two steps. - - -{`// Define a connection string to a SQL database destination +First, define and deploy a connection string to the destination database: + +```csharp +// Define a connection string to a SQL database destination // ======================================================== var sqlConStr = new SqlConnectionString -\{ +{ Name = "sql-connection-string-name", // Define destination factory name @@ -371,79 +412,82 @@ var sqlConStr = new SqlConnectionString // May also need to define authentication and encryption parameters // By default, encrypted databases are sent over encrypted channels ConnectionString = "host=127.0.0.1;user=root;database=Northwind" -\}; - +}; + // Deploy (send) the connection string to the server via the PutConnectionStringOperation // ====================================================================================== var PutConnectionStringOp = new PutConnectionStringOperation(sqlConStr); PutConnectionStringResult connectionStringResult = store.Maintenance.Send(PutConnectionStringOp); -`} - - - - -{`// Define the SQL ETL task configuration +``` + +
+ +Then define and deploy the SQL ETL task: + +```csharp +// Define the SQL ETL task configuration // ===================================== var sqlConfiguration = new SqlEtlConfiguration() -\{ +{ Name = "mySqlEtlTaskName", ConnectionStringName = "sql-connection-string-name", SqlTables = - \{ + { new SqlEtlTable - \{ + { TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false - \}, + }, new SqlEtlTable - \{ + { TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false - \}, - \}, + }, + }, Transforms = - \{ + { new Transformation() - \{ + { Name = "scriptName", - Collections = \{ "Orders" \}, + Collections = { "Orders" }, Script = @" - var orderData = \{ + var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 - \}; + }; - for (var i = 0; i < this.Lines.length; i++) \{ + for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); - \} + }); + } orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; loadToOrders(orderData); ", ApplyToAllDocuments = false - \} - \} -\}; + } + } +}; // Deploy the SQL ETL task to the server // ===================================== var addSqlEtlOperation = new AddEtlOperation(sqlConfiguration); store.Maintenance.Send(addSqlEtlOperation); -`} - - +``` + +
+ `SqlEtlConfiguration`: | Property | Type | Description | @@ -465,5 +509,4 @@ store.Maintenance.Send(addSqlEtlOperation); | **DocumentIdColumn** | `string` | The column in the destination table that will store the document IDs. | | **InsertOnlyMode** | `bool` | When set to `true`, RavenDB will insert data directly without executing DELETE statements beforehand.
Default is `false`. | - - + diff --git a/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx b/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx index 6271bdc5a8..50d08c6960 100644 --- a/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx +++ b/versioned_docs/version-6.2/server/ongoing-tasks/etl/sql.mdx @@ -1,41 +1,42 @@ --- title: "Ongoing Tasks: SQL ETL" sidebar_label: SQL ETL +description: "Replicate RavenDB documents to relational SQL databases using ETL tasks with table mapping and JavaScript transformations." sidebar_position: 2 --- import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; # Ongoing Tasks: SQL ETL * **SQL ETL** is a task that creates an [ETL process](../../../server/ongoing-tasks/etl/basics.mdx) where data from a RavenDB database is extracted, transformed, and loaded into a relational database as the destination. -* In this page: - * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) - * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) - * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) - * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) - * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#themethod) - * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) - * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) - * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) - * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) - * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) - * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) - * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) - * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) - * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) - * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) - * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) +* In this article: + * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) + * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) + * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) + * [Document ID column](../../../server/ongoing-tasks/etl/sql.mdx#document-id-column) + * [Insert only](../../../server/ongoing-tasks/etl/sql.mdx#insert-only) + * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) + * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#the-loadto-method) + * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) + * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) + * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) + * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) + * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) + * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) + * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) + * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) + * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) + * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) + * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) -## Supported relational databases + + * RavenDB supports ETL processes to the following relational databases: * Microsoft SQL Server @@ -50,28 +51,30 @@ import LanguageContent from "@site/src/components/LanguageContent"; For MySQL, the `MySql.Data.MySqlClient` factory name is deprecated. Use `MySqlConnector.MySqlConnectorFactory` instead. Existing configurations that specify `MySql.Data.MySqlClient` continue to work; RavenDB remaps the factory name automatically to maintain backward compatibility. - + * Before starting with SQL ETL, you need to create tables in the target relational database. These tables will serve as the destinations for records generated by the ETL scripts. + - -## Creating the SQL ETL task + To create an SQL ETL task using the Client API, see [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api). -To create an SQL ETL task from the Studio open `Tasks -> Ongoing Tasks`. +To create an SQL ETL task from Studio, open `Tasks` > `Ongoing Tasks`. ![Configure SQL ETL task](./assets/sql-etl-setup.png) + - -## Configuring the SQL tables + Define the target tables where the SQL ETL task will load data. ![Define SQL tables](./assets/sql-etl-tables.png) -#### Document ID Column + + +### Document ID column * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. @@ -98,16 +101,22 @@ Define the target tables where the SQL ETL task will load data. * For performance reasons, you should define indexes on the SQL tables on the relational database side, at least on the column used to store the document ID. -#### Insert only + + + + +### Insert only * The SQL ETL process updates documents in the relational database using DELETE and INSERT statements. * If your system is _append-only_, you can enable the "Insert Only" toggle to instruct RavenDB to insert data without executing DELETE statements beforehand. This can provide a significant performance boost for systems of this kind. + + -## Transformation scripts + The [basic characteristics](../../../server/ongoing-tasks/etl/basics.mdx) of an SQL ETL script are similar to those of other ETL types. The script defines what data to **extract** from the source document, how to **transform** this data, @@ -116,14 +125,17 @@ and which SQL table to **load** it to. A single SQL ETL task can have multiple transformation scripts. The script is defined per collection, and it cannot be empty. The script is executed per document from the source collection once the document is created or modified. -### The `loadTo` method + + + +### The `loadTo` method To specify which SQL table to load the data into, use either of the following methods in your script. The two methods are equivalent, offering alternative syntax: * **`loadTo(obj)`** * Here the target table is specified as part of the function name. - * The target _<TableName>_ in this syntax is Not a variable and cannot be used as one, + * The target _<TableName>_ in this syntax is not a variable and cannot be used as one, it is simply a string literal of the target's name. * **`loadTo('TableName', obj)`** @@ -131,6 +143,8 @@ The two methods are equivalent, offering alternative syntax: * Separating the table name from the `loadTo` command makes it possible to include symbols like `'-'` and `'.'` in table names. This is not possible when the `loadTo` syntax is used because including special characters in the name of a JavaScript function makes it invalid. +
+ | Parameter | Type | Description | |-----------------|--------|----------------------------------------------------------------------------------------------------------------------------------| | **TableName** | string | The name of the target SQL table | @@ -140,6 +154,11 @@ For example, the following two calls, which load data to "OrdersTable", are equi * `loadToOrdersTable(obj)` * `loadTo('OrdersTable', obj)` + + + + + ### Loading to multiple tables The `loadTo` method can be called multiple times in a single script. @@ -147,91 +166,97 @@ That allows you to split a single `Order` document having `Lines` collection int The following is a sample script that processes documents from the Orders collection: - - -{`// Create an orderData object +```javascript +// Create an orderData object // ========================== -var orderData = \{ +var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 -\}; +}; // Update the orderData's TotalCost field // ====================================== -for (var i = 0; i < this.Lines.length; i++) \{ +for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - // Load the object to SQL table 'OrdersTable' + // Load the object to SQL table 'OrderLines' // ========================================== - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); -\} + }); +} orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; // Load to SQL table 'Orders' // ========================== loadToOrders(orderData); -`} - - +``` + + + + + ### Loading related documents Use the `load` method to load a related document with the specified ID during script execution. - - -{`var company = load(this.Company); -`} - - -### Loading Attachments +```javascript +var company = load(this.Company); +``` + + + + + +### Loading attachments You can store binary data that is kept as attachments in RavenDB using the `loadAttachment()` method. For example, if you have the following _Attachments_ table: - - -{`CREATE TABLE [dbo].[Attachments] +```sql +CREATE TABLE [dbo].[Attachments] ( [Id] int identity primary key, [OrderId] [nvarchar](50) NOT NULL, [AttachmentName] [nvarchar](50) NULL, [Data] [varbinary](max) NULL ) -`} - - +``` + +
then you can define the following script that loads the document's attachments: - - -{`var attachments = this['@metadata']['@attachments']; +```javascript +var attachments = this['@metadata']['@attachments']; -for (var i = 0; i < attachments.length; i++) \{ - var attachment = \{ +for (var i = 0; i < attachments.length; i++) { + var attachment = { OrderId: id(this), AttachmentName: attachments[i].Name, Data: loadAttachment(attachments[i].Name) - \}; + }; loadToAttachments(attachment); -\} -`} - - +} +``` + +
-* Attachments can be also accessed using the `getAttachments()` helper function +* Attachments can be also accessed using the `getAttachments()` helper function (instead of grabbing them from metadata). * The existence of an attachment can be checked by the `hasAttachment(name)` function. + + + + ### Loading to VARCHAR and NVARCHAR columns Two additional functions are designed specifically for working with VARCHAR and NVARCHAR types: @@ -241,96 +266,108 @@ Two additional functions are designed specifically for working with VARCHAR and | `varchar(value, size = 50)` | Defines the parameter type as VARCHAR, with the option to specify its size
(default is 50 if not provided). | | `nvarchar(value, size = 50)` | Defines the parameter type as NVARCHAR, with the option to specify its size
(default is 50 if not specified). | - - -{`var names = this.Name.split(' '); +```javascript +var names = this.Name.split(' '); loadToUsers( -\{ +{ FirstName: varchar(names[0], 30), LastName: nvarchar(names[1]), -\}); -`} - - +}); +``` + + + + + ### Loading to specific column types The SQL type of the target column can be explicitly specified in the SQL ETL script. This is done by defining the `Type` and the `Value` properties for the data being loaded. - * **Type**: - The type specifies the SQL column type the value is loaded to. - The type should correspond to the data types used in the target relational database. +* **Type**: + The type specifies the SQL column type the value is loaded to. + The type should correspond to the data types used in the target relational database. - Supported enums for `Type` include: - * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) - * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) - * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) - * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - - Some databases allow combining enum values using `|`. - For example, using `Array | Double` for the Type is valid for PostgreSQL. + Supported enums for `Type` include: + * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) + * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) + * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) + * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - If no type is specified, the column type will be detected automatically. + Some databases allow combining enum values using `|`. + For example, using `Array | Double` for the Type is valid for PostgreSQL. - * **Value**: - The value contains the actual data to be loaded into the column. + If no type is specified, the column type will be detected automatically. - - -{`var orderData = \{ +* **Value**: + The value contains the actual data to be loaded into the column. + +In the following script, the `Type` and `Value` are specified for the `Quantities` and `Products` columns: + +```javascript +var orderData = { Id: id(this), OrderLinesCount: this.OrderLines.length, - Quantities: \{ + Quantities: { // Specify the Type and Value for 'Quantities': // ============================================ Type: 'Array | Double', - Value: this.OrderLines.map(function(l) \{return l.Quantity;\}) - \}, - Products: \{ + Value: this.OrderLines.map(function(l) {return l.Quantity;}) + }, + Products: { // Specify the Type and Value for 'Products': // ========================================== Type: 'Array | Text', - Value: this.OrderLines.map(function(l) \{return l.Product;\}) - \}, -\}; + Value: this.OrderLines.map(function(l) {return l.Product;}) + }, +}; // Load the data into the 'Orders' table loadToOrders(orderData); -`} - - +``` + + + + + ### Filtering To filter some documents out from the ETL, simply omit the `loadTo` call: - - -{`if (this.ShipTo.Country === 'USA') \{ +```javascript +if (this.ShipTo.Country === 'USA') { // Load only orders shipped to USA - loadToOrders(\{ ... \}); -\} -`} - - + loadToOrders({ ... }); +} +``` + + + + + ### Accessing the metadata You can access the metadata in the following way: - - -{`var value = this['@metadata']['custom-metadata-key']; -`} - - +```javascript +var value = this['@metadata']['custom-metadata-key']; +``` + + + + + ### Document extensions The SQL ETL task does not support sending [Counters](../../../document-extensions/counters/overview.mdx), [Time series](../../../document-extensions/timeseries/overview.mdx), or [Revisions](../../../document-extensions/revisions/overview.mdx). + + -## Advanced options + * **Command timeout** Number of seconds after which SQL command will timeout. @@ -347,22 +384,25 @@ The SQL ETL task does not support sending [Counters](../../../document-extension Control whether to force the SQL Server to recompile the query statement using (`OPTION(RECOMPILE)`). Default: `false`. + - -## Transaction processing + All records created in a single ETL run, one for each `loadTo` call, are sent in a single batch and processed within the same transaction. + + -## Creating the SQL ETL task from the Client API +Creating an SQL ETL task from the Client API takes two steps. - - -{`// Define a connection string to a SQL database destination +First, define and deploy a connection string to the destination database: + +```csharp +// Define a connection string to a SQL database destination // ======================================================== var sqlConStr = new SqlConnectionString -\{ +{ Name = "sql-connection-string-name", // Define destination factory name @@ -372,79 +412,82 @@ var sqlConStr = new SqlConnectionString // May also need to define authentication and encryption parameters // By default, encrypted databases are sent over encrypted channels ConnectionString = "host=127.0.0.1;user=root;database=Northwind" -\}; - +}; + // Deploy (send) the connection string to the server via the PutConnectionStringOperation // ====================================================================================== var PutConnectionStringOp = new PutConnectionStringOperation(sqlConStr); PutConnectionStringResult connectionStringResult = store.Maintenance.Send(PutConnectionStringOp); -`} - - - - -{`// Define the SQL ETL task configuration +``` + +
+ +Then define and deploy the SQL ETL task: + +```csharp +// Define the SQL ETL task configuration // ===================================== var sqlConfiguration = new SqlEtlConfiguration() -\{ +{ Name = "mySqlEtlTaskName", ConnectionStringName = "sql-connection-string-name", SqlTables = - \{ + { new SqlEtlTable - \{ + { TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false - \}, + }, new SqlEtlTable - \{ + { TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false - \}, - \}, + }, + }, Transforms = - \{ + { new Transformation() - \{ + { Name = "scriptName", - Collections = \{ "Orders" \}, + Collections = { "Orders" }, Script = @" - var orderData = \{ + var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 - \}; + }; - for (var i = 0; i < this.Lines.length; i++) \{ + for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); - \} + }); + } orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; loadToOrders(orderData); ", ApplyToAllDocuments = false - \} - \} -\}; + } + } +}; // Deploy the SQL ETL task to the server // ===================================== var addSqlEtlOperation = new AddEtlOperation(sqlConfiguration); store.Maintenance.Send(addSqlEtlOperation); -`} - - +``` + +
+ `SqlEtlConfiguration`: | Property | Type | Description | @@ -466,5 +509,4 @@ store.Maintenance.Send(addSqlEtlOperation); | **DocumentIdColumn** | `string` | The column in the destination table that will store the document IDs. | | **InsertOnlyMode** | `bool` | When set to `true`, RavenDB will insert data directly without executing DELETE statements beforehand.
Default is `false`. | - - + diff --git a/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx b/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx index 6271bdc5a8..50d08c6960 100644 --- a/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx +++ b/versioned_docs/version-7.0/server/ongoing-tasks/etl/sql.mdx @@ -1,41 +1,42 @@ --- title: "Ongoing Tasks: SQL ETL" sidebar_label: SQL ETL +description: "Replicate RavenDB documents to relational SQL databases using ETL tasks with table mapping and JavaScript transformations." sidebar_position: 2 --- import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; # Ongoing Tasks: SQL ETL * **SQL ETL** is a task that creates an [ETL process](../../../server/ongoing-tasks/etl/basics.mdx) where data from a RavenDB database is extracted, transformed, and loaded into a relational database as the destination. -* In this page: - * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) - * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) - * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) - * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) - * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#themethod) - * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) - * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) - * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) - * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) - * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) - * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) - * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) - * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) - * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) - * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) - * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) +* In this article: + * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) + * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) + * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) + * [Document ID column](../../../server/ongoing-tasks/etl/sql.mdx#document-id-column) + * [Insert only](../../../server/ongoing-tasks/etl/sql.mdx#insert-only) + * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) + * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#the-loadto-method) + * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) + * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) + * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) + * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) + * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) + * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) + * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) + * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) + * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) + * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) + * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) -## Supported relational databases + + * RavenDB supports ETL processes to the following relational databases: * Microsoft SQL Server @@ -50,28 +51,30 @@ import LanguageContent from "@site/src/components/LanguageContent"; For MySQL, the `MySql.Data.MySqlClient` factory name is deprecated. Use `MySqlConnector.MySqlConnectorFactory` instead. Existing configurations that specify `MySql.Data.MySqlClient` continue to work; RavenDB remaps the factory name automatically to maintain backward compatibility. - + * Before starting with SQL ETL, you need to create tables in the target relational database. These tables will serve as the destinations for records generated by the ETL scripts. + - -## Creating the SQL ETL task + To create an SQL ETL task using the Client API, see [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api). -To create an SQL ETL task from the Studio open `Tasks -> Ongoing Tasks`. +To create an SQL ETL task from Studio, open `Tasks` > `Ongoing Tasks`. ![Configure SQL ETL task](./assets/sql-etl-setup.png) + - -## Configuring the SQL tables + Define the target tables where the SQL ETL task will load data. ![Define SQL tables](./assets/sql-etl-tables.png) -#### Document ID Column + + +### Document ID column * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. @@ -98,16 +101,22 @@ Define the target tables where the SQL ETL task will load data. * For performance reasons, you should define indexes on the SQL tables on the relational database side, at least on the column used to store the document ID. -#### Insert only + + + + +### Insert only * The SQL ETL process updates documents in the relational database using DELETE and INSERT statements. * If your system is _append-only_, you can enable the "Insert Only" toggle to instruct RavenDB to insert data without executing DELETE statements beforehand. This can provide a significant performance boost for systems of this kind. + + -## Transformation scripts + The [basic characteristics](../../../server/ongoing-tasks/etl/basics.mdx) of an SQL ETL script are similar to those of other ETL types. The script defines what data to **extract** from the source document, how to **transform** this data, @@ -116,14 +125,17 @@ and which SQL table to **load** it to. A single SQL ETL task can have multiple transformation scripts. The script is defined per collection, and it cannot be empty. The script is executed per document from the source collection once the document is created or modified. -### The `loadTo` method + + + +### The `loadTo` method To specify which SQL table to load the data into, use either of the following methods in your script. The two methods are equivalent, offering alternative syntax: * **`loadTo(obj)`** * Here the target table is specified as part of the function name. - * The target _<TableName>_ in this syntax is Not a variable and cannot be used as one, + * The target _<TableName>_ in this syntax is not a variable and cannot be used as one, it is simply a string literal of the target's name. * **`loadTo('TableName', obj)`** @@ -131,6 +143,8 @@ The two methods are equivalent, offering alternative syntax: * Separating the table name from the `loadTo` command makes it possible to include symbols like `'-'` and `'.'` in table names. This is not possible when the `loadTo` syntax is used because including special characters in the name of a JavaScript function makes it invalid. +
+ | Parameter | Type | Description | |-----------------|--------|----------------------------------------------------------------------------------------------------------------------------------| | **TableName** | string | The name of the target SQL table | @@ -140,6 +154,11 @@ For example, the following two calls, which load data to "OrdersTable", are equi * `loadToOrdersTable(obj)` * `loadTo('OrdersTable', obj)` + + + + + ### Loading to multiple tables The `loadTo` method can be called multiple times in a single script. @@ -147,91 +166,97 @@ That allows you to split a single `Order` document having `Lines` collection int The following is a sample script that processes documents from the Orders collection: - - -{`// Create an orderData object +```javascript +// Create an orderData object // ========================== -var orderData = \{ +var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 -\}; +}; // Update the orderData's TotalCost field // ====================================== -for (var i = 0; i < this.Lines.length; i++) \{ +for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - // Load the object to SQL table 'OrdersTable' + // Load the object to SQL table 'OrderLines' // ========================================== - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); -\} + }); +} orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; // Load to SQL table 'Orders' // ========================== loadToOrders(orderData); -`} - - +``` + + + + + ### Loading related documents Use the `load` method to load a related document with the specified ID during script execution. - - -{`var company = load(this.Company); -`} - - -### Loading Attachments +```javascript +var company = load(this.Company); +``` + + + + + +### Loading attachments You can store binary data that is kept as attachments in RavenDB using the `loadAttachment()` method. For example, if you have the following _Attachments_ table: - - -{`CREATE TABLE [dbo].[Attachments] +```sql +CREATE TABLE [dbo].[Attachments] ( [Id] int identity primary key, [OrderId] [nvarchar](50) NOT NULL, [AttachmentName] [nvarchar](50) NULL, [Data] [varbinary](max) NULL ) -`} - - +``` + +
then you can define the following script that loads the document's attachments: - - -{`var attachments = this['@metadata']['@attachments']; +```javascript +var attachments = this['@metadata']['@attachments']; -for (var i = 0; i < attachments.length; i++) \{ - var attachment = \{ +for (var i = 0; i < attachments.length; i++) { + var attachment = { OrderId: id(this), AttachmentName: attachments[i].Name, Data: loadAttachment(attachments[i].Name) - \}; + }; loadToAttachments(attachment); -\} -`} - - +} +``` + +
-* Attachments can be also accessed using the `getAttachments()` helper function +* Attachments can be also accessed using the `getAttachments()` helper function (instead of grabbing them from metadata). * The existence of an attachment can be checked by the `hasAttachment(name)` function. + + + + ### Loading to VARCHAR and NVARCHAR columns Two additional functions are designed specifically for working with VARCHAR and NVARCHAR types: @@ -241,96 +266,108 @@ Two additional functions are designed specifically for working with VARCHAR and | `varchar(value, size = 50)` | Defines the parameter type as VARCHAR, with the option to specify its size
(default is 50 if not provided). | | `nvarchar(value, size = 50)` | Defines the parameter type as NVARCHAR, with the option to specify its size
(default is 50 if not specified). | - - -{`var names = this.Name.split(' '); +```javascript +var names = this.Name.split(' '); loadToUsers( -\{ +{ FirstName: varchar(names[0], 30), LastName: nvarchar(names[1]), -\}); -`} - - +}); +``` + + + + + ### Loading to specific column types The SQL type of the target column can be explicitly specified in the SQL ETL script. This is done by defining the `Type` and the `Value` properties for the data being loaded. - * **Type**: - The type specifies the SQL column type the value is loaded to. - The type should correspond to the data types used in the target relational database. +* **Type**: + The type specifies the SQL column type the value is loaded to. + The type should correspond to the data types used in the target relational database. - Supported enums for `Type` include: - * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) - * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) - * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) - * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - - Some databases allow combining enum values using `|`. - For example, using `Array | Double` for the Type is valid for PostgreSQL. + Supported enums for `Type` include: + * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) + * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) + * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) + * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - If no type is specified, the column type will be detected automatically. + Some databases allow combining enum values using `|`. + For example, using `Array | Double` for the Type is valid for PostgreSQL. - * **Value**: - The value contains the actual data to be loaded into the column. + If no type is specified, the column type will be detected automatically. - - -{`var orderData = \{ +* **Value**: + The value contains the actual data to be loaded into the column. + +In the following script, the `Type` and `Value` are specified for the `Quantities` and `Products` columns: + +```javascript +var orderData = { Id: id(this), OrderLinesCount: this.OrderLines.length, - Quantities: \{ + Quantities: { // Specify the Type and Value for 'Quantities': // ============================================ Type: 'Array | Double', - Value: this.OrderLines.map(function(l) \{return l.Quantity;\}) - \}, - Products: \{ + Value: this.OrderLines.map(function(l) {return l.Quantity;}) + }, + Products: { // Specify the Type and Value for 'Products': // ========================================== Type: 'Array | Text', - Value: this.OrderLines.map(function(l) \{return l.Product;\}) - \}, -\}; + Value: this.OrderLines.map(function(l) {return l.Product;}) + }, +}; // Load the data into the 'Orders' table loadToOrders(orderData); -`} - - +``` + + + + + ### Filtering To filter some documents out from the ETL, simply omit the `loadTo` call: - - -{`if (this.ShipTo.Country === 'USA') \{ +```javascript +if (this.ShipTo.Country === 'USA') { // Load only orders shipped to USA - loadToOrders(\{ ... \}); -\} -`} - - + loadToOrders({ ... }); +} +``` + + + + + ### Accessing the metadata You can access the metadata in the following way: - - -{`var value = this['@metadata']['custom-metadata-key']; -`} - - +```javascript +var value = this['@metadata']['custom-metadata-key']; +``` + + + + + ### Document extensions The SQL ETL task does not support sending [Counters](../../../document-extensions/counters/overview.mdx), [Time series](../../../document-extensions/timeseries/overview.mdx), or [Revisions](../../../document-extensions/revisions/overview.mdx). + + -## Advanced options + * **Command timeout** Number of seconds after which SQL command will timeout. @@ -347,22 +384,25 @@ The SQL ETL task does not support sending [Counters](../../../document-extension Control whether to force the SQL Server to recompile the query statement using (`OPTION(RECOMPILE)`). Default: `false`. + - -## Transaction processing + All records created in a single ETL run, one for each `loadTo` call, are sent in a single batch and processed within the same transaction. + + -## Creating the SQL ETL task from the Client API +Creating an SQL ETL task from the Client API takes two steps. - - -{`// Define a connection string to a SQL database destination +First, define and deploy a connection string to the destination database: + +```csharp +// Define a connection string to a SQL database destination // ======================================================== var sqlConStr = new SqlConnectionString -\{ +{ Name = "sql-connection-string-name", // Define destination factory name @@ -372,79 +412,82 @@ var sqlConStr = new SqlConnectionString // May also need to define authentication and encryption parameters // By default, encrypted databases are sent over encrypted channels ConnectionString = "host=127.0.0.1;user=root;database=Northwind" -\}; - +}; + // Deploy (send) the connection string to the server via the PutConnectionStringOperation // ====================================================================================== var PutConnectionStringOp = new PutConnectionStringOperation(sqlConStr); PutConnectionStringResult connectionStringResult = store.Maintenance.Send(PutConnectionStringOp); -`} - - - - -{`// Define the SQL ETL task configuration +``` + +
+ +Then define and deploy the SQL ETL task: + +```csharp +// Define the SQL ETL task configuration // ===================================== var sqlConfiguration = new SqlEtlConfiguration() -\{ +{ Name = "mySqlEtlTaskName", ConnectionStringName = "sql-connection-string-name", SqlTables = - \{ + { new SqlEtlTable - \{ + { TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false - \}, + }, new SqlEtlTable - \{ + { TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false - \}, - \}, + }, + }, Transforms = - \{ + { new Transformation() - \{ + { Name = "scriptName", - Collections = \{ "Orders" \}, + Collections = { "Orders" }, Script = @" - var orderData = \{ + var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 - \}; + }; - for (var i = 0; i < this.Lines.length; i++) \{ + for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); - \} + }); + } orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; loadToOrders(orderData); ", ApplyToAllDocuments = false - \} - \} -\}; + } + } +}; // Deploy the SQL ETL task to the server // ===================================== var addSqlEtlOperation = new AddEtlOperation(sqlConfiguration); store.Maintenance.Send(addSqlEtlOperation); -`} - - +``` + +
+ `SqlEtlConfiguration`: | Property | Type | Description | @@ -466,5 +509,4 @@ store.Maintenance.Send(addSqlEtlOperation); | **DocumentIdColumn** | `string` | The column in the destination table that will store the document IDs. | | **InsertOnlyMode** | `bool` | When set to `true`, RavenDB will insert data directly without executing DELETE statements beforehand.
Default is `false`. | - - + diff --git a/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx b/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx index bfb367ff37..50d08c6960 100644 --- a/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx +++ b/versioned_docs/version-7.1/server/ongoing-tasks/etl/sql.mdx @@ -1,41 +1,42 @@ --- title: "Ongoing Tasks: SQL ETL" sidebar_label: SQL ETL +description: "Replicate RavenDB documents to relational SQL databases using ETL tasks with table mapping and JavaScript transformations." sidebar_position: 2 --- import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; -import LanguageSwitcher from "@site/src/components/LanguageSwitcher"; -import LanguageContent from "@site/src/components/LanguageContent"; +import Panel from "@site/src/components/Panel"; +import ContentFrame from "@site/src/components/ContentFrame"; # Ongoing Tasks: SQL ETL * **SQL ETL** is a task that creates an [ETL process](../../../server/ongoing-tasks/etl/basics.mdx) where data from a RavenDB database is extracted, transformed, and loaded into a relational database as the destination. -* In this page: - * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) - * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) - * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) - * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) - * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#themethod) - * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) - * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) - * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) - * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) - * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) - * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) - * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) - * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) - * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) - * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) - * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) +* In this article: + * [Supported relational databases](../../../server/ongoing-tasks/etl/sql.mdx#supported-relational-databases) + * [Creating the SQL ETL task](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task) + * [Configuring the SQL tables](../../../server/ongoing-tasks/etl/sql.mdx#configuring-the-sql-tables) + * [Document ID column](../../../server/ongoing-tasks/etl/sql.mdx#document-id-column) + * [Insert only](../../../server/ongoing-tasks/etl/sql.mdx#insert-only) + * [Transformation scripts](../../../server/ongoing-tasks/etl/sql.mdx#transformation-scripts) + * [The `loadTo` method](../../../server/ongoing-tasks/etl/sql.mdx#the-loadto-method) + * [Loading to multiple tables](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-multiple-tables) + * [Loading related documents](../../../server/ongoing-tasks/etl/sql.mdx#loading-related-documents) + * [Loading attachments](../../../server/ongoing-tasks/etl/sql.mdx#loading-attachments) + * [Loading to VARCHAR and NVARCHAR columns](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-varchar-and-nvarchar-columns) + * [Loading to specific column types](../../../server/ongoing-tasks/etl/sql.mdx#loading-to-specific-column-types) + * [Filtering](../../../server/ongoing-tasks/etl/sql.mdx#filtering) + * [Accessing the metadata](../../../server/ongoing-tasks/etl/sql.mdx#accessing-the-metadata) + * [Document extensions](../../../server/ongoing-tasks/etl/sql.mdx#document-extensions) + * [Advanced options](../../../server/ongoing-tasks/etl/sql.mdx#advanced-options) + * [Transaction processing](../../../server/ongoing-tasks/etl/sql.mdx#transaction-processing) + * [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api) -## Supported relational databases + + * RavenDB supports ETL processes to the following relational databases: * Microsoft SQL Server @@ -54,24 +55,26 @@ import LanguageContent from "@site/src/components/LanguageContent"; * Before starting with SQL ETL, you need to create tables in the target relational database. These tables will serve as the destinations for records generated by the ETL scripts. + - -## Creating the SQL ETL task + To create an SQL ETL task using the Client API, see [Creating the SQL ETL task from the Client API](../../../server/ongoing-tasks/etl/sql.mdx#creating-the-sql-etl-task-from-the-client-api). -To create an SQL ETL task from the Studio open `Tasks -> Ongoing Tasks`. +To create an SQL ETL task from Studio, open `Tasks` > `Ongoing Tasks`. ![Configure SQL ETL task](./assets/sql-etl-setup.png) + - -## Configuring the SQL tables + Define the target tables where the SQL ETL task will load data. ![Define SQL tables](./assets/sql-etl-tables.png) -#### Document ID Column + + +### Document ID column * For each table, you must specify a column that will store the document ID column. RavenDB will populate this column with the source document ID, enabling the handling of document updates and deletions. @@ -98,16 +101,22 @@ Define the target tables where the SQL ETL task will load data. * For performance reasons, you should define indexes on the SQL tables on the relational database side, at least on the column used to store the document ID. -#### Insert only + + + + +### Insert only * The SQL ETL process updates documents in the relational database using DELETE and INSERT statements. * If your system is _append-only_, you can enable the "Insert Only" toggle to instruct RavenDB to insert data without executing DELETE statements beforehand. This can provide a significant performance boost for systems of this kind. + + -## Transformation scripts + The [basic characteristics](../../../server/ongoing-tasks/etl/basics.mdx) of an SQL ETL script are similar to those of other ETL types. The script defines what data to **extract** from the source document, how to **transform** this data, @@ -116,14 +125,17 @@ and which SQL table to **load** it to. A single SQL ETL task can have multiple transformation scripts. The script is defined per collection, and it cannot be empty. The script is executed per document from the source collection once the document is created or modified. -### The `loadTo` method + + + +### The `loadTo` method To specify which SQL table to load the data into, use either of the following methods in your script. The two methods are equivalent, offering alternative syntax: * **`loadTo(obj)`** * Here the target table is specified as part of the function name. - * The target _<TableName>_ in this syntax is Not a variable and cannot be used as one, + * The target _<TableName>_ in this syntax is not a variable and cannot be used as one, it is simply a string literal of the target's name. * **`loadTo('TableName', obj)`** @@ -131,6 +143,8 @@ The two methods are equivalent, offering alternative syntax: * Separating the table name from the `loadTo` command makes it possible to include symbols like `'-'` and `'.'` in table names. This is not possible when the `loadTo` syntax is used because including special characters in the name of a JavaScript function makes it invalid. +
+ | Parameter | Type | Description | |-----------------|--------|----------------------------------------------------------------------------------------------------------------------------------| | **TableName** | string | The name of the target SQL table | @@ -140,6 +154,11 @@ For example, the following two calls, which load data to "OrdersTable", are equi * `loadToOrdersTable(obj)` * `loadTo('OrdersTable', obj)` + + + + + ### Loading to multiple tables The `loadTo` method can be called multiple times in a single script. @@ -147,91 +166,97 @@ That allows you to split a single `Order` document having `Lines` collection int The following is a sample script that processes documents from the Orders collection: - - -{`// Create an orderData object +```javascript +// Create an orderData object // ========================== -var orderData = \{ +var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 -\}; +}; // Update the orderData's TotalCost field // ====================================== -for (var i = 0; i < this.Lines.length; i++) \{ +for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - // Load the object to SQL table 'OrdersTable' + // Load the object to SQL table 'OrderLines' // ========================================== - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); -\} + }); +} orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; // Load to SQL table 'Orders' // ========================== loadToOrders(orderData); -`} - - +``` + + + + + ### Loading related documents Use the `load` method to load a related document with the specified ID during script execution. - - -{`var company = load(this.Company); -`} - - -### Loading Attachments +```javascript +var company = load(this.Company); +``` + + + + + +### Loading attachments You can store binary data that is kept as attachments in RavenDB using the `loadAttachment()` method. For example, if you have the following _Attachments_ table: - - -{`CREATE TABLE [dbo].[Attachments] +```sql +CREATE TABLE [dbo].[Attachments] ( [Id] int identity primary key, [OrderId] [nvarchar](50) NOT NULL, [AttachmentName] [nvarchar](50) NULL, [Data] [varbinary](max) NULL ) -`} - - +``` + +
then you can define the following script that loads the document's attachments: - - -{`var attachments = this['@metadata']['@attachments']; +```javascript +var attachments = this['@metadata']['@attachments']; -for (var i = 0; i < attachments.length; i++) \{ - var attachment = \{ +for (var i = 0; i < attachments.length; i++) { + var attachment = { OrderId: id(this), AttachmentName: attachments[i].Name, Data: loadAttachment(attachments[i].Name) - \}; + }; loadToAttachments(attachment); -\} -`} - - +} +``` + +
-* Attachments can be also accessed using the `getAttachments()` helper function +* Attachments can be also accessed using the `getAttachments()` helper function (instead of grabbing them from metadata). * The existence of an attachment can be checked by the `hasAttachment(name)` function. + + + + ### Loading to VARCHAR and NVARCHAR columns Two additional functions are designed specifically for working with VARCHAR and NVARCHAR types: @@ -241,96 +266,108 @@ Two additional functions are designed specifically for working with VARCHAR and | `varchar(value, size = 50)` | Defines the parameter type as VARCHAR, with the option to specify its size
(default is 50 if not provided). | | `nvarchar(value, size = 50)` | Defines the parameter type as NVARCHAR, with the option to specify its size
(default is 50 if not specified). | - - -{`var names = this.Name.split(' '); +```javascript +var names = this.Name.split(' '); loadToUsers( -\{ +{ FirstName: varchar(names[0], 30), LastName: nvarchar(names[1]), -\}); -`} - - +}); +``` + + + + + ### Loading to specific column types The SQL type of the target column can be explicitly specified in the SQL ETL script. This is done by defining the `Type` and the `Value` properties for the data being loaded. - * **Type**: - The type specifies the SQL column type the value is loaded to. - The type should correspond to the data types used in the target relational database. +* **Type**: + The type specifies the SQL column type the value is loaded to. + The type should correspond to the data types used in the target relational database. + + Supported enums for `Type` include: + * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) + * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) + * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) + * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - Supported enums for `Type` include: - * _SqlDbType_ - see [Microsoft SQL Server](https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql) - * _NpgsqlDbType_ - see [PostgreSQL](https://www.npgsql.org/doc/api/NpgsqlTypes.NpgsqlDbType.html) - * _MySqlDbType_ - see [MySQL Data Types](https://dev.mysql.com/doc/refman/8.4/en/data-types.html) - * _OracleDbType_ - see [Oracle Data Types](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html) - - Some databases allow combining enum values using `|`. - For example, using `Array | Double` for the Type is valid for PostgreSQL. + Some databases allow combining enum values using `|`. + For example, using `Array | Double` for the Type is valid for PostgreSQL. - If no type is specified, the column type will be detected automatically. + If no type is specified, the column type will be detected automatically. - * **Value**: - The value contains the actual data to be loaded into the column. +* **Value**: + The value contains the actual data to be loaded into the column. - - -{`var orderData = \{ +In the following script, the `Type` and `Value` are specified for the `Quantities` and `Products` columns: + +```javascript +var orderData = { Id: id(this), OrderLinesCount: this.OrderLines.length, - Quantities: \{ + Quantities: { // Specify the Type and Value for 'Quantities': // ============================================ Type: 'Array | Double', - Value: this.OrderLines.map(function(l) \{return l.Quantity;\}) - \}, - Products: \{ + Value: this.OrderLines.map(function(l) {return l.Quantity;}) + }, + Products: { // Specify the Type and Value for 'Products': // ========================================== Type: 'Array | Text', - Value: this.OrderLines.map(function(l) \{return l.Product;\}) - \}, -\}; + Value: this.OrderLines.map(function(l) {return l.Product;}) + }, +}; // Load the data into the 'Orders' table loadToOrders(orderData); -`} - - +``` + + + + + ### Filtering To filter some documents out from the ETL, simply omit the `loadTo` call: - - -{`if (this.ShipTo.Country === 'USA') \{ +```javascript +if (this.ShipTo.Country === 'USA') { // Load only orders shipped to USA - loadToOrders(\{ ... \}); -\} -`} - - + loadToOrders({ ... }); +} +``` + + + + + ### Accessing the metadata You can access the metadata in the following way: - - -{`var value = this['@metadata']['custom-metadata-key']; -`} - - +```javascript +var value = this['@metadata']['custom-metadata-key']; +``` + + + + + ### Document extensions The SQL ETL task does not support sending [Counters](../../../document-extensions/counters/overview.mdx), [Time series](../../../document-extensions/timeseries/overview.mdx), or [Revisions](../../../document-extensions/revisions/overview.mdx). + + -## Advanced options + * **Command timeout** Number of seconds after which SQL command will timeout. @@ -347,22 +384,25 @@ The SQL ETL task does not support sending [Counters](../../../document-extension Control whether to force the SQL Server to recompile the query statement using (`OPTION(RECOMPILE)`). Default: `false`. + - -## Transaction processing + All records created in a single ETL run, one for each `loadTo` call, are sent in a single batch and processed within the same transaction. + + -## Creating the SQL ETL task from the Client API +Creating an SQL ETL task from the Client API takes two steps. - - -{`// Define a connection string to a SQL database destination +First, define and deploy a connection string to the destination database: + +```csharp +// Define a connection string to a SQL database destination // ======================================================== var sqlConStr = new SqlConnectionString -\{ +{ Name = "sql-connection-string-name", // Define destination factory name @@ -372,79 +412,82 @@ var sqlConStr = new SqlConnectionString // May also need to define authentication and encryption parameters // By default, encrypted databases are sent over encrypted channels ConnectionString = "host=127.0.0.1;user=root;database=Northwind" -\}; - +}; + // Deploy (send) the connection string to the server via the PutConnectionStringOperation // ====================================================================================== var PutConnectionStringOp = new PutConnectionStringOperation(sqlConStr); PutConnectionStringResult connectionStringResult = store.Maintenance.Send(PutConnectionStringOp); -`} - - - - -{`// Define the SQL ETL task configuration +``` + +
+ +Then define and deploy the SQL ETL task: + +```csharp +// Define the SQL ETL task configuration // ===================================== var sqlConfiguration = new SqlEtlConfiguration() -\{ +{ Name = "mySqlEtlTaskName", ConnectionStringName = "sql-connection-string-name", SqlTables = - \{ + { new SqlEtlTable - \{ + { TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false - \}, + }, new SqlEtlTable - \{ + { TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false - \}, - \}, + }, + }, Transforms = - \{ + { new Transformation() - \{ + { Name = "scriptName", - Collections = \{ "Orders" \}, + Collections = { "Orders" }, Script = @" - var orderData = \{ + var orderData = { Id: id(this), OrderLinesCount: this.Lines.length, TotalCost: 0 - \}; + }; - for (var i = 0; i < this.Lines.length; i++) \{ + for (var i = 0; i < this.Lines.length; i++) { var line = this.Lines[i]; var cost = (line.Quantity * line.PricePerUnit) * ( 1 - line.Discount); orderData.TotalCost += cost; - loadToOrderLines(\{ + loadToOrderLines({ OrderId: id(this), Qty: line.Quantity, Product: line.Product, Cost: line.PricePerUnit - \}); - \} + }); + } orderData.TotalCost = Math.round(orderData.TotalCost * 100) / 100; loadToOrders(orderData); ", ApplyToAllDocuments = false - \} - \} -\}; + } + } +}; // Deploy the SQL ETL task to the server // ===================================== var addSqlEtlOperation = new AddEtlOperation(sqlConfiguration); store.Maintenance.Send(addSqlEtlOperation); -`} - - +``` + +
+ `SqlEtlConfiguration`: | Property | Type | Description | @@ -466,5 +509,4 @@ store.Maintenance.Send(addSqlEtlOperation); | **DocumentIdColumn** | `string` | The column in the destination table that will store the document IDs. | | **InsertOnlyMode** | `bool` | When set to `true`, RavenDB will insert data directly without executing DELETE statements beforehand.
Default is `false`. | - - +