diff --git a/_data/navigation.yml b/_data/navigation.yml index 1b32ce09..9112551a 100644 --- a/_data/navigation.yml +++ b/_data/navigation.yml @@ -14,70 +14,6 @@ items: - url: /extend/ title: Extending Keboola items: - - url: /extend/component/ - title: Components - items: - - url: /extend/component/tutorial/ - title: Tutorial - items: - - url: /extend/component/tutorial/input-mapping/ - title: Input Mapping - - - url: /extend/component/tutorial/output-mapping/ - title: Output Mapping - - - url: /extend/component/tutorial/configuration/ - title: Configuration - - - url: /extend/component/tutorial/processors/ - title: Processors - - - url: /extend/component/tutorial/debugging/ - title: Debugging - - - url: /extend/component/processors/ - title: Processors - - - url: /extend/component/code-patterns/ - title: Code Patterns - items: - - url: /extend/component/code-patterns/interface/ - title: Interface - - - url: /extend/component/code-patterns/tutorial/ - title: Tutorial - - - url: /extend/component/implementation/ - title: Implementation Notes - items: - - url: /extend/component/implementation/php/ - title: PHP Implementation Notes - - - url: /extend/component/implementation/python/ - title: Python Implementation Notes - - - url: /extend/component/implementation/r/ - title: R Implementation Notes - - - url: /extend/component/running/ - title: Running Components - - - url: /extend/component/ui-options/ - title: UI Options - items: - - url: /extend/component/ui-options/configuration-schema/ - title: Configuration Schema - items: - - url: /extend/component/ui-options/configuration-schema/examples - title: Examples - - url: /extend/component/ui-options/configuration-schema/sync-action-examples - title: Sync Action Examples - - url: /extend/component/ui-options/default-configuration/ - title: Default Configuration - - - url: /extend/component/deployment/ - title: Deployment - - url: /extend/generic-extractor/ title: Generic Extractor items: @@ -201,63 +137,6 @@ items: - url: /extend/generic-writer/configuration-examples/ title: Configuration Examples - - url: /extend/common-interface/ - title: Common Interface - items: - - url: /extend/common-interface/folders/ - title: Data Folders - - - url: /extend/common-interface/config-file/ - title: Configuration File - - - url: /extend/common-interface/environment/ - title: Environment - - - url: /extend/common-interface/manifest-files/ - title: Manifest Files - items: - - url: /extend/common-interface/manifest-files/in-tables-manifests/ - title: IN tables - - - url: /extend/common-interface/manifest-files/in-files-manifests/ - title: IN files - - - url: /extend/common-interface/manifest-files/in-files-s3-staging/ - title: IN files S3 staging - - - url: /extend/common-interface/manifest-files/in-files-abs-staging/ - title: IN files ABS staging - - - url: /extend/common-interface/manifest-files/out-tables-manifests/ - title: OUT tables - - - url: /extend/common-interface/manifest-files/out-tables-manifests-native-types/ - title: OUT tables with Native Types - - - url: /extend/common-interface/manifest-files/out-files-manifests/ - title: OUT files - - - url: /extend/common-interface/oauth/ - title: OAuth2 - - - url: /extend/common-interface/actions/ - title: Actions - - - url: /extend/common-interface/logging/ - title: Logging - - - url: /extend/common-interface/development-branches/ - title: Development branches - - - url: /extend/job-queue/ - title: Job Queue - - - url: /extend/publish/ - title: Publishing Component - items: - - url: /extend/publish/checklist/ - title: Checklist - - url: /integrate/ title: Integration items: diff --git a/_layouts/redirect.html b/_layouts/redirect.html new file mode 100644 index 00000000..04a8e521 --- /dev/null +++ b/_layouts/redirect.html @@ -0,0 +1,65 @@ + +{%- comment -%} +Overrides the template jekyll-redirect-from ships with, for every page carrying +`redirect_to` or `redirect_from`. The plugin only injects its own template when +the site has no `redirect` layout of its own (see Generator#generate). + +The redirect still fires immediately — this markup is what a reader sees when it +does not (JS off, meta refresh blocked, printed copy), so it has to say where the +page went on its own. +{%- endcomment -%} +{%- assign target = page.redirect.to -%} +{%- assign target_label = target | remove: "https://" | remove: "http://" -%} +{%- assign moved_to_help = false -%} +{%- if target contains "help.keboola.com" -%}{%- assign moved_to_help = true -%}{%- endif -%} +{%- comment -%} Same path on the other host, or a genuinely different one? {%- endcomment -%} +{%- assign target_path = target | remove: "https://help.keboola.com" -%} +{%- assign same_path = false -%} +{%- if target_path == page.redirect.from -%}{%- assign same_path = true -%}{%- endif -%} + + + + + {% if moved_to_help %}Moved to the Keboola help docs{% else %}Redirecting…{% endif %} | Keboola Developers Knowledge Base + + + + + + + +
+ + {% if moved_to_help %} +

This page has moved

+

+ {% if page.title %}{{ page.title }} is{% else %}This page is{% endif %} + now part of the Keboola documentation at help.keboola.com, + alongside the rest of the product documentation.{% if same_path %} + The path did not change — only the host.{% endif %} +

+ {{ target_label }} +

+ Taking you there now — use the link above if nothing happens. + Please update any bookmarks, links or component documentation_urls pointing here. +

+ {% else %} +

This page has a new address

+ {{ target_label }} +

Taking you there now — use the link above if nothing happens.

+ {% endif %} +
+ + diff --git a/cli/commands/local/validate/config/index.md b/cli/commands/local/validate/config/index.md index 5a7b0118..88c482a4 100644 --- a/cli/commands/local/validate/config/index.md +++ b/cli/commands/local/validate/config/index.md @@ -7,13 +7,13 @@ permalink: /cli/commands/local/validate/config/ {:toc} -**Validate a [configuration JSON file](/extend/common-interface/config-file/).** +**Validate a [configuration JSON file](https://help.keboola.com/extend/common-interface/config-file/).** ``` kbc local validate config component.id config.json [flags] ``` -Each [component](/extend/component/) definition optionally contains a **schema of the configuration `parameters` key**. +Each [component](https://help.keboola.com/extend/component/) definition optionally contains a **schema of the configuration `parameters` key**. The command validates the content of the specified JSON file against the schema. It can be used both in a project [local directory](/cli/structure/) and also separately. diff --git a/cli/commands/local/validate/row/index.md b/cli/commands/local/validate/row/index.md index 8d2f3f8f..a4b9a6c2 100644 --- a/cli/commands/local/validate/row/index.md +++ b/cli/commands/local/validate/row/index.md @@ -13,7 +13,7 @@ permalink: /cli/commands/local/validate/row/ kbc local validate row component.id row.json [flags] ``` -Each [component](/extend/component/) definition optionally contains a **schema of the configuration row `parameters` key**. +Each [component](https://help.keboola.com/extend/component/) definition optionally contains a **schema of the configuration row `parameters` key**. The command validates the content of the specified JSON file against the schema. It can be used both in a project [local directory](/cli/structure/) and also separately. diff --git a/cli/commands/local/validate/schema/index.md b/cli/commands/local/validate/schema/index.md index 5f60c357..e8e9e41b 100644 --- a/cli/commands/local/validate/schema/index.md +++ b/cli/commands/local/validate/schema/index.md @@ -7,7 +7,7 @@ permalink: /cli/commands/local/validate/schema/ {:toc} -**Validate a [configuration](/extend/common-interface/config-file/)/[row](https://help.keboola.com/components/#configuration-rows) JSON file by a JSON schema file.** +**Validate a [configuration](https://help.keboola.com/extend/common-interface/config-file/)/[row](https://help.keboola.com/components/#configuration-rows) JSON file by a JSON schema file.** ``` kbc local validate schema schema.json config.json [flags] @@ -17,7 +17,7 @@ Validate the content of the specified JSON file against the specified JSON schema file. The JSON schema should contain a schema for the `parameters` key, -just like the configuration/row schema in a [component](/extend/component/) definition. +just like the configuration/row schema in a [component](https://help.keboola.com/extend/component/) definition. The main purpose of this command is to **test a new JSON schema before it is changed in a component definition**. diff --git a/extend/common-interface/actions.md b/extend/common-interface/actions.md index cf36b04a..1a3a30eb 100644 --- a/extend/common-interface/actions.md +++ b/extend/common-interface/actions.md @@ -1,106 +1,5 @@ --- title: Actions permalink: /extend/common-interface/actions/ +redirect_to: https://help.keboola.com/extend/common-interface/actions/ --- - -* TOC -{:toc} - -Actions provide a way to execute very quick tasks in a single Component, using a single code base. -The default component's action (`run`) executes as a background, asynchronous job. It is queued, has plenty of -execution time, and there are cases when you might not want to wait for it. Apart from the default `run`, there -can be synchronous actions with limited execution time and you must wait for them. When we refer to -**actions**, we mean *synchronous actions*. Using actions is fully optional. - -## Use Case -For example, in our database extractor, the main task (`run` action) is the data extraction itself. But we also want to be -able to test the database credentials and list tables available in the database. -These tasks would be very helpful in the UI. It is not possible to do these things directly in the browser. Setting up a -separate component would bring an overhead of maintaining both the extractor's Docker image and the new component. - -## Solution -For each Component, you can specify other actions (apart from the default `run`). These -actions will be executed using the same Docker image, but [Job Queue](/extend/job-queue/) will wait for its execution and use -the returned value as the API response. So, these additional actions are executed *synchronously* and have a very -limited execution time (maximum 30 seconds). These actions also cannot access Storage. - -The [configuration file](/extend/common-interface/config-file/#configuration-file-structure) -contains the `action` property with the name of the currently executed action. Just grab the value and act accordingly. -All actions must be explicitly specified in the component configuration in [Developer Portal](https://components.keboola.com/). - -## Running Actions -Actions are available through the [API](https://api.keboola.com/?service=sync-actions#post-/actions). -They do not load the configuration from Storage, so you need to fully specify the whole configuration in the request body. -If any of your parameters are encrypted, they will be decrypted before they are passed to your component. - -Do not specify the `action` attribute in the request body, it is already in the URI. Use any of `parameters` and `runtime` inside the `configData` root element as you would when creating an asynchronous job. Using `storage` configuration in actions makes no sense, because actions cannot read or write to Storage. For instance: - -{% highlight json %} - -{ - "configData": { - "parameters": { - "key": "val" - } - } -} - -{% endhighlight %} - -### Return Values - -As the component output is passed back through the API, all output from an action **MUST** be JSON (except for errors). - -If your component outputs an invalid JSON on its STDOUT, an application error will be raised. - -## Handling User and Application errors - -Actions use the same [exit codes](/extend/common-interface/environment/#return-values) as the default `run` action. - -If an user or application error is detected, STDERR/STDOUT is handled as the message string and is returned to the user. The message is wrapped into a standardized structure. - -For example - -{% highlight python %} -print('user error message') -sys.exit(1) -{% endhighlight %} - -yields this message on the API (HTTP status code 400) - -{% highlight json %} -{ - "status": "error", - "error": "User error", - "code": 400, - "message": "user error message", - "exceptionId": "docker-7ed4c3b599776e8a2a84a7f185f5f7f2", - "runId": 0 -} -{% endhighlight %} - -and - -{% highlight python %} -print('application error message') -sys.exit(2) -{% endhighlight %} - -yields this message on the API (HTTP status code 500) - -{% highlight json %} -{ - "status": "error", - "error": "Application error", - "code": 500, - "message": "Contact support@keboola.com and attach this exception id.", - "exceptionId": "docker-2a51922e0753cf78297ad6d384200206", - "runId": 0 -} -{% endhighlight %} - -## Limits - -**Sync actions may not read from or write data to the Storage.** -Otherwise actions share the same limits as the default `run` action, only the execution time is limited to 30 seconds. -This time does not include pulling the Docker image. diff --git a/extend/common-interface/config-file.md b/extend/common-interface/config-file.md index 384f6320..25b5614a 100644 --- a/extend/common-interface/config-file.md +++ b/extend/common-interface/config-file.md @@ -1,833 +1,5 @@ --- title: Configuration File Specification permalink: /extend/common-interface/config-file/ +redirect_to: https://help.keboola.com/extend/common-interface/config-file/ --- - -* TOC -{:toc} - -Configuration files are one of the [possible channels](/extend/common-interface/) for exchanging data -between components and Keboola. - -To create a sample configuration file (together with the data directory), -use the [Run Job API call in debug mode](https://api.keboola.com/?service=job-queue#post-/jobs) via the -[Job Queue API](https://api.keboola.com/?service=job-queue). -You will get a zip archive containing all the resources you need in your component. - -All configuration files are always stored in `JSON` format. - -## Configuration File Structure -Each configuration file has the following root nodes: - -- `storage`: Contains both the input and output [mapping](https://help.keboola.com/transformations/mappings/) for both files and tables. -This section is important if your component uses a dynamic input/output mapping. -Simple components can be created with a static input/output mapping. -They do not use this configuration section at all (see [Tutorial](/extend/component/tutorial/)). -- `parameters`: Contains arbitrary parameters passed from the UI to the component. This section can be used in any -way you wish. Your component should validate the contents of this section. For passing sensitive -data, use [encryption](/overview/encryption/). This section is not available in Transformations. -- `image_parameters`: See [below](#image-parameters). -- `authorization`: Contains Oauth2 [authorization contents](/extend/common-interface/oauth/) or -[Workspace credentials](/extend/common-interface/folders/#exchanging-data-via-workspace) . -- `action`: Name of the [action](/extend/common-interface/actions/) to execute; defaults to `run`. All -actions except `run` have a strict execution time limit of 30 seconds. -See [actions](/extend/common-interface/actions/) for more details. - -### Validation -Your application should implement validation of the `parameters` section, which is passed without modification from the UI. -Your application might also implement validation of the `storage` section if you have some specific requirements on the -input mapping or output mapping setting (e.g., certain number of tables, certain names). If you chose to do any validation -outside the `parameters` section, it must always be forward compatible -- i.e. benevolent. While we maintain backward compatibility -very carefully, it is possible for new keys to appear in the configuration structure as we introduce new features. - -### Image Parameters -The `image_parameters` section contains configuration options, which are the same for every configuration of a component. -They cannot be modified by the end-user. This section is typically used for global component -parameters (such as a token, URL, version of your API) which, for any reason, are not practical to be part of the component image itself. -The `image_parameters` contents are configured in the [component settings](https://components.keboola.com/) in JSON format in two -text fields: **Image Parameters** and **Stack Parameters**. - -Both JSONs are merged into the `image_parameters` of the configuration file. The *Stack Parameters* -provide different values for different [Keboola Stacks](/overview/api/#regions-and-endpoints). Values in -*Stack Parameters* are merged with those in *Image Parameters* with *Stack Parameters* having a higher priority. -*Stack Parameters* are indexed with [Storage URL](/overview/api/#regions-and-endpoints) or the given region. - -Given the following *Image Parameters*: - -{% highlight json %} -{ - "name": "my-app-name", - "token": "default" -} -{% endhighlight %} - -And the following *Stack Parameters*: - -{% highlight json %} -{ - "connection.keboola.com": { - "url": "https://my-us-api/", - "token": "abc" - }, - "connection.eu-central-1.keboola.com": { - "url": "https://my-eu-api/", - "token": "def" - } -} -{% endhighlight %} - -The component will receive the following `image_parameters` in the configuration file when run in the **EU region**: -{% highlight json %} -{ - "image_parameters": { - "name": "my-app-name", - "url": "https://my-eu-api/", - "token": "def" - } -} -{% endhighlight %} - -The component will receive the following `image_parameters` in the configuration file when run in the **US region**: -{% highlight json %} -{ - "image_parameters": { - "name": "my-app-name", - "url": "https://my-us-api/", - "token": "abc" - } -} -{% endhighlight %} - -When working with the API, note that the [Developer Portal API](https://api.keboola.com/?service=developer-portal) -(specifically the [Component Detail API call](https://api.keboola.com/?service=developer-portal#get-/vendors/-vendor-/apps/-app-)) -shows separate `stack_parameters` and `image_parameters`, because the API is region agnostic. - -However, when working with the [Storage API](https://api.keboola.com/?service=storage) -(specifically the [Component list API call](https://api.keboola.com/?service=storage#get-/v2/storage)), -the `stack_parameters` and `image_parameters` values are already merged and only those designated for the -current region are visible. - -#### Encryption -Both *Image Parameters* and *Stack Parameters* support [encrypted values](/overview/encryption/). In practice, however, -the encrypted values must always be stored in *Stack Parameters*, because ciphers are not transferable between regions -(i.e. an encrypted value is only usable in the region in which it was encrypted). - -As with configurations, the encrypted values must be prefixed with the hash sign `#`. However, unlike in Keboola configurations, -you **have to encrypt values manually via the API** -- they will not be encrypted automatically when you store *Stack Parameters*! -When using the [encryption API](https://api.keboola.com/?service=encryption#post-/encrypt), provide only the `componentId` -parameter (using `projectId` or `configId` will make the cipher unusable). -Also take care to use the correct [API URL](https://developers.keboola.com/overview/api/#regions-and-endpoints) to obtain -ciphers for each region you need. - -## State File -The state file is used to store the component state for the next run. It provides a two-way communication between -Keboola configuration state storage and the component. The state file only works if the API call -references a stored configuration (`config` is used, not `configData`). - -The location of the state file is: - -- `/data/in/state.json` loaded from a configuration state storage -- `/data/out/state.json` saved to a configuration state storage - -The component reads the input state file and writes any content to the output state -file (valid JSON) that -will be available to the next API call. A missing or an empty file will remove the state value. -A state object is saved to configuration storage only when actually running the app -(not when using the [Run Job API call in debug mode](https://api.keboola.com/?service=job-queue#post-/jobs)). The state must be a valid JSON file. -[Encryption](/overview/encryption/#encrypting-data-with-api) is applied to the state the same way it is applied to -configurations, `KBC::ProjectSecure::` ciphers are used. - -### State File Properties -Because the state is stored as part of a -[component configuration](https://api.keboola.com/?service=storage#tag--Component-Configurations), -the value of the state object is somewhat limited (should not generally exceed 1MB). It should not -be used to store large amounts of data. - -Also, the end-user cannot easily access the data through the UI. -The data can be, however, modified outside of the component itself using the -[component configuration](https://api.keboola.com/?service=storage#tag--Component-Configurations) API calls. -Note however that the content in the contents of the state file is nested: - -I.e., assume that the component generates a state file with the following contents: - - -{% highlight json %} -{ - "time": { - "previousStart": 1587980435 - } -} -{% endhighlight %} - -If you read the configuration through the Component configuration API call, you'll see: - -{% highlight json %} -"state": { - "component": { - "time": { - "previousStart": 1587980435 - } - }, - "storage": { - "input": { - "tables": [] - } - } - } -{% endhighlight %} - -That means the contents of the state file are nested inside the `component` node. There -is also a `storage` node, which is related to the -[Automatic incremental processing](https://help.keboola.com/storage/tables/#automatic-incremental-processing). - -You need to maintain the above structure when manually changing the configuration via the API. - -**Important:** The state file is not thread-safe. If multiple instances of the **same configuration** -are run simultaneously in the **same project**, the one writing data later wins. Use the state file more -as an HTTP cookie than as a database. A typical use for the state file would be saving the last record -loaded from some API to enable incremental loads. - -## Usage File - -Unlike the state file, the **usage file is one way only** and has a pre-defined structure. -The usage file is used to pass information from the component to Keboola. -Metrics stored are used to determine how much resources the job consumed and translate the usage to Keboola -credits; this is very useful when you need your customers to pay using your component or service. - -The usage file is located at `/data/out/usage.json`. It should contain an array of objects -keeping information about the consumed resources. The objects have to contain only two keys, `metric` -and `value`, as in the example bellow: - -{% highlight json %} -[ - { - "metric": "API calls", - "value": 150 - } -] -{% endhighlight %} - -This structure is processed and stored within a job, so it can be analyzed, processed and aggregated later. - -To keep track of the consumed resources in the case of a component failure, **it is recommended to -write the usage file regularly** during the component run, not only at the end. - -*Note: As the structure of the state file is pre-defined, the content of the usage file is strictly -validated and a wrong format will cause a component failure.* - -## Examples -To create an example configuration, use the [Run Job API call in debug mode](/extend/component/running/#preparing-the-data-folder). You will get a -`stage_0.zip` archive in your **Storage** > **File Uploads**, which will contain the `config.json` file. -You can also use these configuration structure to create an API request for -actually [running a component](https://api.keboola.com/?service=job-queue#post-/jobs). -If you want to manually pass configuration options in the API request, be sure to wrap it around in the `configData` node. - -A sample configuration file might look like this: - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.test", - "destination": "source.csv", - "limit": 50, - "columns": [], - "where_values": [], - "where_operator": "eq" - }, - { - "source": "pokus.snaz.test", - "destination": "source1.csv" - } - ], - "files": [] - }, - "output": { - "tables": [ - { - "source": "destination.csv", - "destination": "out.c-main.test", - "incremental": false, - "colummns": [], - "primary_key": [], - "delete_where": [], - "delimiter": ",", - "enclosure": "\"" - }, - { - "source": "write-alwayss.csv", - "destination": "out.c-main.output-even-on-error" - "write_always": true - } - ], - "files": [] - } - }, - "parameters": { - "multiplier": 2 - }, - "image_parameters": [], - "action": "run" -} -{% endhighlight %} - -### Tables -Tables from the input mapping are mounted to `/data/in/tables`. -Input mapping parameters are similar to the [Storage API export table options](https://keboola.docs.apiary.io/#reference/tables/unload-data-asynchronously). -If `destination` is not set, the CSV file will have the same name as the table (without adding `.csv` suffix). -The tables element in a configuration of the **input mapping** is an array and supports the following attributes: - -- `source` -- `destination` -- `days` (internally converted to `changed_since`) -- `columns` -- `column_types` -- `where_column` -- `where_operator` -- `where_values` -- `limit` - -The output mapping parameters are similar -to the [Transformation API output mapping ](https://help.keboola.com/transformations/). -`destination` is the only required parameter. If `source` is not set, the CSV file is expected to have the same name -as the `destination` table. -The tables element in a configuration of the **output mapping** is an array and supports the following attributes: - - - `source` - - `destination` - - `incremental` - - `columns` - - `primary_key` - - `delete_where` - Defines rules for deleting records before loading new data - - `delete_where_column` - **[DEPRECATED]** Use `delete_where` instead - - `delete_where_operator` - **[DEPRECATED]** Use `delete_where` instead - - `delete_where_values` - **[DEPRECATED]** Use `delete_where` instead - - `delimiter` - - `enclosure` - - `write_always` - -#### Input mapping --- basic -Download tables `in.c-ex-salesforce.Leads` and `in.c-ex-salesforce.Accounts` to `/data/tables/in/leads.csv` -and `/data/tables/in/accounts.csv`. - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-ex-salesforce.Leads", - "destination": "leads.csv" - }, - { - "source": "in.c-ex-salesforce.Accounts", - "destination": "accounts.csv" - } - ] - } - } -} -{% endhighlight %} - -In an API request, this would be passed as: - -{% highlight json %} -{ - "configData": { - "storage": { - "input": { - "tables": [ - { - "source": "in.c-ex-salesforce.Leads", - "destination": "leads.csv" - }, - { - "source": "in.c-ex-salesforce.Accounts", - "destination": "accounts.csv" - } - ] - } - } - } -} -{% endhighlight %} - - -#### Input mapping --- incremental load -Download 2 days of data from the `in.c-storage.StoredData` table to `/data/tables/in/in.c-storage.StoredData`. - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-storage.StoredData", - "days": "2" - } - ] - } - } -} -{% endhighlight %} - -#### Input mapping --- select columns - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-ex-salesforce.Leads", - "columns": ["Id", "Revenue", "Date", "Status"] - } - ] - } - } -} -{% endhighlight %} - -#### Input mapping --- column types -This is applicable only to [workspace mapping](/extend/common-interface/folders/#exchanging-data-via-workspace), for CSV files this setting has no effect. The `column_types` setting maps to [Storage API load options](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/workspaces/-workspaceId-/load). It also acts the same way as `columns` setting allowing you to limit the table columns. -If both `column_types` and `columns` setting are used, then the listed columns must match. If you omit `columns` and use only `column_types` (recommended) then `columns` will be propagated automatically from `column_types`. - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-ex-salesforce.Leads", - "column_types": [ - { - "source": "Id", - "type": "VARCHAR", - "destination": "id", - "length": "255", - "nullable": false, - "convert_empty_values_to_null": false - } - ] - } - ] - } - } -} -{% endhighlight %} - -#### Input mapping --- filtered table - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-ex-salesforce.Leads", - "destination": "closed_leads.csv", - "where_column": "Status", - "where_values": ["Closed Won", "Closed Lost"], - "where_operator": "eq" - } - ] - } - } -} -{% endhighlight %} - -#### Output mapping --- basic -Upload `/data/out/tables/out.c-main.data.csv` to `out.c-main.data`. - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "out.c-main.data.csv", - "destination": "out.c-main.data" - } - ] - } - } -} -{% endhighlight %} - -#### Output mapping --- headless CSV -Upload `/data/out/tables/data.csv`, a CSV file without headers on its first line, to the table `out.c-main.data`. - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "data.csv", - "destination": "out.c-main.data", - "columns": ["column1", "column2"] - } - ] - } - } -} -{% endhighlight %} - -#### Output mapping --- set additional properties -Incrementally upload `/data/out/tables/data.csv` to `out.c-main.data` -with a compound primary key set on the columns `column1` and `column2`. - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "data.csv", - "destination": "out.c-main.data", - "incremental": true, - "primary_key": ["column1", "column2"] - } - ] - } - } -} -{% endhighlight %} - -#### Output mapping --- write even if the job fails -If you have a table that you are updating during the execution of the job -and you want to output that table even if the job fails then you can use the `write_always` flag - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "always-output.csv", - "destination": "out.c-main.always-output", - "write_always": true - } - ] - } - } -} -{% endhighlight %} - -#### Output mapping --- delete rows -Delete data from the `destination` table before uploading the CSV file (only makes sense with `incremental: true`). - -The `delete_where` parameter provides a flexible way to specify which records should be deleted from the target table before loading new data into it. It supports time-based filters and multiple filter conditions: - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "data.csv", - "destination": "out.c-main.Leads", - "incremental": true, - "delete_where": [ - { - "changed_since": "-7 days", - "changed_until": "-2 days", - "where_filters": [ - { - "column": "Status", - "operator": "eq", - "values_from_set": ["Closed"] - }, - { - "column": "Status", - "operator": "eq", - "values_from_workspace": { - "workspace_id": "123", - "table": "statuses", - "column": "status_name" - } - } - ] - } - ] - } - ] - } - } -} -{% endhighlight %} - -**Parameters:** - -- `changed_since` (optional) - Starting point for time-based deletion. Can be specified as: - - Relative time (e.g., "-2 days", "-1 month") - - Unix timestamp (e.g., "1360138863") - - ISO 8601 date (e.g., "2013-02-12T15:19:21+00:00") -- `changed_until` (optional) - End point for time-based deletion. Accepts the same formats as `changed_since` -- `where_filters` (optional) - Array of filter conditions: - - `column` - Name of the column to filter on - - `operator` - One of: `eq` (equals), `ne` (not equals) - - `values_from_set` - Array of specific values to match against - - `values_from_workspace` - Reference values from a workspace table: - - `workspace_id` - ID of the workspace. Optional when exchanging data through [database workspace](/extend/common-interface/folders/#exchanging-data-via-database-workspace) - - `table` - Name of the table in the workspace. - - `column` - Name of the column containing values. If not specified, the column name from `where_filters.column` will be used - -**Note:** For each `where_filters` item, you must use only one method to specify values - either `values_from_set` or `values_from_workspace`. Using multiple value sources in a single filter is not allowed. - -You can combine multiple rules and filters to create complex deletion conditions. Each rule in the `delete_where` array is processed independently. - -##### Simple Example -Here's a basic example of deleting records with a specific status: - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "data.csv", - "destination": "out.c-main.Leads", - "incremental": true, - "delete_where": [ - { - "where_filters": [ - { - "column": "Status", - "operator": "eq", - "values_from_set": ["Closed", "Cancelled"] - } - ] - } - ] - } - ] - } - } -} -{% endhighlight %} - -This configuration performs a DELETE operation equivalent to the following SQL: - -```sql -DELETE FROM "out.c-main.Leads" -WHERE "Status" IN ('Closed', 'Cancelled') -``` - -When using `operator: "ne"` (not equals), the operation will use SQL's NOT IN clause instead of IN. For example, if you specify `values_from_set: ["Active", "Pending"]` with `operator: "ne"`, it will delete all records where the column value is NOT one of the specified values. - -##### Multiple Filters -Multiple filters in a single `where_filters` array are combined using AND operator. For example: - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "data.csv", - "destination": "out.c-main.Leads", - "incremental": true, - "delete_where": [ - { - "where_filters": [ - { - "column": "Status", - "operator": "eq", - "values_from_set": ["Closed", "Cancelled"] - }, - { - "column": "Region", - "operator": "ne", - "values_from_set": ["EU"] - } - ] - } - ] - } - ] - } - } -} -{% endhighlight %} - -This configuration performs a DELETE operation equivalent to the following SQL: - -```sql -DELETE FROM "out.c-main.Leads" -WHERE "Status" IN ('Closed', 'Cancelled') - AND "Region" NOT IN ('EU') -``` - -**Important Note:** Multiple rules in the `delete_where` array are processed independently (as separate DELETE statements) - -##### Independent Rules Processing -When multiple rules are specified in the `delete_where` array, each rule is processed as a separate DELETE statement. For example: - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "data.csv", - "destination": "out.c-main.Leads", - "incremental": true, - "delete_where": [ - { - "where_filters": [ - { - "column": "Status", - "operator": "eq", - "values_from_set": ["Closed"] - } - ] - }, - { - "where_filters": [ - { - "column": "Region", - "operator": "eq", - "values_from_set": ["EU"] - } - ] - } - ] - } - ] - } - } -} -{% endhighlight %} - -This configuration performs two separate DELETE operations equivalent to: - -```sql -DELETE FROM "out.c-main.Leads" -WHERE "Status" IN ('Closed'); - -DELETE FROM "out.c-main.Leads" -WHERE "Region" IN ('EU'); -``` - -##### Legacy Delete Configuration (Deprecated) -For backward compatibility, the following parameters are still supported but not recommended for new implementations: - -{% highlight json %} -{ - "storage": { - "output": { - "tables": [ - { - "source": "data.csv", - "destination": "out.c-main.Leads", - "incremental": true, - "delete_where_column": "Status", - "delete_where_values": ["Closed"], - "delete_where_operator": "eq" - } - ] - } - } -} -{% endhighlight %} - -### Files -Another way of downloading files from file uploads is to use an -[Elasticsearch query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) -or filtering with tags. Note that the results of a file mapping are limited to 10 files (to prevent accidental downloads). -If you need more files, use multiple file mappings. - -All files matching the search will be downloaded to the `/data/in/files` folder. -The name of each file has the `fileId_fileName` format. Each file will also contain a -[manifest](/extend/common-interface/manifest-files/) with all information about the file. - -#### Input mapping --- query - -{% highlight json %} -{ - "storage": { - "input": { - "files": [ - { - "tags": ["docker-demo"], - "query": "name:.zip" - } - ] - } - } -} -{% endhighlight %} - -This will download with files with matching `.zip` **and** having the `docker-demo` tag. Depending on the contents of your -**File uploads** in **Storage**, this may produce something like: - - /data/in/files/75807542_fooBar.zip - /data/in/files/75807542_fooBar.zip.manifest - /data/in/files/75807657_fooBarBaz.zip - /data/in/files/75807657_fooBarBaz.zip.manifest - -#### Output mapping --- basic -Define additional properties for uploaded files in the output mapping configuration. -If that file is not present in the `/data/out/files` folder, an error will be thrown. - -{% highlight json %} -{ - "storage": { - "output": { - "files": [ - { - "source": "file.csv", - "tags": ["processed-file", "csv"] - }, - { - "source": "image.jpg", - "is_public": true, - "is_permanent": true, - "tags": ["image", "pie-chart"] - } - ] - } - } -} -{% endhighlight %} - -#### Incremental processing - -**Deprecated:** The `processed_tags` setting is deprecated and is not compatible with -[development branches](/extend/common-interface/development-branches/), because a job running in a -development branch cannot write tags back to files in production storage. New configurations should not -use it, and the UI no longer offers it. Existing configurations continue to work. To process files -incrementally, use [incremental file processing](https://help.keboola.com/transformations/mappings/#incremental-file-processing) -instead. - -The following describes the legacy `processed_tags` behavior, retained for reference only. - -Docker containers may be used to process unknown files incrementally. This means that when a container is run, -it will download any files not yet downloaded and process them. To achieve this behavior, it is necessary -to select only the files which have not been processed yet and tag the processed files. -To achieve the former, use a proper -[Elasticsearch query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax). -The latter is achieved using the `processed_tags` setting. The `processed_tags` setting is an array of tags -which will be added to the **input** files once they are downloaded. A sample contents of `configData`: - -{% highlight json %} -{ - "storage": { - "input": { - "files": [ - { - "query": "tags: toprocess AND NOT tags: downloaded", - "processed_tags": ["downloaded"] - } - ] - } - } -} -{% endhighlight %} - -The above request will download every file with the `toprocess` tag **except** for the files having the `downloaded` tag. -It will mark each such file with the `downloaded` tag; therefore the query will exclude them on the next run. -This allows you to set up an incremental file processing pipeline. diff --git a/extend/common-interface/development-branches.md b/extend/common-interface/development-branches.md index 61d0d4c5..ec0bfca0 100644 --- a/extend/common-interface/development-branches.md +++ b/extend/common-interface/development-branches.md @@ -1,55 +1,5 @@ --- title: Development branches permalink: /extend/common-interface/development-branches/ +redirect_to: https://help.keboola.com/extend/common-interface/development-branches/ --- - -* TOC -{:toc} - -Development branches are a feature for managing change in Keboola projects. Refer to our [user documentation](https://help.keboola.com/components/branches/) -to learn more about how development branches function. - -{% include branches-beta-warning.html %} - -## Running a Component in a Branch - -A component that uses the [Common Interface](/extend/common-interface/) can be run in a branch without any changes to the code. Notable exceptions include -components that modify external resources (e.g., database writers) and those that use [forwarded Storage tokens](/extend/common-interface/environment/#environment-variables) to interact with the Storage API. - -### Is a Component Executed in a Branch Context? - -When the [Job Queue](/extend/job-queue/) executes a job in a branch, it sets the [`KBC_BRANCHID` environment variable](/extend/common-interface/environment/#environment-variables) to the current branch ID, which is unique accross the stack. - -Typically, the fact that a component is executed in a branch is not very important to the component itself. It behaves the same way, and the Keboola job runner does all the heavy lifting. - -The exception is when a component directly interacts with the Storage API using a forwarded Storage token; it must then consider the branch ID. These components are subject to a separate review by Keboola to verify correct implementation. - -### Input and Output Mapping in a Development Branch - -#### Write - -When writing data to Storage, the bucket name in the [input mapping](/extend/component/tutorial/input-mapping/) is automatically prefixed with the branch's internal ID to ensure that data in the production bucket is not overwritten. - -#### Read - -A component first checks for a development version of the production bucket when reading data from Storage. If such a bucket exists, it is used; otherwise, the data is read from the production bucket. This prevents the need to duplicate data extraction jobs in development branches. - -### Configuration State in a Development Branch - -Configuration [states](/integrate/storage/api/configurations/#state) are stored separately for each development branch and are not merged back to the main branch upon merging of the branch itself. - -### Components Interacting with External Resources - -Extra precautions are necessary for components that interact with external resources to prevent unintended impacts on production data. - -For example, in a production environment, a Snowflake writer might write to the PROD_SCHEMA schema in a Snowflake database. Running this writer in a development -branch without appropriate safeguards could result in writing data intended for development to the production schema. Therefore, operational restrictions apply to -certain jobs in development branches based on the component’s features: - -* **dev-branch-configuration-unsafe**: These components can be run in a development branch if `{configuration:{runtime: {safe: true}}}` is set in their configuration. This can be adjusted via the API or through the *Safe for run in branch* toggle in the configuration detail in the UI. The job runner will verify the safety status before execution. This feature is automatically set for applications and writers. It's not set for extractors. -* **dev-branch-job-blocked**: These components are not permitted to run in development branches under any circumstances. -* **dev-mapping-allowed**: These components are allowed to use development buckets in their default branch input mappings, which is typically restricted. - -For details on a component’s features, you can consult the [Developer Portal API](https://api.keboola.com/?service=developer-portal#get-/apps/-app-) or the [Component List in Storage API](https://api.keboola.com/?service=storage#get-/v2/storage). - -To request changes to your component's features, please use the support button in your project to contact our support team. diff --git a/extend/common-interface/environment.md b/extend/common-interface/environment.md index 4adbcb4f..d1f1517d 100644 --- a/extend/common-interface/environment.md +++ b/extend/common-interface/environment.md @@ -1,64 +1,5 @@ --- title: Environment Specification permalink: /extend/common-interface/environment/ +redirect_to: https://help.keboola.com/extend/common-interface/environment/ --- - -* TOC -{:toc} - -Components use several [channels](/extend/common-interface/) to exchange information with Keboola, -primarily through [structured folders](/extend/common-interface/) and [configuration files](/extend/common-interface/config-file/). -Each component has full access to the external network. -Below are the specific aspects of the environment in which your component operates. - -## Environment Variables -The following environment variables are injected into the container: - - - `KBC_DATADIR`: Always `/data/` in Keboola; use this variable during component development to create development and testing environments. - - `KBC_RUNID`: The RunId from Storage; links all events within an API call (useful for logging). - - `KBC_PROJECTID`: The ID of the project in Keboola within a [Keboola stack](/overview/api/#regions-and-endpoints). - - `KBC_STACKID`: The ID of the [Keboola stack](/overview/api/#regions-and-endpoints). - - `KBC_CONFIGID`: The ID of the configuration, or a hash of configuration data if the configuration is not named (e.g., when `configData` is used in an [API call](https://api.keboola.com/?service=job-queue#post-/jobs)). - - `KBC_CONFIGVERSION`: The version of the configuration, or empty if unnamed (when `configData` is used in the [API call](https://api.keboola.com/?service=job-queue#post-/jobs)). - - `KBC_COMPONENTID`: The ID of the component. - - `KBC_CONFIGROWID`: The ID of the configuration row, if available. - - `KBC_BRANCHID`: The ID of the [development branch](https://api.keboola.com/?service=storage#get-/v2/storage/dev-branches/-id-). - - `KBC_STAGING_FILE_PROVIDER`: Either `aws` or `azure`, depending on the type of [stack](/overview/api/#regions-and-endpoints) the container is running. This value refers to the file storage used during [file import/export operations](https://developers.keboola.com/integrate/storage/api/import-export/). - - `KBC_PROJECT_FEATURE_GATES`: A comma-separated list of feature gates activated for the current project. Feature gates are considered internal and may change or disappear without notice. We recommend checking with our support team before relying on any feature gates. - - `KBC_COMPONENT_RUN_MODE`: Either `run` or `debug`. The value `debug` is used when the job is run in debug mode ([learn more](https://developers.keboola.com/extend/component/running/#debugging)). This variable can be helpful, for example, to enable more verbose logging. - - `KBC_DATA_TYPE_SUPPORT`: Either `authoritative`, `hints`, or `none`: - - `authoritative`: The component generates columns with data types in the schema node. - - `hints`: The component generates columns without data types in the schema node. - - `none`: The component generates only column names in the columns node. - -### Additional Variables for Forwarded Token and Token Details - - The following variables are available only if "Forwards token" and "Forwards token details" are - enabled in the [component configuration](https://components.keboola.com/) (and approved by Keboola): - - - `KBC_PROJECTNAME`: The name of the project in Keboola. - - `KBC_TOKENID`: The ID of the token running the container. - - `KBC_TOKENDESC`: A description of the token (e.g., user name or token name). - - `KBC_TOKEN`: The token itself. - - `KBC_URL`: The Storage API URL. - - `KBC_REALUSER`: The user ID provided by [SAML](https://en.wikipedia.org/wiki/Security_Assertion_Markup_Language) authentication. - -### Additional Variables for GELF Logger -The following variables are available when the [GELF Logger](/extend/common-interface/logging/#gelf-logger) is enabled in the -[component configuration](https://components.keboola.com/): - -- `KBC_LOGGER_ADDR`: The IP address of the GELF server. -- `KBC_LOGGER_PORT`: The port of the GELF server. - -## Return Values -The script defined in the Dockerfile's `ENTRYPOINT` or `CMD` should provide an exit status. The -following rules apply: - -- `exit code = 0`: The execution is considered successful. -- `exit code = 1`: The execution fails with a *User Error*. -Both STDOUT and [STDERR](https://en.wikipedia.org/wiki/Standard_streams#Standard_error_.28stderr.29) are sent to Storage API Events. -- `exit code > 1`: The execution fails with an *Application Error*. Both STDOUT and STDERR are logged internally. - -### Modifying Error Behavior -To report all errors as User Errors regardless of the exit code, set `no_application_errors` in the [component configuration](https://components.keboola.com/). -See the [implementation notes](/extend/component/implementation/) for tips on distinguishing between User Errors and Application Errors. diff --git a/extend/common-interface/folders.md b/extend/common-interface/folders.md index 56d2dda2..8702aa45 100644 --- a/extend/common-interface/folders.md +++ b/extend/common-interface/folders.md @@ -1,305 +1,5 @@ --- title: Data Folders Specification permalink: /extend/common-interface/folders/ +redirect_to: https://help.keboola.com/extend/common-interface/folders/ --- - -* TOC -{:toc} - -Data folders are one of the [possible channels](/extend/common-interface/) to exchange data between your component and Keboola. - -## Root Folder /data/ -The `/data/` folder is the root folder for exchanging data. -Your component reads its input from the `/data/in` folder and writes its results to the `/data/out` folder. -Keboola takes care of injecting required tables and files into the input folder and -picking up tables and files generated by your code. -The data folders contain actual data files (tables and files) and metadata. -For each datafile, a [manifest file](/extend/common-interface/manifest-files/) is created. -It contains metadata information (creation time, keys for tables, etc.). - -The data folder is always available in the component under the **absolute `/data/` path**. The relative path to the data folder -depends fully on your component code (or Dockerfile). If you want to use a different path (for component development), -**use the [`KBC_DATADIR` environment variable](/extend/common-interface/environment/#environment-variables)**. In production, -this variable will always be set to `/data/`. During development, you can set it to your liking. - -To create a data folder sample, use the [Debug mode](/extend/component/running/#preparing-the-data-folder) on the -[Create Job API](https://api.keboola.com/?service=job-queue#post-/jobs). -All the resources you need in your component will be provided in a ZIP archive. - -The predefined data exchange folder structure is as follows: - - /data/in/tables - /data/in/files - /data/out/tables - /data/out/files - -This folder structure is always available to your component. -Do not put arbitrary files in the `/data/` folder as they will be uploaded into the user project -(or cause errors in the output [mapping](https://help.keboola.com/transformations/mappings/)). -For working or temporary files, use the `/tmp/` folder. Other directories have 10GB of free space in total. - -### Folder /data/in/tables/ - -The folder contains tables defined in the input [mapping](https://help.keboola.com/transformations/mappings/); -they are serialized in the CSV format: - - - string enclosure `"` - - delimiter `,` - - no escape character - -File names are specified in the input mapping, defaulting to `{tableId}` (a file name can be changed in the UI). -The table metadata is stored in a [manifest file](/extend/common-interface/manifest-files/). - -### Folder /data/out/tables/ - -All output tables from your component must be placed in this folder. The destination table in -[Storage](https://help.keboola.com/storage/) is defined by the following rules (listed in order): - -- If `defaultBucket` (see [below](#default-bucket)) is specified, the table will be uploaded to a bucket whose name is created -from the component and configuration names. -- If the output mapping is specified (through the UI, usually) and its **source** matches the physical file name -in the `/data/out/tables` folder, the **destination** is the name of the table in Storage. An output mapping which -cannot be matched to a physical file produces an error (i.e., fulfilling the output mapping is mandatory). -- If a [manifest file](/extend/common-interface/manifest-files/) exists, it can specify the **destination** of -the table in Storage if no output mapping is present. -- If none of the above options are used, the destination is defined by the name of the file -(for example, `out.c-data.my-table.csv` will create a **my-table** table in the **out-c-data** bucket). The file name -must have at least two dots. -- If neither rule can be applied, an error is thrown. - -Manifests allow you to process files in the `/data/out` folder without explicitly being defined in the -output mapping. That allows for a flexible and dynamic output mapping where the structure is unknown at the beginning. -Using file names (e.g., `out.c-data.my-table.csv`) for an output mapping is great for saving implementation time -in a simple or POC component. - -**Important**: All files in the `/data/out/tables` folder will be uploaded, not only those specified in the output -mapping or manifests. - -This is the expected CSV format ([RFC 4180](https://tools.ietf.org/html/rfc4180)): - - - string enclosure `"` - - delimiter `,` - - no escape character - -A [manifest file](/extend/common-interface/manifest-files/) can specify a different enclosure and delimiter. - -#### Default Bucket -If you cannot define a bucket or want to get it automatically, set -the **Default Bucket** for your component in the [Developer Portal](https://components.keboola.com/). - -All tables in `/data/out/tables` will then be uploaded to a bucket identified by your -component id (generated when the component was created), configuration id (generated when an end-user adds a new component configuration) and stage (`in` or `out`). -The file name (without the `.csv` suffix) will be used as the table name. The `destination` attributes -in the output mapping and file manifests will be overridden. - -**Important**: The `Default Bucket` flag always requires the `config` parameter when creating a job manually using -the API even if the `config` configuration does not exist in Storage. - -#### Sliced Tables - -Sometimes your component will download the CSV file in slices (chunks). You do not need to manually merge them, -simply put them in a subfolder with the same name you would use for a single file. All files found in the -subfolder are considered slices of the table. - - /data/out/tables/myfile.csv/part01 - /data/out/tables/myfile.csv/part02 - /data/out/tables/myfile.csv.manifest - -Sliced files cannot have header rows. They must have their columns specified in the [manifest file](/extend/common-interface/manifest-files/) -or in the output mapping configuration. - -The following is an example of specifying columns in the manifest file `/data/out/tables/myfile.csv.manifest`: - - { - "destination": "in.c-mybucket.table", - "columns": ["col1", "col2", "col3"] - } - -All files from the folder are uploaded irrespective of their name or extension. They are uploaded -to Storage in parallel and in an undefined order. Use sliced tables in case you want to upload tables [larger than 5GB](https://help.keboola.com/storage/file-uploads/#limits). The slices may be compressed by gzip. -A rule of thumb is that slices are [best around 10-100 MB](https://docs.snowflake.net/manuals/user-guide/data-load-considerations-prepare.html#splitting-large-data-files-before-loading) in size **compressed**. - - -### Folder /data/in/files/ - -All files defined in the input mapping are stored in their raw form. File names are numeric and -equal to `{fileId}_{filename}` in Storage. All other information about the files is available -in the [manifest file](/extend/common-interface/manifest-files/). - -### Folder /data/out/files/ - -All files in this folder are uploaded to Storage. File names are preserved, and tags and other upload options -can be specified in the [manifest file](/extend/common-interface/manifest-files/). -Note that all files in the `/data/out/files` folder will be uploaded, not only those specified in the output mapping. - -## Exchanging Data via S3 -The component may also exchange data with Storage [using Amazon S3](https://docs.aws.amazon.com/s3/index.html). -In this case, the data folders contain only [manifest files](/extend/common-interface/manifest-files/) and -not the actual data. This mode of operation can be enabled by setting the **Staging storage input** option to **AWS S3** in -[component settings](https://components.keboola.com/). If this option is enabled, all the data folders -will contain only manifest files, extended with an additional -[`s3` section](/extend/common-interface/manifest-files/#s3-staging). - -**Note**: Exchanging data via S3 is currently only available for input mapping. - -## Exchanging Data via ABS -The component may also exchange data with Storage [using Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/) (ABS). -In this case, the data folders contain only [manifest files](/extend/common-interface/manifest-files/) and -not the actual data. This mode of operation can be enabled by setting the **Staging storage input** option to **ABS** in -[component settings](https://components.keboola.com/). If this option is enabled, all the data folders -will contain only manifest files, extended with an additional -[`abs` section](/extend/common-interface/manifest-files/#abs-staging). - -**Note**: Exchanging data via ABS is currently only available for input mapping. - -## Exchanging Data via Database Workspace - -*Note: this is a preview feature and may change considerably in the future.* - -The component may also exchange data with Storage [using Workspaces](https://keboola.docs.apiary. -io/#reference/workspaces). -This mode of operation can be enabled by setting the **Staging storage input** or **Staging storage output** option -to **Workspace Snowflake**, **Workspace Redshift**, or **Workspace Synapse**. A workspace is an isolated database to -which data are loaded before the component job is run and unloaded when the job finishes. The workspace is created just before the job starts and is -deleted when the job is terminated. - -Using this option will load Storage Tables into the provided storage workspace, but Storage Files will still be -loaded into the local filesystem like in the standard configuration. - -If this option is enabled, the table data folder will contain only manifest files. The actual data will be loaded as -database tables into the workspace database. The `destination` in input and `source` in output refer to database -table names. This mode of operation is useful for components which want to manipulate data using SQL queries. -The component can run arbitrary queries against the database. The database credentials are available in the -[`authorization` section](/extend/common-interface/config-file/#configuration-file-structure) of the configuration file: - -{% highlight json %} -{ - "storage": { - - }, - "parameters": { - ... - }, - "authorization": { - "workspace": { - "host": "database.example.com", - "warehouse": "test", - "database": "my-db", - "schema": "my-schema" - "user": "john-doe", - "password": "secret" - } - } -} -{% endhighlight %} - -Notice that some of the values might be empty for different workspace backends (e.g., Redshift is not using `warehouse`). -They will be always present, though. - -When exchanging data via workspace, there are couple of differences to loading data into files: -- Loading to workspaces supports only [storage tables](/storage/tables/), [storage files](/storage/file-uploads/) -are always saved to the directory structure. -- The `days` attribute is not supported for filtering table, use `changed_since` instead. -- [Automatic Incremental Processing](https://help.keboola.com/storage/tables/#automatic-incremental-processing) (also known as Adaptive Input Mapping) is not supported. -- When used for output mapping, the `columns` of the output table **must be** specified, this can be done either in the [output manifest](/extend/common-interface/manifest-files/#dataouttables-manifests) or in the [output mapping](/extend/common-interface/config-file/#output-mapping--headless-csv). - -**Note**: Currently only some combinations of input/output staging storage settings are supported: -`local<->local`, `local<->s3`, `workspace-snowflake<->workspace-snowflake`, `workspace-redshift<->workspace-redshift`. - -## Exchanging Data via File System Workspace -*Note: this is a preview feature and may change considerably in future.* -*Note: currently only Azure Blob Storage workspaces (abs-workspace) are supported for this type and those only work with Synapse storage backend - -The component may also exchange data with a provisioned file workspace (Azure Blob Storage) [using Workspaces](https://api.keboola.com/?service=storage#tag--Workspaces). -This mode of operation can be enabled by setting the **Staging storage input** or **Staging storage output** option -to **Workspace ABS**. A filesystem workspace is an isolated file storage to which data are loaded before the component job is run (when staging storage input is set) -and unloaded from when the job finishes (when staging storage output is set). -The workspace is created just before the job starts and is deleted when the job is terminated. - -If this option is enabled, the data and the manifests will be loaded to the azure storage blob container under the -data folder similarly to how it does when using the default [local filesystem](extend/common-interface/folders/#root-folder-data). - -### Files -Files are loaded into the workspace as `[file name]/[file ID]`. For example, if a file 'test.txt' with ID '12345' is in -the input mapping then the file will appear in the storage blob container with URL `https://[storage_account_name].blob.core.windows.net/[container-name]/data/in/files/test.txt/12345` - -### Tables -*Note that this is only available on Synapse storage backend* - -Synapse only exports tables as sliced files. -So for example, if you set as table input mapping the table `in.c-main.my-input` as source and `my-input.csv` as -destination then in the ABS workspace you will find it with the following structure: -- [containerName]/data/in/tables/my-inpupt.csv/[random identifier1].txt -- [containerName]/data/in/tables/my-inpupt.csv/[random identifier2].txt -- [containerName]/data/in/tables/my-inpupt.csv/[random identifier3].txt - -### Mappings - -To sum up, below is a sample storage configuration and where the files are written from and to: - -| Direction | Source | Destination | -| --- | --- | --- | -| input | in.c-main.my-table-from-abs-workspace | Many slices like `[abs-workspace-root]/data/in/tables/my-inpupt-table.csv/[random identifier].txt` | -| input | file with tag `my-input-files` named `input-file.txt` | `[abs-workspace-root]/data/in/files/test.txt/12345` | -| output | `[abs-workspace-root]/data/out/tables/my-output-table.csv` | out.c-main.my-table-from-abs-workspace | -| output | `[abs-workspace-root]/data/out/files/my-file.txt` | file `my-file.txt` with tag `uploaded-from-abs-workspace` | - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.my-table-from-abs-workspace", - "destination": "my-input-table.csv" - }, - ... - ], - "files": [ - { - "tags": ["my-input-files"] - } - ] - } - "output": { - "tables": [ - { - "source": "my-output-table.csv", - "destination": "out.c-main.my-table-from-abs-workspace" - }, - ... - ], - "files": [ - { - "source": "my-file.txt", - "tags": ["uploaded-from-abs-workspace"] - } - ] - } - }, - ... -} -{% endhighlight %} - -### Authorization -[`authorization` section](/extend/common-interface/config-file/#configuration-file-structure) of the configuration file: - -To connect to the ABS storage workspace you need to use the [SAS connection string](https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string) which is stored in the authorization section as -shown below. - -{% highlight json %} -{ - "storage": { - ... - }, - "parameters": { - ... - }, - "authorization": { - "workspace": { - "container": "azure-storage-blob-container", - "connectionString": "azure-storage-blob-SAS-connection-string", - } - } -} -{% endhighlight %} diff --git a/extend/common-interface/index.md b/extend/common-interface/index.md index 2e70ab55..289839a9 100644 --- a/extend/common-interface/index.md +++ b/extend/common-interface/index.md @@ -1,36 +1,5 @@ --- title: Common Interface permalink: /extend/common-interface/ +redirect_to: https://help.keboola.com/extend/common-interface/ --- - -To exchange data between your component and Keboola, use - -* a predefined set of input and output [folders](/extend/common-interface/folders) for tables and files, -* a [configuration file](/extend/common-interface/config-file/), -* [environment](/extend/common-interface/environment/) variables and return values. - -Optionally, you can use - -* [logging](/extend/common-interface/logging), -* [manifest files](/extend/common-interface/manifest-files/) for working with table and file meta-data, -* the [OAuth](/extend/common-interface/oauth/) part of the configuration file, and -* [actions](/extend/common-interface/actions/) for quick synchronous tasks. - -In addition to that, [Job Queue](/extend/job-queue/) provides tools for -[encryption](/overview/encryption) and [OAuth2 authorization](/extend/common-interface/oauth/). - -To quickly get the picture, have a look a [random sample data folder](/extend/data.zip). - -Our Python library [PyPi](https://pypi.org/project/keboola.component/) provides a convenient way how to interact with the Common Interface. Learn more in the [documentation](https://keboola.github.io/python-component/index.html#header-submodules). - -### Component Limits -Even though you can define your own limits for your component, all components are also subject to the following service limits: - -* Both memory and swap sizes are set to an equal value -* Docker [overlay2](https://docs.docker.com/storage/storagedriver/overlayfs-driver/) size is set to 10 GB - -The size allocated for overlay2 is consumed by memory swapping, and all other operations in the component -(for instance, ad hoc module installations); only input and output folders (`/data/`) and `/tmp/` are excluded. -As the swap size cannot be larger than the allocated disk space, we cannot safely increase the memory limit over 8 GB. - -If you need more than 8 GB of memory/swap or larger disk space, get in touch with us to discuss possible solutions. diff --git a/extend/common-interface/logging.md b/extend/common-interface/logging.md index 1e7ffaa9..fc8d6f4f 100644 --- a/extend/common-interface/logging.md +++ b/extend/common-interface/logging.md @@ -1,153 +1,5 @@ --- title: Logging Specification permalink: /extend/common-interface/logging/ +redirect_to: https://help.keboola.com/extend/common-interface/logging/ --- - -* TOC -{:toc} - -There are two main, mutually exclusive, ways in which your component can display events Keboola end-users: - -1. Using [standard output and standard error](https://en.wikipedia.org/wiki/Standard_streams) -2. Using [Graylog GELF](http://docs.graylog.org/en/3.1/pages/gelf.html) compatible logger - -Using the standard output option requires **no extra work** from you or your component. -You just print all informational messages to standard output and all error messages to standard error. -These will be forwarded to Storage Events as informational or error messages. See -[implementation notes](/extend/component/implementation/) for best practices in logging. - -Using a [GELF](http://docs.graylog.org/en/3.1/pages/gelf.html) compatible logger requires implementing or including -such a logger in your component. However, it offers much **greater flexibility**: you can send different -kinds of messages (such as error, informational, warning, debug), and they can contain additional -structured information (not only a plain text string). - -## Standard Output and Standard Error -By default -- unless you have turned on [GELF logging](/extend/common-interface/logging/#gelf-logger) in the -[component configuration](https://components.keboola.com/), -[Job Queue](/extend/job-queue/) listens to [STDOUT](https://en.wikipedia.org/wiki/Standard_streams#Standard_output_.28stdout.29) -and [STDERR](https://en.wikipedia.org/wiki/Standard_streams#Standard_error_.28stderr.29) -of the component and forwards the STDOUT content live to [Storage API Events](https://api.keboola.com/?service=storage#tag--Events) -(log level `info`). The content of STDERR is collected and added (if not empty) as the last event of the job with level `error`. -The events are displayed in a [Job detail](https://help.keboola.com/management/jobs/). - -The entire output from a component is filter for sensitive values. The [Job Queue](/extend/job-queue/) -keeps track of all encrypted values and if it encounters them in the component output, it replaces -them by `[hidden]` placeholder. This prevents accidental leaking of sensitive information for -example in exception traces. - -## GELF Logger -[GELF](http://docs.graylog.org/en/3.1/pages/gelf.html) is a log format allowing you to -send [structured](http://docs.graylog.org/en/3.1/pages/gelf.html#gelf-payload-specification) event messages. -The messages can be sent over several transports and you can specify whether they will be silenced or displayed based on their level. - -*Note: The size of the messages is limited. Sending a message larger than 200KB will cause the component job to fail.* - -### Setting Up -If you turn on GELF logging in the [component configuration](https://components.keboola.com/), -our [Job Queue](/extend/job-queue/) will listen -for messages on the **transport** you specify ([UDP](https://en.wikipedia.org/wiki/User_Datagram_Protocol), -[TCP](https://en.wikipedia.org/wiki/Transmission_Control_Protocol) and -[HTTP](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) are supported). -We suggest using TCP as it offers a nice compromise between transport overhead and reliability, but the final choice is up to you. -If you choose UDP as a transport, make sure that there is a little delay between your component start -and the first message sent (about 1s) to give the network sockets some time to initialize. - -Additionally, you can set the visibility of each event message as follows: - -- `none` -- Message is ignored entirely. -- `camouflage` -- Generic error message is shown to the end-user instead of the real message content; the full message is logged internally. -- `normal` -- Event message (GELF `short_message` field) is shown to the end-user; the full message is logged internally. -- `verbose` -- Full message is shown to the user including GELF additional fields. - -Default settings for message visibilities: - -[Keboola Level](https://github.com/php-fig/fig-standards/blob/master/accepted/PSR-3-logger-interface.md#5-psrlogloglevel) | Gelf Log Method | [Syslog Level](https://en.wikipedia.org/wiki/Syslog#Severity_level) | Default Keboola Verbosity -100 | `debug()` | 7 | none -200 | `info()` | 6 | normal -250 | `notice()` | 5 | normal -300 | `warning()` | 4 | normal -400 | `error()` | 3 | normal -500 | `critical()` | 2 | camouflage -550 | `alert()` | 1 | camouflage -600 | `emergency()` | 0 | camouflage - -### Examples -Since GELF is sort of a standard format for structured logs, there are a [number of libraries](https://marketplace.graylog.org/addons?kind=gelf) -available for client implementation. The following examples show how to use the GELF logger in some common languages. -Always use the `KBC_LOGGER_ADDR` and `KBC_LOGGER_PORT` environment variables in your client, -which will be injected into your component by our Job Queue. - -**Important:** Never rely on the default logger settings. - -#### PHP -For PHP, use the official [GELF client](https://github.com/bzikarsky/gelf-php) library. To install it, use - - composer require graylog2/gelf-php - -Then test that logging works with this simple script: - - -{% highlight php %} -emergency("A sample emergency message", ["some" => ["structured" => "data"]]); -{% endhighlight %} - -For other transports, use the -`UdpTransport` or `HttpTransport` class (AMQP transport is not supported yet). For additional examples on using the library, -see its [official documentation](https://github.com/bzikarsky/gelf-php). - -#### Python -For Python, we strongly suggest using the prepared [Component package](/extend/component/implementation/python/#using-keboola-python-package) which takes care of the setup automatically. - -If you want to set a GELF logger yourself, you need to choose from [a number of libraries](https://marketplace.graylog.org/addons?kind=gelf&tag=python) available. For example, the [logging-gelf library](https://pypi.org/project/logging-gelf/). To install it, use - - pip3 install logging_gelf - -Then test that logging works with this simple script: - -{% highlight python %} -import logging_gelf.handlers -import logging_gelf.formatters -import logging -import os - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger() -logging_gelf_handler = logging_gelf.handlers.GELFTCPSocketHandler(host=os.getenv('KBC_LOGGER_ADDR'), port=int(os.getenv('KBC_LOGGER_PORT'))) -logging_gelf_handler.setFormatter(logging_gelf.formatters.GELFFormatter(null_character=True)) -logger.addHandler(logging_gelf_handler) - -# remove default logging to stdout -logger.removeHandler(logger.handlers[0]) - -logging.critical('A sample emergency message') -{% endhighlight %} - -Due to the nature of Python logging, only [some error levels](https://docs.python.org/3.8/library/logging.html#logging-levels) are -permitted. - -#### Node.js -There are a number of libraries available for [NodeJS](https://marketplace.graylog.org/addons?kind=gelf&tag=nodejs). -For example, the [GrayGelf library](https://github.com/wavded/graygelf). - - npm install graygelf - -Then test that logging works with this simple script: - -{% highlight js %} -var log = require('graygelf')({ - host: process.env.KBC_LOGGER_ADDR, - port: process.env.KBC_LOGGER_PORT -}) - -log.info('hello', 'world') -log.info.a('short', 'full', { foo: 'bar' }) -{% endhighlight %} - -Note that the library supports only the UDP transport. diff --git a/extend/common-interface/manifest-files.md b/extend/common-interface/manifest-files.md index fc61d121..b1549527 100644 --- a/extend/common-interface/manifest-files.md +++ b/extend/common-interface/manifest-files.md @@ -1,25 +1,5 @@ --- title: Manifest Files Specification permalink: /extend/common-interface/manifest-files/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/ --- - -A manifest file contains additional information about tables and files injected to the -[`/data/in` folders](/extend/common-interface/folders/). -It also provides a way to specify options for tables and files transferred back to Storage from `/data/out` -folders. Manifest files have the `.manifest` suffix to the original file. - -All files in `/data/in` have the manifest file generated by us. For files generated by your code -in `/data/out`, the manifest file **is recommended**. Also, keep in mind that all manifests have a lower priority -than input and output mapping. - -## Format - -The format of the manifest file is always *JSON*. The manifest -file always has the `.manifest` extension. This applies to files with multiple extensions as well, so the following -filenames are expected: - -| Data File Name | Manifest File Name | -|----------------|--------------------------| -| myfile | myfile.manifest | -| myfile.csv | myfile.csv.manifest | -| myfile.csv.gz | myfile.csv.gz.manifest | diff --git a/extend/common-interface/manifest-files/column-data-type-override.png b/extend/common-interface/manifest-files/column-data-type-override.png deleted file mode 100644 index b38959bd..00000000 Binary files a/extend/common-interface/manifest-files/column-data-type-override.png and /dev/null differ diff --git a/extend/common-interface/manifest-files/column-data-type-use.png b/extend/common-interface/manifest-files/column-data-type-use.png deleted file mode 100644 index b770829e..00000000 Binary files a/extend/common-interface/manifest-files/column-data-type-use.png and /dev/null differ diff --git a/extend/common-interface/manifest-files/column-data-type.png b/extend/common-interface/manifest-files/column-data-type.png deleted file mode 100644 index c82ee625..00000000 Binary files a/extend/common-interface/manifest-files/column-data-type.png and /dev/null differ diff --git a/extend/common-interface/manifest-files/in-files-abs-staging.md b/extend/common-interface/manifest-files/in-files-abs-staging.md index dc709a4d..a8376229 100644 --- a/extend/common-interface/manifest-files/in-files-abs-staging.md +++ b/extend/common-interface/manifest-files/in-files-abs-staging.md @@ -1,31 +1,5 @@ --- title: ABS Staging permalink: /extend/common-interface/manifest-files/in-files-abs-staging/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/in-files-abs-staging/ --- - -When using [Azure Blob Storage for direct data exchange](/extend/common-interface/folders/#exchanging-data-via-abs), -the manifest files in the component’s working directory will contain an additional `abs` section with -credentials for downloading the actual file data. - -{% highlight json %} -{ - "id": "in.c-docker-demo.data", - ... - "abs": { - "is_sliced": true, - "region": "us-east-1", - "container": "exp-2-export-7647-627703071-in-c-docker-test-test", - "name": "627703071.csv.gzmanifest", - "credentials": { - "sas_connection_string": "BlobEndpoint=https://kbcfsdxcgtsezztoqc.blob.core.windows.net;SharedAccessSignature=sv=2017-11-09&sr=c&st=2020-08-27T08:42:08Z&se=2020-08-27T20:42:08Z&sp=rl&sig=UJW4DPh%2Baaaaaaaaaa", - "expiration": "2020-08-27T22:42:08+0200" - } - } -} -{% endhighlight %} - -If the file is sliced and you need to merge it into a single file, read through the guide to -[working with sliced files](/integrate/storage/api/import-export/#working-with-sliced-files). -In that case, the `name` points to another manifest, which contains a list of sliced files. - -Note: Exchanging data via Azure ABS is currently available only for input mapping. diff --git a/extend/common-interface/manifest-files/in-files-manifests.md b/extend/common-interface/manifest-files/in-files-manifests.md index fd4afb54..9f5d326e 100644 --- a/extend/common-interface/manifest-files/in-files-manifests.md +++ b/extend/common-interface/manifest-files/in-files-manifests.md @@ -1,25 +1,5 @@ --- title: /data/in/files manifests permalink: /extend/common-interface/manifest-files/in-files-manifests/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/in-files-manifests/ --- - -#### `/data/in/files` manifests - -An input file manifest stores metadata about a downloaded file from Storage Files to the component’s working directory. - -{% highlight json %} -{ - "id": 75807657, - "created": "2015-01-14T00:47:00+0100", - "is_public": false, - "is_sliced": false, - "is_encrypted": true, - "name": "fooBar.jpg", - "size_bytes": 563416, - "tags": [ - "tag1", - "tag2" - ], - "max_age_days": 15 -} -{% endhighlight %} \ No newline at end of file diff --git a/extend/common-interface/manifest-files/in-files-s3-staging.md b/extend/common-interface/manifest-files/in-files-s3-staging.md index 783d9a8e..1233aaeb 100644 --- a/extend/common-interface/manifest-files/in-files-s3-staging.md +++ b/extend/common-interface/manifest-files/in-files-s3-staging.md @@ -1,32 +1,5 @@ --- title: S3 Staging permalink: /extend/common-interface/manifest-files/in-files-s3-staging/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/in-files-s3-staging/ --- - -When using [AWS S3 for direct data exchange](/extend/common-interface/folders/#exchanging-data-via-s3), -the manifest files in the component’s working directory will contain an additional `s3` section with -credentials for downloading the actual file data. - -{% highlight json %} -{ - "id": "in.c-docker-demo.data", - ... - "s3": { - "isSliced": true, - "region": "us-east-1", - "bucket": "kbc-sapi-files", - "key": "exp-2/1581/table-exports/in/c-docker-test/test/243100072.csv.gzmanifest", - "credentials": { - "access_key_id": "ASI...CDQ", - "secret_access_key": "tCE..I+T", - "session_token": "Ago...POP" - } - } -} -{% endhighlight %} - -If the file is sliced and you need to merge it into a single file, read through the guide to -[working with sliced files](/integrate/storage/api/import-export/#working-with-sliced-files). -In that case, the `key` points to another manifest, which contains a list of sliced files. - -Note: Exchanging data via AWS S3 is currently available only for input mapping. diff --git a/extend/common-interface/manifest-files/in-tables-manifests.md b/extend/common-interface/manifest-files/in-tables-manifests.md index 0410c8e9..59c5de76 100644 --- a/extend/common-interface/manifest-files/in-tables-manifests.md +++ b/extend/common-interface/manifest-files/in-tables-manifests.md @@ -1,107 +1,5 @@ --- title: /data/in/tables manifests permalink: /extend/common-interface/manifest-files/in-tables-manifests/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/in-tables-manifests/ --- - -An input table manifest stores metadata about a downloaded table from Storage Tables to the component’s working directory. -For example, a table -with the ID `in.c-docker-demo.data` will be downloaded into -`/in/tables/in.c-docker-demo.data.csv` (unless stated otherwise in the -[input mapping](/extend/common-interface/config-file/) and a manifest file -'/in/tables/in.c-docker-demo.data.csv.manifest' will be created with the following -contents: - -{% highlight json %} -{ - "id": "in.c-docker-demo.data", - "uri": "https://connection.keboola.com//v2/storage/tables/in.c-docker-demo.data", - "name": "data", - "primary_key": [], - "created": "2015-01-25T01:35:14+0100", - "last_change_date": "2015-01-25T01:35:14+0100", - "last_import_date": "2015-01-25T01:35:14+0100", - "description": "My table description", - "metadata": { - "KBC.createdBy.component.id": "keboola.python-transformation", - "KBC.createdBy.configuration.id": "123456" - }, - "column_metadata": { - "id": [], - "name": [], - "text": [] - }, - "schema": [ - { - "name": "id", - "data_type": { - "base": { - "type": "INTEGER" - }, - "snowflake": { - "type": "NUMBER", - "length": "38,0" - } - }, - "nullable": false, - "primary_key": true, - "description": "Identifier of the record" - }, - { - "name": "name", - "data_type": { - "base": { - "type": "STRING" - }, - "snowflake": { - "type": "VARCHAR", - "length": "16777216" - } - }, - "nullable": true, - "primary_key": false - }, - { - "name": "text", - "data_type": { - "base": { - "type": "STRING" - }, - "snowflake": { - "type": "VARCHAR", - "length": "16777216" - } - }, - "nullable": true, - "primary_key": false - } - ] -} -{% endhighlight %} - -The `name` node refers to the name of the component configuration. -The `metadata` and `column_metadata` fields contain -Metadata for the table and its columns. -The `metadata` field corresponds to the [Table Metadata API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/metadata). -The `column_metadata` field corresponds to the [Column Metadata API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/columns/-id-/metadata). - -The `description` field contains the table description. It is read primarily from the table's -native description field; when that field is empty, it falls back to the `KBC.description` value -in the table `metadata`. The field is omitted when no description is available. - -The `schema` field describes the columns of the downloaded table, including their data types. It is -built from the [table definition](https://help.keboola.com/storage/tables/data-types/). The columns it -lists, and their order, match the `columns` node. Each object in the `schema` array represents one column: -- The `name` field specifies the column name. -- The `data_type` field describes the column's data type. The `base` type is the backend-agnostic - [base type](https://help.keboola.com/storage/tables/data-types/#base-types); an additional key named - after the table's [storage backend](https://help.keboola.com/storage/#storage-data) (e.g., `snowflake`, - `bigquery`) carries the type as it exists on that backend, together with its `length` and `default` - when set. This field is omitted for columns that are not typed. -- The `nullable` field indicates whether the column can contain null values. -- The `primary_key` field indicates whether the column is part of the table's primary key. -- The `description` field contains the column description, read primarily from the column's native - description field and falling back to its `KBC.description` metadata. It is omitted when no description - is available. - -The `schema` field uses the same structure as the -[output table manifest schema](/extend/common-interface/manifest-files/out-tables-manifests-native-types/). \ No newline at end of file diff --git a/extend/common-interface/manifest-files/out-files-manifests.md b/extend/common-interface/manifest-files/out-files-manifests.md index 71a9b2a7..bfbb11e2 100644 --- a/extend/common-interface/manifest-files/out-files-manifests.md +++ b/extend/common-interface/manifest-files/out-files-manifests.md @@ -1,26 +1,5 @@ --- title: /data/out/files manifests permalink: /extend/common-interface/manifest-files/out-files-manifests/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/out-files-manifests/ --- - -#### `/data/out/files` manifests - -An output file manifest sets options for transferring a file to Storage. The following example lists available -manifest fields; all of them are optional. - -{% highlight json %} -{ - "is_permanent": true, - "is_encrypted": true, - "notify": false, - "tags": [ - "image", - "pie-chart" - ] -} -{% endhighlight %} - -These parameters can be used (taken from [Storage API File Import](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/files/prepare)): - -- If `is_permanent` is false, the file will be automatically deleted after 15 days. -- When `notify` is true, the members of the project will be notified that a file has been uploaded to the project. diff --git a/extend/common-interface/manifest-files/out-tables-manifests-native-types.md b/extend/common-interface/manifest-files/out-tables-manifests-native-types.md index 0c7eecbd..f9d4ceb2 100644 --- a/extend/common-interface/manifest-files/out-tables-manifests-native-types.md +++ b/extend/common-interface/manifest-files/out-tables-manifests-native-types.md @@ -1,1087 +1,5 @@ --- title: /data/out/tables manifests with Native Types permalink: /extend/common-interface/manifest-files/out-tables-manifests-native-types/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/out-tables-manifests-native-types/ --- - -Native Types provide a structured way for components to define their handling of data types, referred to as “Native Types.” - -The level of type handling is specified by the dataTypeSupport property, which can take one of three values: -- Authoritative: The component reliably enforces specific data types. -- Hints: The component provides type suggestions that may not always be reliable. -- None: Represents the legacy state with no explicit type handling. - -This design overcomes limitations in current settings, where all components automatically produce typed tables when a project switches to Native Types. For instance, Data Sources that output unreliable type hints (e.g., an int column containing values like N/A) can now explicitly signal their limitations, reducing downstream issues. - -An output table manifest sets options for transferring a table to Storage. The following examples list available -manifest fields; **all of them are optional**. The `destination` field overrides the table name generated -from the file name; it can (and commonly is) overridden by the end-user configuration. - -{% highlight json %} -{ - "destination": "out.c-main.Leads", - "incremental": true, - "delimiter": "\t", - "enclosure": "\"", - "manifest_type": "output", - "has_header": true, - "description": "Best table", - "table_metadata": ... - "schema": ... -} -{% endhighlight %} - -The `table_metadata` fields allow you to set -Metadata for the table. -The `table_metadata` field corresponds to the [Table Metadata API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/metadata). -The `key` and `value` of the object are passed directly to the API; the `provider` value is -filled by the Id of the running component (e.g., `keboola.ex-db-snowflake`). - -{% highlight json %} -{ - ... - "table_metadata": { - "something else": "a value" - } -} -{% endhighlight %} - -Additionally, the following options will cause the specified rows to be deleted from the source table before the new -table is imported. See an [example](/extend/common-interface/config-file/#output-mapping---delete-rows). -Using this option makes sense only with [incremental loads](/extend/generic-extractor/incremental/). - -{% highlight json %} -{ - ... - "delete_where": [ - { - "where_filters": [ - { - "column": "column name", - "operator": "eq", - "values_from_set": ["value1", "value2"] - } - ] - } - ] -} -{% endhighlight %} - -The `schema` [optional] field allow you to create a table with Native Data Types columns. -Each object in the `schema` array represents one column: -- The `name` [required] field specifies the column name. -- The `data_type` [optional] field defines the data type for different [storage backends](https://help.keboola.com/storage/#storage-data), referred to as "Native Types". - - The `base` [required] type is always required and can have values specified in the [Base Types documentation](https://help.keboola.com/storage/tables/data-types/#base-types). - - Other types like Snowflake and BigQuery are optional and allows you to specify settings for a particular database backend. -- The `nullable` [optional] field indicates if the column can be null. -- The `primary_key` [optional] field specifies if the column is a primary key. -- The `description` [optional] field provides a description of the column. -- The `metadata` [optional] field allows setting additional metadata for the column. - -{% highlight json %} -{ - "schema": [ - { - "name": "id", - "data_type": { - "base": { - "type": "INTEGER", - "length": "11", - "default": "123" - }, - "snowflake": { - "type": "GEOMETRY", - "length": "123,123,4455", - "default": "POINT(1 1)" - }, - "bigquery": { - "type": "VARCHAR", - "length": "123", - "default": null - } - }, - "nullable": false, - "primary_key": true, - "description": "Optional description of the column", - "metadata": { - "KBC.someColumnMetadata": "value1" - "KBC.someOther": "value2" - } - } - ] -} -{% endhighlight %} - -## Base Types -Source data types are mapped to a destination using a **base type**. The current base types are -[`STRING`](#string), [`INTEGER`](#integer), [`NUMERIC`](#numeric), [`FLOAT`](#float), [`BOOLEAN`](#boolean), -[`DATE`](#date), and [`TIMESTAMP`](#timestamp). This means that, for example, a MySQL extractor -may store the value `BIGINT` as a type of column; that type maps to the `INTEGER` base type. When the Snowflake writer consumes this value, it will -read the base type `INTEGER` and choose a corresponding type for Snowflake, which happens to be also `INTEGER`. -This ensures high interoperability between the components. Please take a look at the [conversion table below](#data-type-conversions). - -View the extracted data types in the storage tables detail: - -{: .image-popup} -![Screenshot - View Column Data Type](/extend/common-interface/manifest-files/column-data-type.png) - -You can also override the data type: - -{: .image-popup} -![Screenshot - Set Column Data Type](/extend/common-interface/manifest-files/column-data-type-override.png) - -When you use the table (e.g., in the Snowflake writer), you'll see the data type you have configured: - -{: .image-popup} -![Screenshot - Set Column Data Type](/extend/common-interface/manifest-files/column-data-type-use.png) - -The data is converted only when writing or copying (e.g., to a transformation or a writer). -That means that you can extract an *integer* column, mark it as a *timestamp* in storage and write it as -an *integer* into a target database (though you'll be offered to write it as a timestamp). - -You access both the source and base data type through the corresponding [API](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/tables/-id-). - -## Nullable Conversion -Nullable conversion, which transforms an empty string originating from data into a null value, refers to the process where a textual value consisting solely of an empty string `""` is replaced with the value null. - -## Data Type Conversions -As described above, the **source data type** is converted to a **base data type** stored in metadata storage. The base type is then converted to the **target data type**. The following tables show mappings for each base type. The mapping -causes possible information loss (e.g., assigning `SMALLINT` to `INTEGER`). To minimize this, we also keep track of the data type -size and transfer that if possible. For example, a `SMALLINT` column would be stored as base type `INTEGER` with size `2`. If the target database supports integer sizes, you will be offered to set the type in the target database as `INTEGER(2)`. - -### STRING -Base type `STRING` represents any textual type; both `CHARACTER VARYING` (or `VARCHAR`) and `TEXT` types are included. -Also, the string base type is used for any other unrecognized type on input. It means that the -*source type* column is **not an exhaustive list** in the following table. It's a list of suitable string types converted to a string. All -other unknown types are converted to a string as well. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceSource TypeBase TypeTarget TypeTarget
GenericcharSTRINGN/A
character varying
text
varchar
STRINGHive
STRINGImpala
TEXTMS SQL Server
MySQLCHARVARCHARMySQL
TEXT
VARCHAR
VARCHAR2Oracle
VARCHARPostgreSQL
RedshiftBPCHARVARCHARRedshift
CHAR
CHARACTER
CHARACTER VARYING
NCHAR
NVARCHAR
TEXT
VARCHAR
VARCHARSiSense
SnowflakeBINARYVARCHARSnowflake
CHAR
CHARACTER
STRING
TEXT
VARBINARY
VARCHAR
SynapseBINARYNVARCHARSynapse
CHAR
NCHAR
NVARCHAR
VARBINARY
VARCHAR
VARCHARThoughtspot
SourceSource TypeBase TypeTarget TypeTarget
- -### INTEGER -The `INTEGER` base type represents data types for whole numbers. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceSource TypeBase TypeTarget TypeTarget
GenericbigintINTEGER
bigserial
mediumint
smallint
int
int2
int4
int64
int8
integer
serial8
tinyint
INTHive
INTImpala
BIGINTMS SQL
MySQLBIGINTINTEGERMySQL
INT
INTEGER
MEDIUMINT
SMALLINT
TINYINT
N/AOracle
INTEGERPostgres
RedshiftBIGINTINTEGERRedshift
INT
INT2
INT4
INT8
INTEGER
SMALLINT
BIGINTSiSense
SnowflakeBIGINTINTEGERSnowflake
BYTEINT
INT
INTEGER
SMALLINT
TINYINT
SynapseBIGINTINTSynapse
INT
SMALLINT
TINYINT
INTThoughtspot
SourceSource TypeBase TypeTarget TypeTarget
- -### NUMERIC -The `NUMERIC` base type represents [fixed-point](https://en.wikipedia.org/wiki/Fixed-point_arithmetic) fractional numbers -(`real`, `numeric` or `decimal` data types). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceSource TypeBase TypeTarget TypeTarget
GenericdecNUMERIC
decimal
fixed
money
number
numeric
smallmoney
DECIMALHive
DECIMALImpala
DECIMALMS SQL Server
MySQLDECNUMERICMySQL
DECIMAL
FIXED
NUMERIC
NUMBEROracle
NUMERICPostgreSQL
RedshiftDECIMALNUMERICRedshift
NUMERIC
DECIMALSiSense
SnowflakeDECIMALNUMBERSnowflake
NUMBER
NUMERIC
SynapseNUMERICNUMERICSynapse
DECIMAL
N/AThoughtspot
SourceSource TypeBase TypeTarget TypeTarget
- -### FLOAT -The `FLOAT` base type represents [floating-point](https://en.wikipedia.org/wiki/Floating_point) fractional numbers -(`float` or `double` data types). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceSource TypeBase TypeTarget TypeTarget
Genericbinary_doubleFLOAT
binary_float
double
double precision
d_float
float
float4
float8
quad
real
FLOATHive
FLOATImpala
FLOATMS SQL Server
MySQLDOUBLEFLOATMySQL
DOUBLE PRECISION
FLOAT
REAL
N/AOracle
REALPostgreSQL
RedshiftDOUBLE PRECISIONFLOATRedshift
FLOAT
FLOAT4
FLOAT8
REAL
FLOATSiSense
SnowflakeFLOATDOUBLESnowflake
DOUBLE PRECISION
FLOAT
FLOAT4
FLOAT8
REAL
SynapseFLOATFLOATSynapse
REAL
FLOATThoughtspot
SourceSource TypeBase TypeTarget TypeTarget
- -### BOOLEAN -The `BOOLEAN` base type represents a true or false value. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceSource TypeBase TypeTarget TypeTarget
GenericboolBOOLEAN
boolean
BOOLEANHive
BOOLEANImpala
BITMS SQL Server
N/AMySQL
N/AOracle
BOOLEANPostgreSQL
RedshiftBOOLBOOLEANRedshift
BOOLEAN
BITSiSense
SnowflakeBOOLEANBOOLEANSnowflake
SynapseBITBITSynapse
BOOLThoughtspot
SourceSource TypeBase TypeTarget TypeTarget
- -### DATE -The `DATE` base type represents a date value without a time portion. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceSource TypeBase TypeTarget TypeTarget
GenericdateDATEDATE
N/AHive
N/AImpala
DATEMS SQL Server
MySQLDATEDATEMySQL
DATEOracle
DATEPostgreSQL
RedshiftDATEDATERedshift
DATESiSense
SnowflakeDATEDATESnowflake
SynapseDATEDATESynapse
DATEThoughtspot
SourceSource TypeBase TypeTarget TypeTarget
- -### TIMESTAMP -The `TIMESTAMP` base type represents a date value with a time portion. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceSource TypeBase TypeTarget TypeTarget
GenericdatetimeTIMESTAMP
datetime2
datetimeoffset
smalldatetime
timestamp
timestamptz
timestamp_LTZ
timestamp_NTZ
TIMESTAMP_TZ
timestamp with local time zone
timestamp with time zone
timestamp without time zone
TIMESTAMPHive
TIMESTAMPImpala
DATETIME2MS SQL Server
MySQLDATETIMETIMESTAMPMySQL
TIMESTAMP
TIMESTAMPOracle
TIMESTAMPPostgreSQL
RedshiftTIMESTAMPTIMESTAMPRedshift
TIMESTAMPTZ
TIMESTAMP WITH TIME ZONE
TIMESTAMP WITHOUT TIME ZONE
N/ASiSense
SnowflakeDATETIMETIMESTAMPSnowflake
TIMESTAMP
TIMESTAMP_NTZ
TIMESTAMP_LTZ
TIMESTAMP_TZ
SynapseDATETIMEOFFSETDATETIMEOFFSETSynapse
DATETIME
DATETIME2
SMALLDATETIME
TIME
TIMESTAMPThoughtspot
SourceSource TypeBase TypeTarget TypeTarget
diff --git a/extend/common-interface/manifest-files/out-tables-manifests.md b/extend/common-interface/manifest-files/out-tables-manifests.md index 5e413488..7c6b09c0 100644 --- a/extend/common-interface/manifest-files/out-tables-manifests.md +++ b/extend/common-interface/manifest-files/out-tables-manifests.md @@ -1,78 +1,5 @@ --- title: /data/out/tables manifests permalink: /extend/common-interface/manifest-files/out-tables-manifests/ +redirect_to: https://help.keboola.com/extend/common-interface/manifest-files/out-tables-manifests/ --- - -An output table manifest sets options for transferring a table to Storage. The following examples list available -manifest fields; **all of them are optional**. The `destination` field overrides the table name generated -from the file name; it can (and commonly is) overridden by the end-user configuration. The `columns` option defines -the columns of the imported table. If the `columns` option is provided, then the CSV files are **assumed to be headless**. -If you the component is producing [Sliced tables](/extend/common-interface/folders/#sliced-tables), then they are always -assumed to be headless and you *have to* use the `columns` option. - -{% highlight json %} -{ - "destination": "out.c-main.Leads", - "columns": ["column1", "column2", "column3"], - "incremental": true, - "primary_key": ["column1", "column2"], - "delimiter": "\t", - "enclosure": "\"", - "metadata": ..., - "column_metadata": ... -} -{% endhighlight %} - -Additionally, the following options can be specified: - -{% highlight json %} -{ - ... - "delete_where": [ - { - "where_filters": [ - { - "column": "column name", - "operator": "eq", - "values_from_set": ["value1", "value2"] - } - ] - } - ] -} -{% endhighlight %} - -The options will cause the specified rows to be deleted from the source table before the new -table is imported. See an [example](/extend/common-interface/config-file/#output-mapping---delete-rows). -Using this option makes sense only with [incremental loads](/extend/generic-extractor/incremental/). - -The `metadata` and `column_metadata` fields allow you to set -Metadata for the table and its columns. -The `metadata` field corresponds to the [Table Metadata API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/metadata). -The `column_metadata` field corresponds to the [Column Metadata API call](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/columns/-id-/metadata). -In both cases, the `key` and `value` are passed directly to the API; the `provider` value is -filled by the Id of the running component (e.g., `keboola.ex-db-snowflake`). - -{% highlight json %} -{ - ..., - "metadata": [ - { - "key": "an.arbitrary.key", - "value": "Some value" - }, - { - "key": "another.arbitrary.key", - "value": "A different value" - } - ], - "column_metadata": { - "column1": [ - { - "key": "yet.another.key", - "value": "Some other value" - } - ] - } -} -{% endhighlight %} diff --git a/extend/common-interface/oauth.md b/extend/common-interface/oauth.md index 4d299104..25c41e14 100644 --- a/extend/common-interface/oauth.md +++ b/extend/common-interface/oauth.md @@ -1,95 +1,5 @@ ---- -title: OAuth Interface Specification -permalink: /extend/common-interface/oauth/ ---- - -* TOC -{:toc} - -[OAuth Broker API](https://api.keboola.com/?service=oauth) integration provides a safe way to retrieve stored authorizations. - -When you are building a component that communicates with a 3rd party API and that API authorizes using OAuth, -Keboola stores the users' credentials/access tokens in the OAuth Broker API. They are revealed and -decrypted only for a target component and project. End-users can be assured that their authorized access will not leak. - -*Note: This feature must be enabled by our [support](mailto:support@keboola.com).* - -## Initialize -Create a configuration for the given component and project in the OAuth Broker API. -The `OAUTH_API_ID` is the id provided when storing authorization via the OAuth Broker API. -Set `"version": 3` to use the latest OAuth Broker API. The old OAuth V2 API is deprecated but still usable. - -{% highlight json %} -{ - - "storage": { ... }, - "parameters": { ... }, - "authorization": { - "oauth_api": { - "id": "{OAUTH_API_ID}" - "version": 3 - } - } -} -{% endhighlight %} - -## Authorize -[Job Queue](/extend/job-queue/) then retrieves, decrypts and injects the credentials to the -configuration file in the `authorization.oauth_api.credentials` attribute. - -{% highlight json %} -{ - "storage": { ... }, - "parameters": { ... }, - "authorization": { - "oauth_api": { - "id": "{OAUTH_API_ID}", - "version": 3, - "credentials": { - "id": "main", - "authorizedFor": "Myself", - "creator": { - "id": "1234", - "description": "me@keboola.com" - }, - "created": "2016-01-31 00:13:30", - "oauthVersion": "2.0", - "appKey": "w51u7j30oghe412", - "#data": "KBC::Encrypted==ENCODEDSTRING==", - "#appSecret": "KBC::Encrypted==ENCODEDSTRING==" - } - } - } -} -{% endhighlight %} - -The `authorization.oauth_api.credentials.#data` configuration node stores the response from -the authorized API as a raw string. Parse the string accordingly, as OAuth Broker API has intentionally -no knowledge about the authorized APIs. - -**Important:** None of the [sandbox API calls](/extend/component/running/) -decrypt the `authorization.oauth_api.credentials.#data` and `authorization.oauth_api.credentials.#appSecret` keys. - -## Credentials Injection - -If you want to bypass the OAuth Broker API integration, you can paste all required credential parameters in the configuration directly. -Fields requiring encryption will be encrypted and decrypted as usual. That means that you can save the following configuration -via the [configuration API](/integrate/storage/api/configurations/). - -{% highlight json %} -{ - "storage": { ... }, - "parameters": { ... }, - "authorization": { - "oauth_api": { - "credentials": { - "#data": "{\"oauth_token\":\"xx\",\"oauth_token_secret\":\"xxx\",\"x_auth_expires\":\"0\"}", - "appKey": "...", - "#appSecret": "..." - } - } - } -} -{% endhighlight %} - -This comes in very handy for quick component iterations and for testing whether your component works before having the OAuth support enabled. +--- +title: OAuth Interface Specification +permalink: /extend/common-interface/oauth/ +redirect_to: https://help.keboola.com/extend/common-interface/oauth/ +--- diff --git a/extend/common-interface/sandbox-output.png b/extend/common-interface/sandbox-output.png deleted file mode 100644 index babc31ec..00000000 Binary files a/extend/common-interface/sandbox-output.png and /dev/null differ diff --git a/extend/component/code-patterns/index.md b/extend/component/code-patterns/index.md index d084cb2c..1951e4b1 100644 --- a/extend/component/code-patterns/index.md +++ b/extend/component/code-patterns/index.md @@ -1,21 +1,5 @@ ---- -title: Code Patterns -permalink: /extend/component/code-patterns/ ---- - -* TOC -{:toc} - -Code Patterns are a special type of [component](/extend/component/). They - -- **generate code** for [transformations](https://help.keboola.com/transformations/#new-transformations), -- implement the [generate action](/extend/component/code-patterns/interface#generate-action), and -- use the [configuration format](/extend/component/code-patterns/interface#configuration-format). - -The generated code is written in a specific [output format](/extend/component/code-patterns/interface#output-format). - -## Next Steps - -- To see code patterns from users' perspective, go to [Code Patterns Help](https://help.keboola.com/transformations/code-patterns/). -- To learn how code patterns work internally, go to [Interface](/extend/component/code-patterns/interface). -- To implement your first code pattern, see this [tutorial](/extend/component/code-patterns/tutorial). +--- +title: Code Patterns +permalink: /extend/component/code-patterns/ +redirect_to: https://help.keboola.com/extend/component/code-patterns/ +--- diff --git a/extend/component/code-patterns/interface-1-add-component.png b/extend/component/code-patterns/interface-1-add-component.png deleted file mode 100644 index 878105b8..00000000 Binary files a/extend/component/code-patterns/interface-1-add-component.png and /dev/null differ diff --git a/extend/component/code-patterns/interface-2-schema.png b/extend/component/code-patterns/interface-2-schema.png deleted file mode 100644 index 62b8207d..00000000 Binary files a/extend/component/code-patterns/interface-2-schema.png and /dev/null differ diff --git a/extend/component/code-patterns/interface-3-supported-list.png b/extend/component/code-patterns/interface-3-supported-list.png deleted file mode 100644 index 13a19b3e..00000000 Binary files a/extend/component/code-patterns/interface-3-supported-list.png and /dev/null differ diff --git a/extend/component/code-patterns/interface-4-new-transformation.png b/extend/component/code-patterns/interface-4-new-transformation.png deleted file mode 100644 index 2b0c149f..00000000 Binary files a/extend/component/code-patterns/interface-4-new-transformation.png and /dev/null differ diff --git a/extend/component/code-patterns/interface-5-edit-component.png b/extend/component/code-patterns/interface-5-edit-component.png deleted file mode 100644 index 5ba5638b..00000000 Binary files a/extend/component/code-patterns/interface-5-edit-component.png and /dev/null differ diff --git a/extend/component/code-patterns/interface.md b/extend/component/code-patterns/interface.md index 22bf16ee..20e124ac 100644 --- a/extend/component/code-patterns/interface.md +++ b/extend/component/code-patterns/interface.md @@ -1,190 +1,5 @@ ---- -title: Code Pattern Interface -permalink: /extend/component/code-patterns/interface/ ---- - -* TOC -{:toc} - -This page describes how code patterns work internally as part of Keboola. - -## Common Interface -Code pattern is a special type of [component](/extend/component/), therefore the common interface applies to it. -To integrate your own components into Keboola, use the following links: - -- [Component common interface](/extend/common-interface/) -- [Environment](/extend/common-interface/environment/) -- [Implementation notes](/extend/component/implementation/) - -It's important to know that - -- the component code is wrapped in a Docker image. -- each component gets a [configuration file](#configuration). -- the correct exit code must be used; read about [return values](/extend/common-interface/environment/#return-values) and - [how to handle user and application errors](/extend/common-interface/actions/#handling-user-and-application-errors). -- the [Storage API token](https://help.keboola.com/management/project/tokens/) can be forwarded to the `KBC_TOKEN` environment variable. - - For example, if you need to know the details about the table in the input mapping. - - It must be enabled and approved by us. - - Read more in [Environment](/extend/common-interface/environment/). - -## Code Generation Process -This section shows how the code generation process works from start to end: - -- First, there must be a [published](/extend/publish/) code pattern component, for example, `keboola.example-pattern`. -- The component must have [supported transformations](#supported-components) configured. - - For instance, it supports `keboola.snowflake-transformation`. -- [Create a transformation with the code pattern](https://help.keboola.com/transformations/code-patterns/#new-transformation-with-code-pattern) in the user interface. -- Click the **Generate Code** button. -- User interface calls the [generate action](#generate-action) on the `keboola.example-pattern` component. -- The action finishes with the correct exit code: - - If **successful**: `exit code = 0` - - The component `stdout` contains JSON in the [output format](#output-format). - - The code blocks in the [parameters](#output-format) are stored to the transformation. - - If **failed**: `exit code = 1 or 2` - - The error is processed according to the [exit code](/extend/common-interface/environment/#return-values). - - The previous version of the generated code remains in the transformation. -- The generated code is displayed read-only in the user interface. - -## Generate Action -There are two types of component actions: - -- [Asynchronous, background](/integrate/jobs/) **run** actions -- [Synchronous actions](/extend/common-interface/actions/) with limited execution time - -Code patterns do not implement the **run** action. They only implement the **generate** [synchronous action](/extend/common-interface/actions/). - -The expected behavior of the **generate** action: - -- The action is started by the [Run Component Action](https://api.keboola.com/?service=sync-actions#post-/actions) API call. -- The `CMD` process defined in the `Dockerfile` is started in the container. -- The component generates a transformation code based on the [configuration](#configuration). -- The result is written in the [output format](#output-format) to `stdout`. -- The process will end successfully with `exit code = 0` (or with another [return value](/extend/common-interface/environment/#return-values) if an error occurs). -- API returns the result of the action. -- The user interface modifies the transformation's configuration and saves it. - -### Configuration -The [configuration file](/extend/common-interface/config-file/) `config.json` in the `KBC_DATADIR` contains: - -- **`action`** key set to the `generate` value as a name of the [action](/extend/common-interface/actions/) to execute -- **`storage`** key – contains the current input and output mapping from the transformation. - - Go to [Configuration File - Tables](/extend/common-interface/config-file/#tables) for a schema description and examples. - - Go to [Overview - Input and Output Mapping](https://help.keboola.com/transformations/code-patterns/#input-and-output-mapping) for an exemplary user interface. -- **`parameters`** key – modifies the generated code. - - **`_componentId`** key contains the ID of the target transformation component. - - For example, `keboola.snowflake-transformation` - - Based on this, it is possible to customize the generated code, e.g., for various SQL dialects. - - The other keys come from the [parameters form](https://help.keboola.com/transformations/code-patterns/#parameters-form), filled in by the user. - - The schema of the form is defined in the [configuration schema](#configuration-schema). - - The values should be [validated](/extend/common-interface/config-file/#validation) in the component's code. - -**Note**: [Learn more](/extend/common-interface/environment/) about the `KBC_DATADIR` environment variable. - -An example configuration (examples of the `storage` key can be found [here](/extend/common-interface/config-file/#tables)): - -```json -{ - "action": "generate", - "storage": { - "input": { - "tables": ["..."] - }, - "output": { - "tables": ["..."] - } - }, - "parameters": { - "_componentId": "keboola.snowflake-transformation", - "form_parameter_1": "value 1", - "form_parameter_2": "value 2" - } -} -``` - -### Output Format -The component must write the generated code to `stdout` in the following JSON format: - -- **`storage`** key contains the new transformation's input and output mapping. - - It is optional. If absent, the mapping remains unchanged. - - It is copied into the transformation's configuration `storage` key. - - A schema and examples can be found in [Configuration File - Tables](/extend/common-interface/config-file/#tables). -- **`parameters`** key with the generated code - - It is copied into the transformation's configuration `parameters` key. - - [Schema](https://help.keboola.com/transformations/#writing-scripts) `blocks` -> `codes` -> `script` must be used. See below. - - Each statement must be a separate item in the `script` array. - -An example configuration (examples of the `storage` key can be found [here](/extend/common-interface/config-file/#tables)): - -```json -{ - "storage": { - "input": { - "tables": ["..."] - }, - "output": { - "tables": ["..."] - } - }, - "parameters": { - "blocks": [ - { - "name": "Generated block", - "codes": [ - { - "name": "Generated code", - "script": [ - "CREATE TABLE table1;", - "SELECT foo1, foo2 FROM table2 INTO bar;" - ] - } - ] - } - ] - } -} -``` - -## Developer Portal -Each newly created component must be registered in the [Keboola Developer Portal](https://components.keboola.com/). - -Start with creating a simple [“Hello, World!”](/extend/component/tutorial/) component. To create -a code pattern component, you must take the following **additional steps**: - -First, create a component with the `Code Pattern` type. - -{: .image-popup} -![Screenshot -- Add component](/extend/component/code-patterns/interface-1-add-component.png) - -Open the component edit page, and modify the settings described in the following sections. - -{: .image-popup} -![Screenshot -- Edit component page](/extend/component/code-patterns/interface-5-edit-component.png) - -### Configuration Schema -- [Parameters form](https://help.keboola.com/transformations/code-patterns/#parameters-form) in the user interface -is generated from the [configuration schema](/extend/component/ui-options/configuration-schema/). -- Click the **Preview** button to see the preview of the form. - -{: .image-popup} -![Screenshot -- Configuration schema](/extend/component/code-patterns/interface-2-schema.png) - -### Supported Components -Each code pattern can generate a code for one or more transformation component types. -They are specified in the [configuration schema](/extend/component/ui-options/configuration-schema/) in -the root-level `supported_components` key, as an array of component IDs. - -{: .image-popup} -![Screenshot -- List of the supported components](/extend/component/code-patterns/interface-3-supported-list.png) - -When creating one of the listed transformation components, the [published](/extend/publish/) code pattern will be -available in the select box. - -{: .image-popup} -![Screenshot -- Create a new transformation](/extend/component/code-patterns/interface-4-new-transformation.png) - -The code pattern's [configuration](/extend/component/code-patterns/interface#configuration) contains -the `parameters._componentId` key, so it is possible to distinguish for which transformation component the code is generated. - -## Next Steps -- [Tutorial](/extend/component/code-patterns/tutorial) helps you to implement your first code pattern. -- [Code Patterns Help](https://help.keboola.com/transformations/code-patterns/) shows the code patterns from the user's point of view. \ No newline at end of file +--- +title: Code Pattern Interface +permalink: /extend/component/code-patterns/interface/ +redirect_to: https://help.keboola.com/extend/component/code-patterns/interface/ +--- diff --git a/extend/component/code-patterns/tutorial-1-add-component.png b/extend/component/code-patterns/tutorial-1-add-component.png deleted file mode 100644 index 878105b8..00000000 Binary files a/extend/component/code-patterns/tutorial-1-add-component.png and /dev/null differ diff --git a/extend/component/code-patterns/tutorial-2-project.png b/extend/component/code-patterns/tutorial-2-project.png deleted file mode 100644 index a31c3d1e..00000000 Binary files a/extend/component/code-patterns/tutorial-2-project.png and /dev/null differ diff --git a/extend/component/code-patterns/tutorial-3-modal.png b/extend/component/code-patterns/tutorial-3-modal.png deleted file mode 100644 index f4c1d364..00000000 Binary files a/extend/component/code-patterns/tutorial-3-modal.png and /dev/null differ diff --git a/extend/component/code-patterns/tutorial-4-new-transformation.png b/extend/component/code-patterns/tutorial-4-new-transformation.png deleted file mode 100644 index b9bca477..00000000 Binary files a/extend/component/code-patterns/tutorial-4-new-transformation.png and /dev/null differ diff --git a/extend/component/code-patterns/tutorial.md b/extend/component/code-patterns/tutorial.md index 6a8def19..de8d3a59 100644 --- a/extend/component/code-patterns/tutorial.md +++ b/extend/component/code-patterns/tutorial.md @@ -1,161 +1,5 @@ ---- -title: Code Pattern Quick Start -permalink: /extend/component/code-patterns/tutorial/ ---- - -* TOC -{:toc} - -Code patterns are a special type of [component](/extend/component/). -Their behavior is specified in their [interface](/extend/component/code-patterns/interface). - -The instructions below refer to the general [component quick start](/extend/component/tutorial/), -highlighting the steps specific to code patterns. - -## Creating Component -First, follow the common steps for [creating a component](/extend/component/tutorial/): - -- [Before you start](/extend/component/tutorial/#before-you-start) -- [Create a component](/extend/component/tutorial/#creating-component) -- [Create a deployment account](/extend/component/tutorial/#creating-deployment-account) - -Select `Code Pattern` as the type. - -{: .image-popup} -![Screenshot -- Add component](/extend/component/code-patterns/tutorial-1-add-component.png) - -Modify the settings described in [Interface - Developer Portal](/extend/component/code-patterns/interface#developer-portal). - -## Implementation -Go on following the [component quick start](/extend/component/tutorial/): - -- [Initialize a component](/extend/component/tutorial/#initializing-component) -- [Build a component](/extend/component/tutorial/#building-component) - -Learn how the whole [code generation process](/extend/component/code-patterns/interface#code-generation-process) works. - -**Implement the Generate action** as specified in [Interface - Generate Action](/extend/component/code-patterns/interface#generate-action): - -- First, load the [configuration](/extend/component/code-patterns/interface#configuration). -- Validate the configuration, `action = generate` is expected. -- Do not implement other actions (not even `run`). -- In case of an error, use the correct [return value](/extend/common-interface/environment/#return-values). -- Generate code based on the configuration. -- Write the result in the standard [output format](/extend/component/code-patterns/interface#output-format) to `stdout`. -- Exit with the `exit code = 0` if successful. - -## Running Component -Unlike other components, the code pattern component [**cannot be run**](/extend/component/tutorial/#running-component), -and will be **invisible in the user interface until it is [published](#publishing-component)**. - -However, there are two other ways to try it as a component, as described below. - -### Run via API -The first option is to call the [generate action](/extend/component/code-patterns/interface#generate-action) via the API: - -- You can test that the component returns the desired results based on the specified inputs. -- Use the [Run Component Action](https://api.keboola.com/?service=sync-actions#post-/actions) API call. -- An [API token](https://help.keboola.com/management/project/tokens/) is needed. - -### Modify Transformation via API -The second option is to modify the transformation to use an unpublished code pattern. - -#### Create empty transformation -First, click **Transformations** on the project menu. Then click **New Transformation** to create a new transformation. - -{: .image-popup} -![Screenshot -- Transformations page](/extend/component/code-patterns/tutorial-2-project.png) - -In the modal, click on the selected **type of the transformation**. - -{: .image-popup} -![Screenshot -- Add new transformation modal](/extend/component/code-patterns/tutorial-3-modal.png) - -Fill in the **name** and, optionally, the **description**. Do not select any code pattern. - -{: .image-popup} -![Screenshot -- Net transformation](/extend/component/code-patterns/tutorial-4-new-transformation.png) - -You have created an empty transformation. - -#### Set code pattern to transformation - -**Make note of the component and configuration ID from the URL.** You will need them in the API calls. - -``` -/admin/projects/{PROJECT_ID}/transformations-v2/{COMPONENT_ID}/{CONFIGURATION_ID} -``` - -**Set the code pattern to the transformation via [Storage API](/overview/api/).** - -Load the configuration in the JSON format via the [Configuration Detail](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-) API call. - -``` -curl \ - --include \ - --header "X-StorageApi-Token: {API_TOKEN}" \ -'{STORAGE API}/v2/storage/components/{COMPONENT_ID}/configs/{CONFIGURATION_ID}' -``` - -This is an example response, some keys are omitted. - -```json -{ - "id": "1234", - "name": "API test", - "configuration": {} -} -``` - -It is necessary to set the **ID of the code pattern component** to the configuration. - -```json -{ - "configuration": { - "runtime": { - "codePattern": { - "componentId": "keboola.example-pattern" - } - } - } -} -``` - -Update the configuration via the [Update Configuration](https://api.keboola.com/?service=storage#put-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-) API call. -JSON must be url-encoded. - -``` -curl - --include \ - --request PUT \ - --header "X-StorageApi-Token: {API_TOKEN}" \ - --header "Content-Type: application/x-www-form-urlencoded" \ - --data-binary "configuration=%7B%22runtime%22%3A%7B%22codePattern%22%3A%7B%22componentId%22%3A%22keboola.example-pattern%22%7D%7D%7D" \ -'{STORAGE API}/v2/storage/components/{COMPONENT_ID}/configs/{CONFIGURATION_ID}' -``` - -This is an example response, some keys are omitted. - -``` json -{ - "id": "1234", - "name": "API test", - "configuration": { - "runtime": { - "codePattern": { - "componentId": "keboola.example-pattern" - } - } - } -} -``` - -The transformation now uses the code pattern, and you can **test it in the [user interface](https://help.keboola.com/transformations/code-patterns/#configuration)**. - -## Publishing Component -Make sure the component is set up according to [Interface - Developer Portal](/extend/component/code-patterns/interface#developer-portal). -Then follow the tutorial [Publish Component](/extend/publish/). - -## Next Steps -- [Interface](/extend/component/code-patterns/interface) describes how the code patterns work internally. -- [Code Patterns Help](https://help.keboola.com/transformations/code-patterns/) shows the code patterns from the user's point of view. +--- +title: Code Pattern Quick Start +permalink: /extend/component/code-patterns/tutorial/ +redirect_to: https://help.keboola.com/extend/component/code-patterns/tutorial/ +--- diff --git a/extend/component/deployment/bitbucket-1.png b/extend/component/deployment/bitbucket-1.png deleted file mode 100644 index 4f98ff54..00000000 Binary files a/extend/component/deployment/bitbucket-1.png and /dev/null differ diff --git a/extend/component/deployment/bitbucket-2.png b/extend/component/deployment/bitbucket-2.png deleted file mode 100644 index 58b8eacd..00000000 Binary files a/extend/component/deployment/bitbucket-2.png and /dev/null differ diff --git a/extend/component/deployment/bitbucket-3.png b/extend/component/deployment/bitbucket-3.png deleted file mode 100644 index 5204470d..00000000 Binary files a/extend/component/deployment/bitbucket-3.png and /dev/null differ diff --git a/extend/component/deployment/configuration-sample.png b/extend/component/deployment/configuration-sample.png deleted file mode 100644 index 1d02e66e..00000000 Binary files a/extend/component/deployment/configuration-sample.png and /dev/null differ diff --git a/extend/component/deployment/deploy-config-1.png b/extend/component/deployment/deploy-config-1.png deleted file mode 100644 index be05fd9a..00000000 Binary files a/extend/component/deployment/deploy-config-1.png and /dev/null differ diff --git a/extend/component/deployment/deploy-config-2.png b/extend/component/deployment/deploy-config-2.png deleted file mode 100644 index 1812233e..00000000 Binary files a/extend/component/deployment/deploy-config-2.png and /dev/null differ diff --git a/extend/component/deployment/deploy-config-3.png b/extend/component/deployment/deploy-config-3.png deleted file mode 100644 index e4066e6a..00000000 Binary files a/extend/component/deployment/deploy-config-3.png and /dev/null differ diff --git a/extend/component/deployment/deploy-final.png b/extend/component/deployment/deploy-final.png deleted file mode 100644 index dc0c7813..00000000 Binary files a/extend/component/deployment/deploy-final.png and /dev/null differ diff --git a/extend/component/deployment/deploy-log-1.png b/extend/component/deployment/deploy-log-1.png deleted file mode 100644 index 464d9860..00000000 Binary files a/extend/component/deployment/deploy-log-1.png and /dev/null differ diff --git a/extend/component/deployment/deploy-log-2.png b/extend/component/deployment/deploy-log-2.png deleted file mode 100644 index b6ff7279..00000000 Binary files a/extend/component/deployment/deploy-log-2.png and /dev/null differ diff --git a/extend/component/deployment/gitlab-1.png b/extend/component/deployment/gitlab-1.png deleted file mode 100644 index a57e64e2..00000000 Binary files a/extend/component/deployment/gitlab-1.png and /dev/null differ diff --git a/extend/component/deployment/gitlab-2.png b/extend/component/deployment/gitlab-2.png deleted file mode 100644 index 9e17fa6c..00000000 Binary files a/extend/component/deployment/gitlab-2.png and /dev/null differ diff --git a/extend/component/deployment/index.md b/extend/component/deployment/index.md index 9128b125..5e1d3c83 100644 --- a/extend/component/deployment/index.md +++ b/extend/component/deployment/index.md @@ -4,378 +4,5 @@ permalink: /extend/component/deployment/ redirect_from: - /extend/docker/tutorial/automated-build/ - /extend/registration/deployment/ +redirect_to: https://help.keboola.com/extend/component/deployment/ --- - -* TOC -{:toc} - -If you created your component according to the [tutorial](/extend/component/tutorial/), you already have -a deployment pipeline set up. This article explains in more detail how the pipeline works. It also describes -alternative set ups. Assuming your component is similar to the [example component](https://github.com/keboola/ex-docs-tutorial) -created in the [tutorial](/extend/component/tutorial/), you see the following behavior: - -- Every commit & push to the git repository triggers a build. -- Every new tag pushes the built image into our [AWS ECR registry](https://aws.amazon.com/ecr/). -- Every [normal version tag](https://semver.org/#spec-item-2) (x.y.z) updates the image tag in the [Developer Portal](https://components.keboola.com/) and subsequently makes the image available in Keboola. - -We highly recommend the above setup (or a similar one) as it imposes very little extra work on the developer, yet -it deploys new versions of the component in a controlled and traceable manner. - -## How It Works -The following text explains the default setup in detail so that you know what to do if something -breaks or how to set up the pipeline manually. - -### Integration -The first step is the integration between GitHub and Travis. This is best set from the Travis side -by enabling the repository: - -{: .image-popup} -![Screenshot -- Add Repository](/extend/component/deployment/deploy-config-1.png) - -Enable builds for the repository. If you don't see the repository, use the **Sync account** button: - -{: .image-popup} -![Screenshot -- Enable Travis Repository](/extend/component/deployment/deploy-config-2.png) - -This causes Travis to trigger a build on every GitHub commit & push. - -### Build Setting -What the Travis build does is defined in the -[`.travis.yml`](https://github.com/keboola/component-generator/blob/master/templates-common/.travis.yml) file in -your repository. You should have something similar to this: - -{% highlight yaml %} -sudo: required -language: bash -services: - - docker - -before_script: - - export APP_IMAGE=keboola-component - - docker build . --tag=$APP_IMAGE - -after_success: - - docker images - -deploy: - provider: script - skip_cleanup: true - script: ./deploy.sh - on: - tags: true -{% endhighlight %} - -The `.travis.yml` file offers a vast number of [configuration options](https://docs.travis-ci.com/user/customizing-the-build/). -We only need a few of them though. The options `sudo`, `language` and `services` define that all we need is Docker. -The `before_script` section executes a single shell command which -[builds the image](/extend/component/tutorial/debugging/#step-2--build-the-image) and tags it `keboola-component`. The -tag is completely arbitrary at this moment, but we'll need it later. The `after_success` section simply lists the -built images in the log. - -The `deploy` section defines when a deploy will be triggered (`on tags`) and what should be done `deploy.sh`. This means that -when Travis encounters a tagged commit, it triggers the `deploy.sh` shell script (after everything else was done). - -### Deploy Script -In your repository, you should have a [deploy script](https://github.com/keboola/component-generator/blob/master/templates-common/deploy.sh) -similar to the one below: - -{% highlight bash %} -#!/bin/bash -set -e - -# Obtain the component repository and log in -docker pull quay.io/keboola/developer-portal-cli-v2:latest -export REPOSITORY=`docker run --rm \ - -e KBC_DEVELOPERPORTAL_USERNAME \ - -e KBC_DEVELOPERPORTAL_PASSWORD \ - quay.io/keboola/developer-portal-cli-v2:latest \ - ecr:get-repository ${KBC_DEVELOPERPORTAL_VENDOR} ${KBC_DEVELOPERPORTAL_APP}` -eval $(docker run --rm \ - -e KBC_DEVELOPERPORTAL_USERNAME \ - -e KBC_DEVELOPERPORTAL_PASSWORD \ - quay.io/keboola/developer-portal-cli-v2:latest \ - ecr:get-login ${KBC_DEVELOPERPORTAL_VENDOR} ${KBC_DEVELOPERPORTAL_APP}) - -# Push to the repository -docker tag ${APP_IMAGE}:latest ${REPOSITORY}:${TRAVIS_TAG} -docker tag ${APP_IMAGE}:latest ${REPOSITORY}:latest -docker push ${REPOSITORY}:${TRAVIS_TAG} -docker push ${REPOSITORY}:latest - -# Update the tag in Keboola Developer Portal -> Deploy to Keboola -if echo ${TRAVIS_TAG} | grep -c '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' -then - docker run --rm \ - -e KBC_DEVELOPERPORTAL_USERNAME \ - -e KBC_DEVELOPERPORTAL_PASSWORD \ - quay.io/keboola/developer-portal-cli-v2:latest \ - update-app-repository ${KBC_DEVELOPERPORTAL_VENDOR} ${KBC_DEVELOPERPORTAL_APP} ${TRAVIS_TAG} ecr ${REPOSITORY} -else - echo "Skipping deployment to KBC, tag ${TRAVIS_TAG} is not allowed." -fi -{% endhighlight %} - -The script uses our [Developer Portal CLI tool](https://github.com/keboola/developer-portal-cli-v2) to communicate with -the [Developer Portal API](https://api.keboola.com/?service=developer-portal). The tool itself is provided as a Docker -image `quay.io/keboola/developer-portal-cli-v2`. The entire script uses the following environment variables: - -- `KBC_DEVELOPERPORTAL_USERNAME` -- Service account user name -- `KBC_DEVELOPERPORTAL_PASSWORD` -- Service account password -- `KBC_DEVELOPERPORTAL_VENDOR` -- Vendor ID -- `KBC_DEVELOPERPORTAL_APP` -- Component ID - -You can read more about using the Developer Portal CLI in the [chapter about running components](/extend/component/running/#running-a-component). -The deploy script first pulls the image and then calls the `ecr:get-repository` command (while passing in the -`KBC_DEVELOPERPORTAL_USERNAME` and `KBC_DEVELOPERPORTAL_PASSWORD` variables). The result of that command is stored in the `REPOSITORY` -variable. After that the `ecr:get-login` command is called; it returns -a command line to authorize against our AWS ECR registry (e.g `docker login -u AWS -p ey...ODAzOH0= -147946154733.dkr.ecr.us-east-1.amazonaws.com`). The return value is `eval`ed -- i.e., the login command is executed. - -Then there are two `docker tag` and `docker push` commands which tag the image build as `keboola-component` with the `latest` tag -and the git commit tag (stored in `TRAVIS_TAG` variable). Push the two resulting images into the AWS ECR registry. - -The last part of the script begins with a check that the commit tag (`TRAVIS_TAG`) is a [normal version tag](https://semver.org/#spec-item-2) -(`x.y.z`). If not, the component is not updated in the Developer Portal. However, at this stage the image was already pushed into the registry -so it can be used by [running it explicitly](/extend/component/tutorial/debugging/#running-specific-tags). If the git tag -is a normal version tag, the component is updated in the Developer Portal using the `update-app-repository` command. -This means that the new version of the component is immediately deployed into Keboola. Keep in mind that it takes a couple of minutes -to propagate the change to all Keboola instances. - -When modifying the deploy script, make sure the `deploy.sh` file line ending is set to **Unix (LF)**. Also make sure that the file is executable, -i.e., by executing `git update-index --chmod=+x deploy.sh`. If the script is not executable, you'll get the following error message: - - Script failed with status 127 - -or - - bash: ./deploy.sh: Permission denied - - -### Deploy Configuration -The above deploy script requires four environment variables to be set. Set the following environment variables in the repository configuration: - - - `KBC_DEVELOPERPORTAL_APP` the component ID -- e.g.: `keboola-test.ex-docs-tutorial` - - `KBC_DEVELOPERPORTAL_PASSWORD` with the [**Service Account**](/extend/component/tutorial/#creating-a-deployment-account) password - - `KBC_DEVELOPERPORTAL_USERNAME` with the [**Service Account**](/extend/component/tutorial/#creating-a-deployment-account) login - - `KBC_DEVELOPERPORTAL_VENDOR` with the vendor of the component -- e.g.: `keboola-test` - -{: .image-popup} -![Screenshot -- Repository Configuration](/extend/component/deployment/deploy-config-3.png) - -### Trigger Build -Commit and push anything to the repository to trigger the build. In Travis, you should see an output similar to this: - -{: .image-popup} -![Screenshot -- Build Log](/extend/component/deployment/deploy-log-1.png) - -Now push a tag to the repository (we recommend using [Semantic Versioning](http://semver.org/)): - - git tag 0.0.6 - git push origin --tags - -In Travis, you should see an output similar to this: - -{: .image-popup} -![Screenshot -- Build and Deploy Log](/extend/component/deployment/deploy-log-2.png) - -If no errors occurred, the component is now deployed into Keboola. In the Developer Portal, you can verify that the -component repository and tag were automatically set: - -{: .image-popup} -![Screenshot -- Deploy Verification](/extend/component/deployment/deploy-final.png) - -The component is now runnable in Keboola. You can view all settings in our -[example repository](https://github.com/keboola/ex-docs-tutorial). You can also -review [Travis Configuration](https://travis-ci.org/keboola/ex-docs-tutorial/). - -*Note that it takes up to **5 minutes** before the changes in the Developer Portal propagate to all Keboola instances in all regions.* - -## Bitbucket Integration -The [development tutorial](/extend/component/tutorial/) as well as the above description assume you're using -Travis CI Service for building and deploying the image. Travis integrates very well with [GitHub](https://github.com/), but not with -[Bitbucket](https://bitbucket.org/). However, Bitbucket has its own continuous integration service -- -[Bitbucket Pipelines](https://bitbucket.org/product/features/pipelines). - -You have to enable Bitbucket Pipelines in your repository: - -{: .image-popup} -![Screenshot -- Bitbucket Pipelines](/extend/component/deployment/bitbucket-1.png) - -Note that only the owner of the repository can enable pipelines. Then set the environment variables in settings: - -{: .image-popup} -![Screenshot -- Bitbucket Environment Variables](/extend/component/deployment/bitbucket-2.png) - -Add the following [`bitbucket-pipelines.yml`](https://github.com/keboola/component-generator/blob/master/templates/bitbucket-deploy/bitbucket-pipelines.yml) file to your repository: - -{% highlight yaml %} -options: - docker: true - -pipelines: - default: - - step: - script: - - export APP_IMAGE=keboola-component - - docker build . --tag=$APP_IMAGE - - docker images - - tags: - '*': - - step: - script: - - export APP_IMAGE=keboola-component - - docker build . --tag=$APP_IMAGE - - docker images - - ./deploy.sh -{% endhighlight %} - -Also add the [`deploy.sh` script](https://github.com/keboola/component-generator/blob/master/templates/bitbucket-deploy/deploy.sh), -which is modified to use the [`BITBUCKET_TAG`](https://confluence.atlassian.com/bitbucket/environment-variables-794502608.html) variable (instead of `TRAVIS_TAG`). When done, commit and push; a build will automatically appear in the **Pipelines** section: - -{: .image-popup} -![Screenshot -- Bitbucket Build](/extend/component/deployment/bitbucket-3.png) - -With the above settings, the Bitbucket Pipelines will behave in exactly the same way as the Travis configuration described above. -You can also have a look at a [10 minute video](https://www.youtube.com/watch?v=Pf_hfM_zNyU) showing the Bitbucket setup on a new component. - -## GitLab Integration -The [development tutorial](/extend/component/tutorial/) as well as the above description assume you're using -Travis CI Service for building and deploying the image. Travis integrates very well with [GitHub](https://github.com/), but not with -[GitLab](https://about.gitlab.com/). However, GitLab has its own continuous integration service -- -[CI Pipelines](https://docs.gitlab.com/ee/ci/pipelines.html). - -You have to set the environment variables in settings: - -{: .image-popup} -![Screenshot -- GitLab Environment Variables](/extend/component/deployment/gitlab-1.png) - -Then add the following [`.gitlab-ci.yml`](https://github.com/keboola/component-generator/blob/master/templates/gitlab-deploy/.gitlab-ci.yml) file to your repository: - -{% highlight yaml %} -image: docker:latest - -variables: - DOCKER_DRIVER: overlay2 - APP_IMAGE: keboola-component - -services: -- docker:dind - -before_script: -- docker info - -build-component: - stage: build - script: - - docker build . --tag=$APP_IMAGE - -deploy-component: - stage: deploy - script: - - docker build . --tag=$APP_IMAGE - - pwd - - ls -la - - export - - ./deploy.sh - only: - - tags -{% endhighlight %} - -Also add the [`deploy.sh` script](https://github.com/keboola/component-generator/blob/master/templates/gitlab-deploy/deploy.sh), -which is modified to use the [`CI_COMMIT_TAG`](https://docs.gitlab.com/ce/ci/variables/README.html) (instead of `TRAVIS_TAG`) and use `sh` shell (instead of `bash`). When done, commit and push; a build will automatically appear in the **Pipelines** section: - -{: .image-popup} -![Screenshot -- GitLab Build](/extend/component/deployment/gitlab-2.png) - -With the above settings, the GitLab CI Pipelines will behave in exactly the same way as the Travis configuration described above. -You can also have a look at a [10 minute video](https://www.youtube.com/watch?v=TC-tN-zYgEw) showing the GitLab setup on a new component. - -## Manual Deployment -If you want to use another continuous integration setting or deploy to the repository manually, you can do so without limitations. -As in the [above script](/extend/component/deployment/#deploy-script), -we recommend using the [Developer Portal CLI client](https://github.com/keboola/developer-portal-cli-v2). This CLI tool (runnable in Docker or PHP) -allows you to obtain the repository for a component and push credentials to that repository. See the chapter about -[running components](/extend/component/running/#running-a-component), for example, how to obtain the AWS registry credentials. -If you want to get even more low level, you can use the [Developer Portal API](https://api.keboola.com/?service=developer-portal) directly. -It also allows you to [generate credentials for a service account](https://api.keboola.com/?service=developer-portal#post-/vendors/-vendor-/credentials) -programmatically. We use our AWS ECR registry for hosting all component images. - -## Test Live Configurations -Testing your component can be simply added as part of the script in `.travis.yml` file. See an example in -[Python](https://github.com/keboola/component-generator/blob/master/templates/python-tests/.travis.yml) or -[PHP](https://github.com/keboola/component-generator/blob/master/templates/php-component/.travis.yml). - -However, you may want to test the component on some 'real' configurations -in your project. You can do this by extending the build script and adding certain environment variables to -Travis with an appropriate [Storage token](https://help.keboola.com/storage/tokens/) -and configuration ID. It is highly recommended to create a dedicated token for this task. - -The commands will need two extra environment variables apart from the -[ones listed above](/extend/component/deployment/#deploy-configuration): - -- `KBC_STORAGE_TOKEN` --- the Storage token that the test(s) will run under -- `KBC_APP_TEST_CONFIG` --- the ID of the configuration to test - -{: .image-popup} -![Screenshot -- Sample Configurations](/extend/component/deployment/configuration-sample.png) - -If you are still using our [sample component code](https://github.com/keboola/ex-docs-tutorial), -create a configuration and set an arbitrary table on input. - -The following extended `.travis.yml` will do the trick: - -{% highlight yaml %} -sudo: false - -services: - - docker - -before_script: - - export APP_IMAGE=keboola-component - - docker build -t $APP_IMAGE . - - docker run $APP_IMAGE flake8 - - docker run $APP_IMAGE python -m unittest discover - # push test image to ECR - - docker pull quay.io/keboola/developer-portal-cli-v2:latest - - export REPOSITORY=`docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-repository $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP` - - docker tag $APP_IMAGE:latest $REPOSITORY:test - - eval $(docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-login $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP) - - docker push $REPOSITORY:test - -script: - - curl --location --request POST "https://queue.keboola.com/jobs" --header "X-StorageApi-Token:$KBC_STORAGE_TOKEN" --header "Content-Type:application/json" --data-raw '{"component":"'"$KBC_DEVELOPERPORTAL_APP"'","config":"'"$KBC_APP_TEST_CONFIG"'","tag":"test","mode":"run"}' - -after_success: - - docker images - -deploy: - provider: script - skip_cleanup: true - script: "./deploy.sh" - on: - tags: true -{% endhighlight %} - -The commands above - -- build the component image and tag it `keboola-component`. -- run the [flake8](http://flake8.pycqa.org/en/latest/) code style check. -- run [unittest](https://docs.python.org/3.6/library/unittest.html) tests. -- pull the [Developer Portal CLI client](https://github.com/keboola/developer-portal-cli-v2). -- get the component's Keboola registry from the Developer Portal and store it in the `REPOSITORY` variable. -- tag the image as `test`. -- get the command to login to the registry (`ecr:get-login`) and execute it (i.e., log in). -- push the image to the registry. -- run the specified test job on Keboola using the [Queue API](https://api.keboola.com/?service=job-queue#job-queue). The tag used is `test`. - -If you want to run multiple test jobs, simply repeat the command with the different configuration IDs -that you would like to test. - -When you commit to the component repository, the Docker image will be built, and using a `test` tag, it will be tested in production Keboola. -However, it will not be deployed to production! To get it into production, create a new normal version tag (`x.y.z`) in the repository. -The Docker image will be built and tested using the `test` tag, and if all succeeds, it will be deployed -with the `x.y.z` tag into Keboola --- a new version will be available in production. -You can see the [Python code](https://github.com/keboola/component-generator/tree/master/templates/python-tests) or -[PHP code](https://github.com/keboola/component-generator/tree/master/templates/php-component/) in our -[Templates repository](https://github.com/keboola/component-generator/tree/master/templates). diff --git a/extend/component/dynamic-mapping.png b/extend/component/dynamic-mapping.png deleted file mode 100644 index 189a865d..00000000 Binary files a/extend/component/dynamic-mapping.png and /dev/null differ diff --git a/extend/component/implementation/index.md b/extend/component/implementation/index.md index bf615b31..13c64fe1 100644 --- a/extend/component/implementation/index.md +++ b/extend/component/implementation/index.md @@ -1,140 +1,7 @@ ---- -title: Implementation Notes -permalink: /extend/component/implementation/ -redirect_from: - - /extend/docker/images/ ---- - -* TOC -{:toc} - -Here are some good practices in developing component code. They're best to be followed -across all components, especially if you want your component to be published. We also recommend -that you check our [component templates](https://github.com/keboola/component-generator). - -Developing a component is a challenging task. To maximize your efficiency, follow these basic rules: - -- **Do not repeat the functions of existing components.** For example, if you want to download data from Google Drive and transpose the -table, you do not have to create a component for that -- just load the table using the existing extractor and transpose it using -the existing R Transpose Table application. -- **Every component should do only one thing.** For example, if you want to extract data from an API and compute some metrics on them, -make two components -- one for extracting the data and a second one for computing the metrics. -- **Do as little data processing as possible.** As in the above example, having the processing tied to extraction makes it hard to -identify errors in data -- was it extracted incorrectly, or was it processed incorrectly? It also allows the end user to split the -task into smaller ones and have better control over their execution. -- **Avoid optional data modification.** For example, if you have a component which sometimes extracts data in ISO8892 encoding and sometimes -in UTF8 encoding, you do not need to implement this conversion in the component. You can let the end user configure -[processors](/extend/component/processors/) to load the incompatible data. -- **Avoid iterations.** For example, your component is downloading multiple files from a system and converts them to CSV files -for Storage import. You do not need to implement the loop around the files, you can use -[configuration rows](/integrate/storage/api/configurations/#configuration-rows) and implement processing of only a single table. - -Before you create any complex components, be sure to read about -[configurations](/integrate/storage/api/configurations/) and [processors](/extend/component/processors/) -as they can substantially simplify your component code. We also recommend that you use our -[common interface](/extend/common-interface/) library, which is available for -[Python](/extend/component/implementation/python/#using-the-kbc-package), -[R](/extend/component/implementation/r/#using-the-kbc-package), -and [PHP](/extend/component/implementation/php/#using-the-kbc-package). -You may use any Docker image you see fit. We recommend to base your images on those from an [official repository](https://hub.docker.com/search?q=&type=image) -because they are the most stable ones. - -## Memory -Keboola [components](/extend/component/) can be used to process substantial amounts of data (i.e., dozens of gigabytes), which are not -going to fit into memory. Every component should therefore be written so that it processes data in chunks of -a limited size (typically rows of a table). Many of the Keboola components run with less than 100MB memory. -While the Keboola platform is capable of running jobs with ~8GB of memory without problems, we are not particularly -happy to allow it, and we certainly do not want to allow components where the amount of used memory -depends on the size of the processed data. - -## Error Handling -Depending on the component [exit code](/extend/common-interface/environment/#return-values), the component job is marked as -successful or failed. - -- `exit code = 0` The job is considered **successful**. -- `exit code = 1` The job fails with a **user error**. -- `exit code > 1` The job fails with an **application error**. - -During a component execution, all the output sent to STDOUT is captured and sent live to job events. -The output to [STDERR](https://en.wikipedia.org/wiki/Standard_streams#Standard_error_.28stderr.29) is captured too, and -in case the job is successful or fails with a user error, it is displayed as the last event of the job. In case the -job ends with an application error, the entire contents of STDERR is hidden from the end user and sent only to -vendor internal logs. The end user will see only a canned response ('An application error occurred') with -the option to contact our support. - -This means that you do not have to worry about the internals of your component leaking to the end user provided that -the component exit code is correct. On the other hand, the user error is supposed to be solvable by the end user. When creating an error message, stick by the following rules: - -- Avoid **nonsense** messages. For example: 'Banana Error: Exceeding trifling witling' or only numeric errors. -- Avoid **errors users cannot solve**. For example: 'An outdated OpenSSL library, update to OpenSSL 1.0.2'. -- Provide **guidance** on what the user should do. For example: 'The input table is missing; make sure the output mapping destination is set to `items.csv`'. -- Avoid deliberate **leaking** sensitive information. For example: credentials, tokens. The output of each component is [filtered](/extend/common-interface/logging/#standard-output-and-standard-error) to prevent *accidental* leaks of sensitive information. That means you don't need to implement filtering for example for exception messages. - -Also keep in mind that the output of the components (job events) serve to pass only informational and error messages; **no data** can be passed through. -The event message size is limited (about 64KB). If the limit is exceeded, the message will be trimmed. If the component produces -obscene amount (dozens of MBs) of output in a very short time, it may be terminated with an internal error. -Also make sure your component does not use any [output buffering](#language-specific-notes), otherwise all events will be cached after the application finishes. - -## Implementing Processors -[Processors](/extend/component/processors/) -allow the end user to customize the input to the component and the output from it. That means -that many custom requirements can be solved by processors, keeping the component -code general. - -Choosing whether to implement a specific feature as a processor or as part of your -component may be difficult. A processor might be a good solution if the following are true: - -- The feature is **optional** (not all end users are interested in it). -- The feature is **simple** (one operation, contains no internal logic). -- The feature is **universal** (it is always applied to all input/output or none). - -The first condition is especially important. Another way to read it is that a processor must never supply a function expected from the component. -In other words: **Each component should be able to consume/generate a valid input/output without any processors.** For example, if an extractor can -produce tables without any further processing, good, let it be tables, but if can not, it should output only files and processors should do the rest. -If processors are used together with [configuration rows](/integrate/storage/api/configurations/#configuration-rows), -the last condition is weakened, because a different set of processors may be applied to each configuration row. - -### Design -Implementing a processor is in principle the same as implementing any other -[component](/extend/component/). However, processors are designed to be -[single responsibility](https://en.wikipedia.org/wiki/Single_responsibility_principle) components. This -means, for example, that processors should require no or very little configuration, should not communicate -over a network and should be fast. - -Processors take data from the `in` [data folders](/extend/common-interface/folders/) and -store it in the `out` [data folders](/extend/common-interface/folders/) as any other components. Keep in mind, however, -that any files not copied to the `out` folders will be ignored (i.e., lost). That means if a processor is supposed to -"not touch" something, it actually has to copy that something to the `out` folder. - -The processors should be aware of [manifest files](/extend/common-interface/manifest-files/). This means that -the processor: - -- Must exclude manifests from processing (they are not data files). -- If the processor changes something stored in the manifest, it must process it (read the manifests in `in` folder, modify and store it in the `out` folder). Typical example is modification of table columns which must be reflected in the manifest. -- If the processor is doing change unrelated to manifest, it should copy the manifest from `in` to `out`. -- If the processor is not doing a 1:1 operation (e.g merges multiple tables into one), it should not do anything about the manifest, which means that it will be discarded. - -Keep in mind that processors can be [chained](/extend/component/processors/#chaining-processors); -you can, for example, rely on - -- the table CSV files being in [standard format](https://help.keboola.com/storage/tables/csv-files/#output-csv-format). -- table manifest always present. -- the CSV file being orthogonal. - -If the above conditions are not met, then another processor should be added before yours. I.e. you should keep the -processor simple and delegate the assumptions to other processors (and [**document** them](#publishing-a-processor)). If possible the -processor should also assume that the CSV files are headless and stored in arbitrary sub-folders. When implemented with this assumption -the processor will support [sliced tables](/extend/common-interface/folders/#sliced-tables). - -### Publishing Processor -The process of processor registration is the same as [publishing any other component](/extend/publish/). -However, many of the fields do not apply, because processors have no UI. -The following fields are important: - -- Vendor -- Component name and component type (`processor`) -- Short and full description -- Component documentation (`documentationUrl`) - - must be public - - must state whether the processor is capable of working with [sliced tables](/extend/common-interface/folders/#sliced-tables) - - whether it requires/processes manifests +--- +title: Implementation Notes +permalink: /extend/component/implementation/ +redirect_from: + - /extend/docker/images/ +redirect_to: https://help.keboola.com/extend/component/implementation/ +--- diff --git a/extend/component/implementation/php.md b/extend/component/implementation/php.md index 88ee6c41..8c3faf1b 100644 --- a/extend/component/implementation/php.md +++ b/extend/component/implementation/php.md @@ -3,274 +3,5 @@ title: PHP Implementation Notes permalink: /extend/component/implementation/php/ redirect_from: - /extend/custom-science/php/ +redirect_to: https://help.keboola.com/extend/component/implementation/php/ --- - -* TOC -{:toc} - -## Docker -Use the [official images](https://hub.docker.com/_/php/) if possible. Usually, the `alpine` versions are sufficient and are the -smallest and fastest. If you need Composer, use its [official image](https://hub.docker.com/_/composer/) or -[our templates](https://github.com/keboola/component-generator/blob/master/templates/). - -## Working with CSV Files -We recommend using our [CSV library](https://github.com/keboola/php-csv), which provides a convenience wrapper -around the build-in [CSV functions](https://www.php.net/manual/en/function.fgetcsv.php). However, the functions work well on their own too. -If you are using bare PHP functions, the following code illustrates their use: - -{% highlight php %} -writeRow(['number', 'someText', 'double_number']); - -// read input file and write rows of output file -$inFile = new Keboola\Csv\CsvFile($dataDir . 'in' . DIRECTORY_SEPARATOR . 'tables' . DIRECTORY_SEPARATOR . 'source.csv'); -foreach ($inFile as $rowNum => $row) { - if ($rowNum == 0) { - // skip header - continue; - } - $outFile->writeRow([ - $row[0], - $row[1], - $row[0] * $multiplier - ]); -} -{% endhighlight %} - -## Using Keboola Package -Keboola's [PHP component package](https://github.com/keboola/php-component) provides functions to - -- read and parse the configuration file and parameters: - `getConfig` method or `getConfig()->getParameters()` methods. -- list input files and tables: `getConfig()->getInputFiles()`, `getConfig()->getInputTables()` methods. -- work with manifests containing table and file metadata: `getManifestManager()->getTableManifest()`, `getManifestManager()->writeTableManifest()`, `getManifestManager()->getFileManifest()`, `getManifestManager()->writeFileManifest()` methods. -- list expected outputs: `getConfig()->getExpectedOutputFiles()` or `getConfig()->getExpectedOutputTables()` methods. - -You can go through the [generated docs](https://keboola.github.io/php-component/master/classes.html) of all available methods and classes. -The package can be installed by [Composer](https://getcomposer.org/): - - composer require keboola/php-component - -The package can be used standalone (good for existing code), or you can inherit your own component from it (good for new components). -When inheriting from the package, see the [GitHub repository](https://github.com/keboola/php-component) for examples, or -our [component template](https://github.com/keboola/component-generator/tree/master/templates). -Using the package as a standalone class does not require anything else than creating its instance: - -{% highlight php %} -getConfig()->getValue(['parameters']); -var_export($parameters); - -$inputTables = $component->getConfig()->getInputTables(); -var_export($inputTables); -{% endhighlight %} - -The configuration is read from the [data folder](/extend/common-interface/config-file/) specified by the -[KBC_DATADIR environment variable](https://developers.keboola.com/extend/common-interface/environment/). -Given the following `config.json` file: - -{% highlight json %} -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.sample", - "destination": "source.csv" - } - ], - "files": [] - } - }, - "parameters": { - "myParameter": "myValue", - "repeat": 2 - } -} -{% endhighlight %} - -The above PHP code would output: - -{% highlight php %} -array ( - 'myParameter' => 'myValue', - 'repeat' => 2, -)array ( - 0 => - array ( - 'source' => 'in.c-main.sample', - 'destination' => 'source.csv', - ), -) -{% endhighlight %} - -### Dynamic Input/Output Mapping -In the [tutorial](/extend/component/tutorial/) and the above examples, we show -applications which have names of their input/output tables hard-coded. -The following example shows how to read an input and output mapping specified by the end user, -which is accessible in the [configuration file](/extend/common-interface/config-file/). It demonstrates -how to read and write tables and table manifests. File manifests are handled the same way. For a full authoritative list -of items returned in table list and manifest contents, see [the specification](/extend/common-interface/config-file/). - -Note that the `destination` label in the script refers to the destination from the -[mapper](/extend/component/tutorial/input-mapping/) perspective. -The input mapper takes `source` tables from the user's storage and produces `destination` tables that become -the input of your component. The output tables of your component are consumed by the output mapper -whose `destination` are the resulting tables in Storage. - -The following piece of code reads an arbitrary number of tables and adds an auto-generated primary key -to them. The name of the added column is configured in parameters (`primaryKeyName`). Also, the -step of the generator is configured in parameters (`primaryKeyStep`). The end of the code writes -a table [manifest file](/extend/common-interface/manifest-files/) which stores the configuration of -the primary key and optional table metadata. - -{% highlight php %} -getConfig()->getInputTables(); - -$j = 0; -foreach ($inputTables as $inputTable) { - // get csv file name - $inFileName = $component->getDataDir() . '/in/tables/' . $inputTable['destination']; - - // get file name from output mapping - $outFileName = $component->getDataDir() . '/out/tables/' . - $component->getConfig()->getExpectedOutputTables()[$j]['destination']; - - // read table manifest - $manifest = $component->getManifestManager()->getTableManifest($inFileName); - - // open input and output files - $inFile = new \Keboola\Csv\CsvFile($inFileName); - $outFile = new \Keboola\Csv\CsvFile($outFileName); - // get value of `primaryKeyName` parameter - $columnName = $component->getConfig()->getParameters()['primaryKeyName']; - - // process table data - $header = $inFile->getHeader(); - array_push($header, $columnName); - $outFile->writeRow($header); - $i = 0; - foreach ($inFile as $row) { - // skip the first line with header - if ($i != 0) { - // add generated primary key - array_push($row, $i); - $outFile->writeRow($row); - } - // get value of `primaryKeyStep` parameter (different approach to `primaryKeyName` above) - $i = $i + $component->getConfig()->getValue(['parameters', 'primaryKeyStep']); - } - - // store table metadata - $metadata = $manifest['metadata']; - array_push($metadata, [["key" => "sample", "value" => "metadata"]]); - - // create table manifest with primary ket and metadata - $component->getManifestManager()->writeTableManifestFromArray( - $outFileName, - [ - 'primary_key' => [$columnName], - 'metadata' => $metadata, - ] - ); - $j++; -} -{% endhighlight %} - -## Logging -For simple applications, printing with `echo` or `print` is enough. To print to STDERR, you have to use -e.g., `fwrite(STDERR, "Hello, world!" . PHP_EOL);`. The best option is to use the [Monolog package](https://github.com/Seldaek/monolog). -The following is a useful initialization: - -{% highlight php %} -$formatter = new LineFormatter("%message%\n"); -$errHandler = new StreamHandler('php://stderr', Logger::NOTICE, false); -$errHandler->setFormatter($formatter); -$handler = new StreamHandler('php://stdout', Logger::INFO); -$handler->setFormatter($formatter); -$logger = new Logger('main', [$errHandler, $handler]); -{% endhighlight %} - -This means that a log with the [NOTICE level](https://github.com/Seldaek/monolog/blob/master/doc/01-usage.md#log-levels) and above -will go to STDERR, and the INFO level will go to STDOUT. The formatter removes unnecessary fields like `timestamp` and `context`. - -## Error Handling -The following [piece of code](https://github.com/keboola/component-generator/blob/master/templates/php-component/src/run.php) is a good entry point: - -{% highlight php %} -$logger = new Logger(); -try { - $app = new Component($logger); - $app->execute(); - exit(0); -} catch (UserException $e) { - $logger->error($e->getMessage()); - exit(1); -} catch (\Throwable $e) { - $logger->critical( - get_class($e) . ':' . $e->getMessage(), - [ - 'errFile' => $e->getFile(), - 'errLine' => $e->getLine(), - 'errCode' => $e->getCode(), - 'errTrace' => $e->getTraceAsString(), - 'errPrevious' => $e->getPrevious() ? get_class($e->getPrevious()) : '', - ] - ); - exit(2); -} -{% endhighlight %} - -In this case, we consider everything derived from `UserException` to be an error which should be shown to the end user. -You have to create that exception class in your component. Every other error will lead to a generic message; only -the developer will see the details, and the code will follow the [general error handling rules](#error-handling). -Here we use the [`Throwable`](https://www.php.net/manual/en/class.throwable.php) ancestor, which also catches PHP errors. You can, of -course, modify this logic to your liking. diff --git a/extend/component/implementation/python.md b/extend/component/implementation/python.md index 65d339c0..74ffbc91 100644 --- a/extend/component/implementation/python.md +++ b/extend/component/implementation/python.md @@ -3,424 +3,5 @@ title: Python Implementation Notes permalink: /extend/component/implementation/python/ redirect_from: - /extend/custom-science/python/ +redirect_to: https://help.keboola.com/extend/component/implementation/python/ --- - -* TOC -{:toc} - -## Docker -Use the [official images](https://hub.docker.com/_/python/) if possible. Usually, the `alpine` versions are sufficient and are the -smallest and fastest. We recommend using [our templates](https://github.com/keboola/component-generator/tree/master/templates). - -## Working with CSV Files -We advise you to follow the guidelines for the [Python transformation](https://help.keboola.com/transformations/python/#development-tutorial). - -The build-in CSV functions for Python work well except when the data in the CSV file contain a null character. This is -[usually fixed](https://stackoverflow.com/questions/4166070/python-csv-error-line-contains-null-byte) by -adding `lazy_lines = (line.replace('\0', '') for line in in_file)`. The expression -is a [generator](https://wiki.python.org/moin/Generators) which makes sure that -[null characters](https://en.wikipedia.org/wiki/Null_character) are properly handled. -It is also important to use `encoding='utf-8'` when reading and writing files. - -{% highlight python %} -import csv - -csvlt = '\n' -csvdel = ',' -csvquo = '"' -with open('in/tables/source.csv', mode='rt', encoding='utf-8') as in_file, open('out/tables/destination.csv', mode='wt', encoding='utf-8') as out_file: - writer = csv.DictWriter(out_file, fieldnames=['col1', 'col2'], lineterminator=csvlt, delimiter=csvdel, quotechar=csvquo) - writer.writeheader() - - lazy_lines = (line.replace('\0', '') for line in in_file) - reader = csv.DictReader(lazy_lines, lineterminator=csvlt, delimiter=csvdel, quotechar=csvquo) - for row in reader: - # do something and write row - - writer.writerow({'col1': row['first'] + 'ping', 'col2': int(row['second']) * 42}) -{% endhighlight %} - -Note that we open both the input and output files simultaneously; as soon as a row is processed, -it is immediately written to the output file. This approach keeps only a single row of data in the memory and is -generally very efficient. It is recommended to implement the processing in this way because data files -coming from Keboola can be quite large (i.e., dozens of gigabytes). - -## Using Keboola Python Package -The [Python component package](https://github.com/keboola/python-component) provides a Python wrapper over the -[Keboola Common Interface](https://developers.keboola.com/extend/common-interface/). It simplifies all tasks related - to the communication of the [component](https://developers.keboola.com/extend/component/) with - Keboola that is defined by the Common Interface. Such tasks are config manipulation, validation, - component state, I/O handling, I/O metadata and manifest files, logging, etc. - - **NOTE:** That this package is a replacement for the previous legacy [Python docker application](https://github.com/keboola/python-docker-application) - -The `CommonInterface` class provides following methods: - -- read and parse the configuration file and parameters: `configuration` object and `configuration.parameters` properties. -- list input files and tables represented by Python objects for easier manipulation: -- work with [manifests](/extend/common-interface/manifest-files/) containing table and file metadata: `get_table_manifest()`, `get_file_manifest()`, `write_table_manifest()`, `write_file_manifest()` methods. -- list expected outputs: `configuration.files_input_mapping`, `configuration.tables_input_mapping` properties. - -The library is a standard Python package that is available by default in the production environment. -It is a public PYPI project [keboola.component](https://pypi.org/project/keboola.component/), so it can be installed -locally with `pip3 install keboola.component`. - -A generated [API documentation](https://htmlpreview.github.io/?https://raw.githubusercontent.com/keboola/python-component/main/docs/api-html/component/interface.html) -is available for the package, and an actual working example can be found in our -[Python template](https://bitbucket.org/kds_consulting_team/kbc-python-template/src/master/src/component.py). - -### Initialization - -The core class is `keboola.component.interface.CommonInterface`, upon its initialization the environment is -created. e.g. - -- data folder initialized (either from the [Environment Variable](/extend/common-interface/environment/#environment-variables) or manually) -- [Configuration file](/extend/common-interface/config-file/) is loaded -- All Environment variables are loaded - -The optional parameter `data_folder_path` of the constructor is the path to the data directory. -If not provided, [`KBC_DATADIR` environment variable](/extend/common-interface/environment/#environment-variables) will be used. - -The class can be either extended or just instantiated and manipulated like object. -The `CommonInterface` class is exposed in the `keboola.component` namespace: - -```python -from keboola.component import CommonInterface -# init the interface -# A ValueError error is raised if the KBC_DATADIR does not exist or contains non-existent path. -ci = CommonInterface() -``` - -### Loading configuration parameters - -The below example loads initializes the common interface class and automatically loading config.json from the -[data folder](https://developers.keboola.com/extend/common-interface/folders/) - - **NOTE:** The `configuration` object is initialized upon access and a ValueError is thrown if the `config.json` does not exist - in the data folder. E.g., `cfg = ci.configuration` may throw a ValueError even though the data folder exists and ci (CommonInterface) - is properly initialized. - -```python -from keboola.component import CommonInterface -# Logger is automatically set up based on the component setup (GELF or STDOUT) -import logging - -SOME_PARAMETER = 'myParameter' -REQUIRED_PARAMETERS = [SOME_PARAMETER] - -# init the interface -# A ValueError error is raised if the KBC_DATADIR does not exist or contains non-existent path. -ci = CommonInterface() - -# A ValueError error is raised if the config.json file does not exists in the data dir. -# Checks for required parameters and throws ValueError if any is missing. -ci.validate_configuration(REQUIRED_PARAMETERS) - -# print Keboola Project ID from the environment variable if present: -logging.info(ci.environment_variables.project_id) - -# load particular configuration parameter -logging.info(ci.configuration.parameters[SOME_PARAMETER]) -``` - - -The above would read the `somemyParameter_user_parameter` parameter from the user-supplied configuration: - -{% highlight json %} -{ - "myParameter": "myValue" -} -{% endhighlight %} - -The following piece of code shows how to read parameters: - -{% highlight python %} -import csv -from keboola.component import CommonInterface - -# initialize the library and read parameter 'multiplier' -ci = CommonInterface() -multiplier = ci.configuration.parameters['multiplier'] - -# open the input and output files -with open('in/tables/source.csv', mode='rt', encoding='utf-8') as in_file, open('out/tables/destination.csv', mode='wt', encoding='utf-8') as out_file: - # write output file header - writer = csv.DictWriter(out_file, fieldnames=['number', 'someText', 'double_number'], dialect='kbc') - writer.writeheader() - - # read input file line-by-line - lazy_lines = (line.replace('\0', '') for line in in_file) - csv_reader = csv.DictReader(lazy_lines, dialect='kbc') - for row in csv_reader: - # do something and write row - writer.writerow({'number': row['number'], 'someText': row['someText'], 'double_number': int(row['number']) * multiplier}) -{% endhighlight %} - -Note that we have also simplified reading and writing of the CSV files using the `dialect='kbc'` option. The dialect is -registered automatically when the `CommonInterface` class is initialized. - -### Processing input tables -- Manifest vs I/O mapping - -Input and output tables specified by the user are listed in the [configuration file](/extend/common-interface/config-file/). -Apart from that, all input tables provided by user also include manifest file with additional metadata. - -Tables and their manifest files are represented by the `keboola.component.dao.TableDefinition` object and may be loaded -using the convenience method `get_input_tables_definitions()`. The result object contains all metadata about the table, -such as [manifest file](/extend/common-interface/manifest-files/#dataintables-manifests) representations (if present), system path and name. - -#### Manifest & input folder content - -```python -from keboola.component import CommonInterface -import logging - -# init the interface -ci = CommonInterface() - -input_tables = ci.get_input_tables_definitions() - -# print path of the first table (random order) -first_table = input_tables[0] -logging.info(f'The first table named: "{first_table.name}" is at path: {first_table.full_path}') - -# get information from table manifest -logging.info(f'The first table has following columns defined in the manifest {first_table.column_names}') - -``` - -#### Get input table by name - -```python -from keboola.component import CommonInterface - -# init the interface -ci = CommonInterface() -table_def = ci.get_input_table_definition_by_name('input.csv') -``` - -#### Using I/O mapping - -```python -import csv -from keboola.component import CommonInterface - -# initialize the library -ci = CommonInterface() - -# get list of input tables from the input mapping () -tables = ci.configuration.tables_input_mapping -j = 0 -for table in tables: - # get csv file name - inName = table.destination - - # read input table manifest and get its physical representation - table_def = ci.get_input_table_definition_by_name(table.destination) - - # get csv file name with full path from output mapping - outName = ci.configuration.tables_output_mapping[j].full_path - - # get file name from output mapping - outDestination = ci.configuration.tables_output_mapping[j]['destination'] -``` - -### Output tables - manifest files and processing results - -The component may define output [manifest files](https://developers.keboola.com/extend/common-interface/manifest-files/#dataouttables-manifests) -that define options on storing the results back to the Keboola Storage. This library provides methods that simplifies -the manifest file creation and allows defining the export options. - -`TableDefinition` object serves as a result container containing all the information needed to store the Table into the Storage. -It contains the manifest file representation and initializes all attributes available in the manifest. -This object represents both Input and Output manifests. All output manifest attributes are exposed in the class. - -There are convenience method for manifest creation `CommonInterface.write_tabledef_manifest()`. -Also it is possible to create the container for the output table using the `CommonInterface.create_out_table_definition()` -(useful particularly when working with [sliced tables](/extend/common-interface/folders/#sliced-tables)). - -```python -from keboola.component import CommonInterface -from keboola.component.dao import ColumnDefinition, DataType, SupportedDataTypes, BaseType - -# init the interface -ci = CommonInterface(data_folder_path='data') - -# create container for the result -out = ci.create_out_table_definition("testDef", - schema=['foo', 'bar'], - destination='some-destination', - primary_key=['foo'], - incremental=True, - delete_where={'column': 'lilly', - 'values': ['a', 'b'], - 'operator': 'eq'}) - -# update column -out.update_column('foo', - ColumnDefinition(data_types=BaseType(dtype=SupportedDataTypes.INTEGER, length='20'))) - -# add new columns -out.add_column('note', ColumnDefinition(nullable=False)) -out.add_column('test1') -out.add_columns(['test2', 'test3', 'test4']) - -# add new typed column -out.add_column('id', ColumnDefinition(primary_key=True, - data_types={'snowflake': DataType(dtype="INTEGER", length='200')}) - ) - -out.add_columns({ - 'phone': ColumnDefinition(primary_key=True, - data_types={'snowflake': DataType(dtype="INTEGER", length='200'), - 'bigquery': DataType(dtype="BIGINT")}), - 'new2': ColumnDefinition(data_types={'snowflake': DataType(dtype="INTEGER", length='200')}), - }) - -# delete columns -out.delete_column('bar') -out.delete_columns(['test2', 'test3']) - - -# write some content -with open(out.full_path, 'w') as result: - result.write('line') - -# write manifest -ci.write_manifest(out) -``` - - -### Processing input files - -Similarly as tables, [files and their manifest files](/extend/common-interface/folders/#folder-datainfiles) are represented by the `keboola.component.dao.FileDefinition` object and may be loaded -using the convenience method `get_input_files_definitions()`. The result object contains all metadata about the file, -such as manifest file representations, system path and name. - -The `get_input_files_definitions()` supports filter parameters to filter only files with a specific tag or retrieve only the latest file of each. -This is especially useful because the Keboola input mapping will by default include all versions of files matching specific tag. By default, the method -returns only the latest file of each. - -```python -from keboola.component import CommonInterface -import logging - -# init the interface -ci = CommonInterface() - -input_files = ci.get_input_files_definitions(tags= ['my_tag'], only_latest_files=True) - -# print path of the first file (random order) matching the criteria -first_file = input_files[0] -logging.info(f'The first file named: "{input_files.name}" is at path: {input_files.full_path}') -``` - - -When working with files it may be useful to retrieve them in a dictionary structure grouped either by name or a tag group. -For this there are convenience methods `get_input_file_definitions_grouped_by_tag_group()` and `get_input_file_definitions_grouped_by_name()` - - -```python -from keboola.component import CommonInterface -import logging - -# init the interface -ci = CommonInterface() - -# group by tag -input_files_by_tag = ci.get_input_file_definitions_grouped_by_tag_group(only_latest_files=True) - -# print list of files matching specific tag -logging.info(input_files_by_tag['my_tag']) - -# group by name -input_files_by_name = ci.get_input_file_definitions_grouped_by_name(only_latest_files=True) - -# print list of files matching specific name -logging.info(input_files_by_name['image.jpg']) - -``` - -#### Processing state files - -[State files](/extend/common-interface/config-file/#state-file) can be easily loaded and written -using the `get_state_file()` and `write_state_file()` methods: - -```python -from keboola.component import CommonInterface -from datetime import datetime -import logging - -# init the interface -ci = CommonInterface() - -last_state = ci.get_state_file() - -# print last_updated if exists -logging.info(f'Previous job stored following last_updated value: {last_state.get("last_updated","")})') - -# store new state file -ci.write_state_file({"last_updated": datetime.now().isoformat()}) -``` - -### Logging - -The library automatically initializes STDOUT or GELF logger based on the presence of the `KBC_LOGGER_PORT/HOST` environment variables -upon the `CommonInterface` initialization. To use the GELF logger just enable the logger for your application in the -[Developer Portal](https://components.keboola.com/). -More details about logging options are available in a [dedicated article](/extend/common-interface/logging/#examples). - -With either setting, you can log your messages using the logging library: - -```python -from keboola.component import CommonInterface -from datetime import datetime -import logging - -# init the interface -ci = CommonInterface() - -logging.info("Info message") -``` - -To fully leverage the benefits of the GELF logger such as outputting the `Stack Trace` into the log event detail (available by clicking on the log event) -log exceptions using `logger.exception(ex)`. - -**TIP:** When the logger verbosity is set to `verbose` you may leverage `extra` fields to log the detailed message -in the detail of the log event by adding extra fields to you messages: - -```python -logging.error(f'{error}. See log detail for full query. ', - extra={"failed_query": json.dumps(query)}) -``` - - -If you use STDOUT logging note that in Python components, the output is buffered. The buffering may -be [switched off](https://stackoverflow.com/questions/107705/disable-output-buffering). The easiest solution is to run your script -with the `-u` option: you would use `CMD python -u ./main.py` in your `Dockerfile`. - -## Error Handling -The following [piece of code](https://github.com/keboola/component-generator/blob/master/templates/python-tests/src/main.py) is a good entry point: - -{% highlight python %} -import my_component -import os -import sys -import traceback - -try: - datadir = os.environ.get('KBC_DATADIR') or '/data/' - my_component.run(datadir) -except ValueError as err: - print(err, file=sys.stderr) - sys.exit(1) -except Exception as err: - print(err, file=sys.stderr) - traceback.print_exc(file=sys.stderr) - sys.exit(2) -{% endhighlight %} - -In this case, we consider everything derived from `ValueError` to be an error which should be shown to the end user. -Every other error will lead to a generic message, and only developers will see the details. -If you maintain that any user error is a `ValueError`, then whatever happens in the `my_component.run` will follow -the [general error handling rules](#error-handling). -You can, of course, modify this logic to your liking. diff --git a/extend/component/implementation/r.md b/extend/component/implementation/r.md index ff0b1401..1321934f 100644 --- a/extend/component/implementation/r.md +++ b/extend/component/implementation/r.md @@ -3,127 +3,5 @@ title: R Implementation Notes permalink: /extend/component/implementation/r/ redirect_from: - /extend/custom-science/r/ +redirect_to: https://help.keboola.com/extend/component/implementation/r/ --- - -* TOC -{:toc} - -## Docker -We recommend using the [Rocker version-stable](https://github.com/rocker-org/rocker-versioned) [images](https://hub.docker.com/r/rocker/r-ver/). -The [R base image](https://hub.docker.com/r/rocker/r-base/) does not keep older R versions, so the upgrades are not under your control. -If you want to use the same environment as in transformations, use [our image](#docker). - -## Working with CSV Files -We recommend that you follow the guidelines for the [R transformation](https://help.keboola.com/transformations/r/#development-tutorial). -The standard R functions for CSV files work without problems: - -{% highlight R %} -data <- read.csv(file = "in/tables/source.csv"); - -df <- data.frame( - col1 = paste0(data$first, 'ping'), - col2 = data$second * 42 -) -write.csv(df, file = "out/tables/result.csv", row.names = FALSE) -{% endhighlight %} - -You can also use the `write_csv` function from the [readr packages](https://cran.r-project.org/web/packages/readr/readr.pdf). It is faster. - -## Using Keboola Package -Keboola's [R component package](https://github.com/keboola/r-docker-application) provides functions to - -- read and parse the configuration file and parameters: `configData` property and `getParameters()` method. -- list input files and tables: `getInputFiles()`, `getInputTables()` methods. -- work with manifests containing table and file metadata: `getTableManifest()`, `getFileManifest()`, `writeTableManifest()`, `writeFileManifest()` methods. -- list expected outputs: `getExpectedOutputFiles()` and `getExpectedOutputTables()` methods. - -The library is a standard R package that is available by default in the production environment. -[Ready for use on GitHub](https://github.com/keboola/r-docker-application), it can be installed locally with `devtools::install_github('keboola/r-docker-application', ref = 'master')`. - -Use the library to read a user-supplied configuration parameter 'myParameter': - -{% highlight r %} -library(keboola.r.docker.application) -# initialize library -app <- keboola.r.docker.application::DockerApplication$new() -app$readConfig() - -# access the supplied value of 'myParameter' -app$getParameters()$myParameter -{% endhighlight %} - -The library contains a single [RC class](http://adv-r.had.co.nz/OO-essentials.html#rc) `DockerApplication`; the parameter of the constructor is the path to the data directory. -After that you can call `readConfig()` to actually read and parse the configuration file, and then read the `myParameter` parameter from the user-supplied configuration: - -{% highlight json %} -{ - "myParameter": "myValue" -} -{% endhighlight %} - -When the application is initialized `app <- keboola.r.docker.application::DockerApplication$new()`, it read the configuration file from the constructor -argument, if no argument is provided, the [`KBC_DATADIR` environment variable](/extend/common-interface/environment/#environment-variables) is used. -You can obtain inline help and the list of library functions by running the `?DockerApplication` command. - -### Dynamic Input/Output Mapping -In our [tutorial](/extend/component/tutorial/), we show components which have names of their input/output tables hard-coded. -The following example shows how to read the input and output mapping specified by the end user, -which is accessible in the [configuration file](/extend/common-interface/config-file/). It demonstrates -how to read and write tables and table manifests. File manifests are handled the same way. For a full authoritative list -of items returned in table list and manifest contents, see [the specification](/extend/common-interface/config-file/). - -Note that the `destination` label in the script refers to the destination from the -[mapper perspective](/extend/component/tutorial/input-mapping/). The input mapper takes `source` tables -from the user's storage and produces `destination` tables that become the input of the component. The output tables -of the component are consumed by the output mapper whose `destination` are the resulting tables in Storage. - -{% highlight r %} -# initialize library -app <- DockerApplication$new() -app$readConfig() - -# get list of input tables -tables <- app$getInputTables() -for (i in 1:nrow(tables)) { - # get csv file name - name <- tables[i, 'destination'] - - # get csv full path and read table data - data <- read.csv(tables[i, 'full_path']) - - # read table metadata - manifest <- app$getTableManifest(name) - if ((length(manifest$primary_key) == 0) && (nrow(data) > 0)) { - # no primary key present, create one - data[['primary_key']] <- seq(1, nrow(data)) - } else { - data[['primary_key']] <- NULL - } - - - # do something clever - names(data) <- paste0('batman_', names(data)) - - # get csv file name with full path from output mapping - outName <- app$getExpectedOutputTables()[i, 'full_path'] - # get file name from output mapping - outDestination <- app$getExpectedOutputTables()[i, 'destination'] - - # write output data - write.csv(data, file = outName, row.names = FALSE) - - # write table metadata - set new primary key - app$writeTableManifest(outName, destination = outDestination, primaryKey = c('batman_primary_key')) -} -{% endhighlight %} - -To test the code, set an arbitrary number of input/output mapping tables. Keep in mind to set the same number -of inputs and outputs. The names of the CSV files are arbitrary. - -{: .image-popup} -![Dynamic mapping screenshot](/extend/component/dynamic-mapping.png) - -## Logging -In R components, the outputs printed in rapid succession are sometimes joined into a single event; -this is a known behavior of R and it has no workaround. See a [dedicated article](/extend/common-interface/logging/#examples) if you want to -implement a GELF logger. diff --git a/extend/component/index.md b/extend/component/index.md index bd2f5c39..5150457b 100644 --- a/extend/component/index.md +++ b/extend/component/index.md @@ -1,103 +1,9 @@ ---- -title: Components -permalink: /extend/component/ -redirect_from: - - /extend/docker/ - - /extend/custom-science/ - - /extend/custom-science/development/ - ---- - -* TOC -{:toc} - -Components allow you to [extend](/extend/) Keboola. -The data interface to components is very similar to [Transformations](https://help.keboola.com/transformations/) --- data is exchanged as -CSV files in [designated directories](/extend/common-interface/). - -### Intro to Component Creation -As a developer, you implement the application logic in a language of your choice and store it in a -git repository. The component must adhere to our [common interface](/extend/common-interface/). -To start quickly, use our component generators for [Python](https://github.com/keboola/cookiecutter-python-component) or [PHP, CI pipelines](https://github.com/keboola/component-generator) that can generate a skeleton of the component for you. We also provide libraries to help you with implementation in -[Python](https://github.com/keboola/python-component) and -[PHP](https://github.com/keboola/php-docker-application). -Check our example component in [Python](https://github.com/keboola/cookiecutter-python-component/tree/main/%7B%7Bcookiecutter.repository_folder_name%7D%7D). - -The main part of the [common interface](/extend/common-interface/) is the specification how -[CSV files and designated folders](/extend/common-interface/folders/) are used to exchange data between Keboola and components: - -- Applications process input tables stored in CSV files and generate result tables in CSV files. -- Extractors write results in the same way as applications, but instead of reading their -input from Keboola tables, they get it from an external source (usually an API). -- Writers, on the other hand, access their input tables in the same way as applications, but push their results into external systems and do not generate any Keboola tables. - - -Apart from this basic usage, the common interface offers many more features: - -- Passing parameters -- Error control -- Working with metadata -- OAuth support -- Working with non-CSV files -- Logging configuration -- Etc. - -Our [Job Queue](/extend/job-queue/) service makes sure that the common interface is honoured -from our side. It also takes care of executing your component in its own [isolated environment](/extend/job-queue/). - -## Requirements -Before you start developing a new component, you should - -- have a [Keboola project](/#development-project) where you can test your code. -- get yourself acquainted with Docker. You should be -able to run `docker` commands. Strictly speaking, you can get away -with not using them, but it will certainly speed things up for you. -- be able to send API requests. Although you can use the [api.keboola.com](https://api.keboola.com/) console, we -recommend using [Postman](https://www.getpostman.com/) as it is -more convenient. A list of [sample requests](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest) -is available. -- have a git repository ([Github](https://github.com/) or [Bitbucket](https://bitbucket.org/) are recommended, -although any other host should work as well). - -You can work with your component in your Keboola projects immediately as soon as you -[create it](/extend/component/tutorial/). However, to make the component publicly available to all users, -it must be [published](/extend/publish/). - -## Component Types -The following component types are currently allowed: - -- **Extractor** -- a component designed to bring data into Keboola Storage -- **Writer** -- a component designed to bring data to an external system -- **Application** -- another arbitrary component -- **Processor** -- a [processor](/extend/component/processors/) -- **Code Pattern** -- a [code pattern](/extend/component/code-patterns/) -- Transformation -- a special type of component for transformations -- Other -- a completely special component - -The type of a component has no effect on the component internals -- i.e., all components regardless of their type -share the identical features of the [common interface](/extend/common-interface/). The component type just tells -the end-user what behavior to expect from a given component. For example, an extractor is expected to extract data -from an external system. Nothing prevents it from processing data from Storage -(e.g., [Geocoding](https://help.keboola.com/components/extractors/other/geocoding-augmentation/) does that), -but its primary reason of existence is to bring data into Storage. - -The expected behavior of the above component types can be described in more detail as follows: - -- **Extractor** -- extracts data from an external system into Storage, uses no Storage tables on input, has a UI. -- **Writer** -- loads data from Storage into an external system, generates no Storage tables on output, has a UI. -- **Application** -- processes data in Storage, either something like a wrapped transformation or using an external service. Or, it does not work with data in Storage at all. This component has a UI. -- **Processor** -- is designed for post-processing or pre-processing data of other components. It is not designed to run alone and has no UI. -- **Code Pattern** -- generates code for the transformation's component. This component has a UI. -- Transformation -- represents a transformation engine. The UI treats these components specially and expects that they have similar capabilities -and configuration options. These are created by Keboola. If you wish to bring your own, please contact us first. -- Other -- this component type has a special role in the UI, it has no standard component UI. Notable "other" components are: - - `keboola.variables` -- Component for storing [variables](/integrate/variables/) configurations. Use the standard [configurations API](https://api.keboola.com/?service=storage#tag--Component-Configurations). No jobs can be made. - - `keboola.storage` -- Placeholder component for actions from Storage service. Neither configurations nor jobs can be made. Use the - [dedicated API](https://api.keboola.com/?service=storage) to work with Storage. - -## Next Steps -- Create a [developer account](/extend/component/tutorial/#before-you-start) so that you can create your own components. -- Follow our [tutorial](/extend/component/tutorial/) to build a "Hello, World!" component in 10 minutes. -- Follow the [next steps](/extend/component/tutorial/input-mapping/) of the tutorial to understand how your component interacts with Keboola. -- See more about [testing and debugging of components](/extend/component/tutorial/debugging/) in the Keboola environment. -- Request [publication](/extend/publish/) of your component. +--- +title: Components +permalink: /extend/component/ +redirect_from: + - /extend/docker/ + - /extend/custom-science/ + - /extend/custom-science/development/ +redirect_to: https://help.keboola.com/extend/component/ +--- diff --git a/extend/component/processors.md b/extend/component/processors.md index d292163c..b45a0ec7 100644 --- a/extend/component/processors.md +++ b/extend/component/processors.md @@ -1,307 +1,8 @@ ---- -title: Processors -permalink: /extend/component/processors/ -redirect_from: - - /integrate/docker-runner/processors/ - - /extend/docker-runner/processors/ ---- - -* TOC -{:toc} - -Processors are additional components which may be used **before or after** running an arbitrary component -(extractor, writer, etc.). - -When [Job Queue](/extend/job-queue/) runs a Docker image (a container is created), a processor -may be used to **pre-process the inputs** (files or tables) supplied to that container, or it may be used to **post-process -the container outputs**. For example, if an extractor extracts CSV data in a non-UTF8 encoding, you can use the -[`iconv` processor](https://github.com/keboola/processor-iconv/blob/master/README.md) as a post-processor to -convert the CSV to UTF-8 as expected by [Storage](https://help.keboola.com/storage/). See the -[tutorial](/extend/component/tutorial/configuration/) for a quick example of using processors. - -Processors are technically supported in any configuration of any component. However, as an **advanced feature**, they have little to no -[support in the UI](/extend/component/ui-options/#genericdockerui-processors). To manually configure processors, -you have to use the [Component Configuration API](https://api.keboola.com/?service=storage#tag--Component-Configurations). -See the respective part of our [documentation](/integrate/storage/api/configurations/) for -examples of working with the [Component Configuration API](/integrate/storage/api/configurations/). -If you want to implement your own processor, see our [implementation notes](/extend/component/implementation/#implementing-processors). - -If the component does not contain the [respective configuration field](/extend/component/ui-options/#genericdockerui-processors) or -an [advanced configuration mode](https://help.keboola.com/extractors/other/aws-s3/#advanced), processors are -completely **invisible in the UI**. In such case, modifying the configuration through the UI may delete the processor configuration -(though you can always [rollback](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-/versions/-versionId-/rollback)). -Therefore be sure to add an **appropriate warning** to the configuration description. - -## Configuration -By running the -[Get Configuration Detail](https://api.keboola.com/?service=storage#get-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-) -request for a specific component ID and configuration ID, you obtain the actual configuration contents. -You can see [an example request](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) -for getting a configuration with ID `365111648` for the component called Email Attachments extractor (ID `keboola.ex-email-attachments`): - -{% highlight json %} -{ - "id": "365111648", - "name": "Processor test", - "description": "", - "created": "2018-03-10T08:13:08+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "version": 3, - "changeDescription": "Update name", - "isDeleted": false, - "configuration": { - "parameters": { - "email": "572-365111648-5aa3858e91ed1@import.keboola.com", - "delimiter": ",", - "enclosure": "\"", - "primaryKey": [], - "incremental": false - } - }, - "rowsSortOrder": [], - "rows": [], - "state": { - "lastDownloadedFileTimestamp": "1520666119" - }, - "currentVersion": { - "created": "2018-03-10T08:16:54+0100", - "creatorToken": { - "id": 27865, - "description": "ondrej.popelka@keboola.com" - }, - "changeDescription": "Update name" - } -} -{% endhighlight %} - -From this, the actual configuration is the **contents** of the `configuration` node. Therefore: - -{% highlight json %} -{ - "parameters": { - "email": "572-365111648-5aa3858e91ed1@import.keboola.com", - "delimiter": ",", - "enclosure": "\"", - "primaryKey": [], - "incremental": false - } -} -{% endhighlight %} - -## Adding Processor -Processors are configured in the `processors` section in the `before` array or the `after` array (rarely both). -For example, you might want to configure the [`processor-skip-lines`](https://github.com/keboola/processor-skip-lines): - -{% highlight json %} -{ - "parameters": { - "email": "572-365111648-5aa3858e91ed1@import.keboola.com", - "delimiter": ",", - "enclosure": "\"", - "primaryKey": [], - "incremental": false - }, - "processors": { - "after": [ - { - "definition": { - "component": "keboola.processor-skip-lines" - }, - "parameters": { - "lines": 1, - "direction_from": "top" - } - } - ] - } -} -{% endhighlight %} - -The configuration parameters of a processor are always described in [its documentation](https://github.com/keboola/processor-skip-lines). -The above configuration defines that a `keboola.processor-skip-lines` (which removes a certain number of lines from the file) -will run **after** this particular configuration of the Email Attachment extractor is finished, -but **before** its results are loaded into Storage. When the processor is finished, its outputs are loaded -into Storage as if they were the outputs of the extractor itself. - -### Specifying Processor Version - -You can specify a particular version of the processor by adding an optional `tag` parameter in the `definition`. This parameter allows you to select a specific version: - -{% highlight json %} -{ - "definition": { - "component": "keboola.processor-skip-lines", - "tag": "1.0.0" - } -} -{% endhighlight %} - -If the tag parameter is omitted, the processor will automatically use the latest released version. - -To save the configuration, use the [Update Configuration API call](https://api.keboola.com/?service=storage#put-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-). -When updating the configuration, you must provide `componentId`, `configurationId`, and the actual contents of -the configuration in the `configuration` form field. Make sure to supply only the **contents** of the `configuration` -node and to properly escape the form data. - -See our [configuration documentation](/integrate/storage/api/configurations/#modifying-a-configuration) for -a more thorough description and the *Add processor to Email Attachments Extractor Configuration* example -in our [collection](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). -Remember, the processors can be [chained](/extend/component/tutorial/processors/#chaining-processors) to -achieve more advanced processing. - -### Available Processors -You can obtain a list of available processors using the -[Developer Portal UI](https://components.keboola.com/components) or the [List Components Public API](https://api.keboola.com/?service=developer-portal#get-/apps) -of the Developer Portal. The important parts are `id`, which is required for configuration, -and `documentationUrl`, which describes additional parameters of the processor. - -### Configuring Parameters -A processor may allow (or require) parameters. These are entered in the `parameters` section. -The below configuration sets values for two parameters --- `lines` and `direction_from`: - -{% highlight json %} -{ - "processors": { - "after": [ - { - "definition": { - "component": "keboola.processor-skip-lines" - }, - "parameters": { - "lines": 1, - "direction_from": "top" - } - } - ] - } -} -{% endhighlight %} - -The names and allowed values of the parameters are fully up to the processor interpretation and validation -and are described in the respective processor documentation. - -### Using Processors with Configuration Rows -If the configuration uses [Configuration Rows](/integrate/storage/api/configurations/#configuration-rows), -you have to use the [Update Configuration Row](https://api.keboola.com/?service=storage#put-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-/rows/-rowId-) -API call to set the processors. - -Provide `componentId`, `configurationId`, `rowId` and the contents of the configuration in -the same manner as when [adding a processor to configuration](#adding-a-processor). - -See an example *Add processor to S3 Extractor configuration Row* in -[our collection](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). -It shows how to set a processor for the configuration row with ID `364481153` in configuration `364479526` of -the AWS S3 extractor (component ID `keboola.ex-aws-s3`). The configuration is the following: - -{% highlight json %} -{ - "parameters": { - "bucket": "travis-php-db-import-tests-s3filesbucket-vm9zhtm5jd7s", - "key": "tw_accounts.csv", - "saveAs": "first-table", - "includeSubfolders": false, - "newFilesOnly": true - }, - "processors": { - "after": [ - { - "definition": { - "component": "keboola.processor-skip-lines" - }, - "parameters": { - "lines": 1 - } - } - ] - } -} -{% endhighlight %} - -## Chaining Processors -Remember, processors can be [chained](/extend/component/tutorial/processors/#chaining-processors) and therefore -should be as simple as possible. For example, a processor reading tables in CSV should assume that these are -available in the [standard format](https://help.keboola.com/storage/tables/csv-files/#output-csv-format) and that the -table manifests are available. - -### Extractor Example -For example, assume that you have a component which extracts the following data: - - Dump from ACME Anvil CRM - SLA: 24h - Day|AnvilsDelivered - 2050-12-10|100|5|4|4 - 2050-12-11|56|1|2 - 2050-12-12|131|9|7|3 - -First apply the [processor-skip-lines](https://github.com/keboola/processor-skip-lines) to obtain something -resembling a CSV file: - - Day|AnvilsDelivered - 2050-12-10|100|5|4|4 - 2050-12-11|56|1|2 - 2050-12-12|131|9|7|3 - -Then apply the [processor-create-manifest](https://github.com/keboola/processor-create-manifest) to -set the delimiter and enclosure in the file manifest. - -After that, use the [processor-format-csv](https://github.com/keboola/processor-format-csv) to convert the file -from the format specified in the manifest to the standard format: - - "Day","AnvilsDelivered" - "2050-12-10","100","5","4","4" - "2050-12-11","56","1","2" - "2050-12-12","131","9","7","3" - -Finally, you can use the [processor-headers](https://github.com/keboola/processor-headers) to make the data orthogonal: - - "Day","AnvilsDelivered","col1","col2","col3" - "2050-12-10","100","5","4","4" - "2050-12-11","56","1","2","" - "2050-12-12","131","9","7","3" - -### Writer Example -A chain similar to the above can be used for a writer too. Assume that you need to send the following data to -the very special ACME Anvil CRM: - - Import: CRM - ImportFormat: AnvilPSV - Date: 2018-10-01 - Type: MANF-DLVR-PLAN - - Day|AnvilManufacturingPlan|AnvilDeliveryPlan - 2050-12-10|100|533 - 2050-12-11|100|695 - 2050-12-12|100|923 - -The data exported from Storage will be in the following format: - - "Day","AnvilManufacturingPlan","AnvilDeliveryPlan" - "2050-12-10","100","533" - "2050-12-11","100","695" - "2050-12-12","100","923" - -Then apply the [processor-format-csv](https://github.com/keboola/processor-format-csv) to convert the file -from the standard format to the format required by the Anvil CRM writer: - - Day|AnvilManufacturingPlan|AnvilDeliveryPlan - 2050-12-10|100|533 - 2050-12-11|100|695 - 2050-12-12|100|923 - -Create a custom processor to put the header in: - - Import: CRM - ImportFormat: AnvilPSV - Date: 2018-10-01 - Type: MANF-DLVR-PLAN - - Day|AnvilManufacturingPlan|AnvilDeliveryPlan - 2050-12-10|100|533 - 2050-12-11|100|695 - 2050-12-12|100|923 - -Finally, the Anvil CRM writer can send the result to the CRM system. Or you can have the header function be part of the -writer itself. That decision should be made depending on whether the header must always be present (part of the writer) or is optional (processor). +--- +title: Processors +permalink: /extend/component/processors/ +redirect_from: + - /integrate/docker-runner/processors/ + - /extend/docker-runner/processors/ +redirect_to: https://help.keboola.com/extend/component/processors/ +--- diff --git a/extend/component/running/index.md b/extend/component/running/index.md index d32fc274..0fccfb42 100644 --- a/extend/component/running/index.md +++ b/extend/component/running/index.md @@ -1,285 +1,8 @@ ---- -title: Running Components -permalink: /extend/component/running/ -redirect_from: - - /extend/docker/running/ - - /extend/common-interface/sandbox/ ---- - -* TOC -{:toc} - -One of the great advantages of dockerized components is that the components always run in the -same environment defined by the Docker image. When running in Keboola, there are, however, some outside -environment bindings for you to take care of. - -Before you start, make sure you have Docker set up correctly, -particularly that you know your **host path** for sharing files -and that you understand the basic concepts of creating a Dockerized application. -In this guide, we will use `/user/johndoe/data/` as the **host path** containing the -[data folder](/extend/common-interface/folders/). - -You can also run your component in your own environment. In that case, set the `KBC_DATADIR` environment -variable to point to the data folder. With this approach, you loose the advantage of the properly defined -environment, but in some cases, it may be a nice shortcut. - -For more details on how to develop a component, see the corresponding [tutorial](/extend/component/tutorial/), -especially the part on [debugging](/extend/component/tutorial/debugging/). - -## Basic Run -The basic run command we use (assuming that we want to run the -[`keboola-test.ex-docs-tutorial`](https://github.com/keboola/ex-docs-tutorial) component) is as follows: - - docker run --volume=/user/johndoe/data/:/data --memory=4000m --net=bridge -e KBC_RUNID=123456789 -e KBC_PROJECTID=123 -e KBC_DATADIR=/data/ -e KBC_CONFIGID=test-123 quay.io/keboola/keboola-test.ex-docs-tutorial - -The `--volume` parameter ensures the `/data/` folder will be mounted into the image. This is used -to inject the input data and configuration into the image. Make sure not to put any spaces around the `:` character. - -The `--memory` and `--net` parameters are component limits and are specified in the [Developer Portal](https://components.keboola.com/). - -The `-e` parameters define [environment variables](/extend/common-interface/environment/). When entering -environment variables on the command line, do **not** put any spaces around the `=` character. - -### Test -Download our [sample data folder](/extend/data.zip), extract it into your **host folder**, and run this command: - - docker run --volume=/user/johndoe/data/:/data --memory=4000m --net=bridge -e KBC_RUNID=123456789 -e KBC_PROJECTID=123 -e KBC_DATADIR=/data/ -e KBC_CONFIGID=test-123 quay.io/keboola/keboola-test.ex-docs-tutorial - -You should see the following output: - - All done - - Environment variables: - KBC_RUNID: 123456789 - KBC_PROJECTID: 123 - KBC_DATADIR: /data/ - KBC_CONFIGID: test-123 - -In addition, the `destination.csv` file will be created in your **host folder** in the `data/out/tables/` folder, with the following contents: - - number,someText,double_number - 10,ab,20 - 20,cd,40 - 25,ed,50 - 26,fg,52 - 30,ij,60 - -If you encounter any errors, you can run the image interactively: - - docker run --volume=/user/johndoe/data/:/data --memory=4000m --net=bridge -e KBC_RUNID=123456789 -e KBC_PROJECTID=123 -e KBC_DATADIR=/data/ -e KBC_CONFIGID=test-123 -i -t --entrypoint=/bin/bash quay.io/keboola/keboola-test.ex-docs-tutorial - -Then you can inspect the container with standard OS (CentOS) commands and/or run the script manually with -`php /home/main.php`. - -After you have mastered this step, you can run any Docker component on your machine. - -## Debugging -There are two main variants of the Run Job API call useful for debugging purposes: - - - Debug - - Run Tag - -The [Run Job](https://api.keboola.com/?service=job-queue#post-/jobs) API call is useful for obtaining an -environment configuration for a component. It will create a snapshot of the -[data folder](/extend/common-interface/folders/) -(including input mapping and configuration files) and then it will upload the snapshot to the [Files section](https://help.keboola.com/storage/file-uploads/) -of Storage. Then the component will be run and another snapshot will be created with the resulting contents of the data directory. -This gives you snapshots of the data directory before and after a component is run. The debug -API call does not write any tables or files (other than the archive) to the Keboola project, so it is very safe to run. Note however that -any side effects of the component are still performed (e.g. writers still write data to their destination). - -The Run Job API call also allows you to run a job in the production environment but using a specific tag of the Docker image. -This means you can test your unreleased image on real configurations in real projects without affecting -any users using that component. See the [tutorial](/extend/component/tutorial/debugging/#running-specific-tags) -for instructions. - -## Preparing Data folder -In order to run and debug a Keboola component (including [R](https://help.keboola.com/transformations/r/) and [Python](https://help.keboola.com/transformations/python/) Transformations) -on your own computer, you need to manually supply the component with -a [data folder and configuration file](/extend/common-interface/). The above mentioned -[Run Job API call](https://api.keboola.com/?service=job-queue#post-/jobs) -is designed to do that. - -We recommend that you use the [API console or Postman](/overview/api/) to call the API. -A [collection of examples](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) of the -Run Job API call in debug mode is available in Postman Docs. - -### Prepare -[Create a table](https://help.keboola.com/tutorial/load/) in Keboola Storage. -In the following example, the table is stored in the `in.c-main` bucket and is called `sample`. The table ID is -therefore `in.c-main.sample`. You also need a [Storage API token](https://help.keboola.com/storage/tokens/). - -{: .image-popup} -![Storage Screenshot](/extend/component/running/sandbox-data.png) - -### Running without Configuration -In the [collection of sample requests](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb), -there is a **Run without Configuration** example with the following JSON in its body: - -{% highlight json %} -{ - "configData": { - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.sample", - "destination": "source.csv" - } - ] - } - }, - "parameters": { - "sound": "Moo", - "repeat": 2 - } - } -} -{% endhighlight %} - -The node `configData.storage.input.tables.source` refers to the existing table ID (the table created -in the previous step) in Storage. The `configData.storage.input.tables.destination` node refers to the -destination to which the table will be downloaded for the component; it will therefore be the -**source** for the component. - -The entire `configData.storage` node is generated by the UI. The node `parameters` contains arbitrary -parameters which are passed to the component. The URL of the request -is `https://queue.keboola.com/jobs` (in the [US Region](/overview/api/#regions-and-endpoints)). - -The request is same to the one used when running a job. - -### Running with Configuration -In the [collection of sample requests](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb), -there is a **Run with Configuration** example with the the following JSON in its body: - -{% highlight json %} -{ - "config": "328831433" -} -{% endhighlight %} - -When you create a configuration in Keboola, it is assigned a configuration ID --- `328831433` --- in our example. -Use this ID instead of manually crafting the request body. You need to replace `328831433` with your own -configuration ID. The request URL is as follows: - -{: .image-popup} -![Configuration screenshot](/extend/component/running/input-configuration.png) - -You can create a configuration for non-public components by visiting the direct URL: - - https://connection.keboola.com/admin/projects/{PROJECT_ID}/extractors/{COMPONENT_ID} - -In this case replace `COMPONENT_ID` with `keboola-test.ex-docs-tutorial` and PROJECT_ID with the id of your testing project. - -**Important**: If you actually want to **run** the above 328831433 configuration, you also need -to set the output mapping from `destination.csv` to a table. - -### Getting Result -When running the request with valid parameters, you should receive a response similar to this: - -{% highlight json %} -{ - "id": "176883685", - "url": "https://queue.keboola.com/jobs/176883685", - "status": "waiting" -} -{% endhighlight %} - -This means that an [asynchronous job](/integrate/jobs/) which will prepare the archive of the data folder has been created. -If curious, view the job progress under **Jobs** in Keboola: - -{: .image-popup} -![Job progress screenshot](/extend/component/running/sandbox-progress.png) - -The job will usually take slightly longer than the normal run job. When finished go to **Storage** --- **Files** in -Keboola. There you will find a `stage_0.zip` file with the data folder before the component was run and `stage_output.zip` before -the component output mapping was supposed to be done. You can now use this folder from `stage_0.zip` to run -the component locally. You should now be able to run the component with it: - - docker run --volume=/user/johndoe/data/:/data --memory=4000m --net=bridge -e KBC_RUNID=123456789 -e KBC_PROJECTID=123 -e KBC_DATADIR=/data/ -e KBC_CONFIGID=test-123 -i -t --entrypoint=/bin/bash quay.io/keboola/keboola-test.ex-docs-tutorial - - -## Running Component -If you want to run a component during development, it is the easiest to build it locally and -[run the built version](/extend/component/tutorial/debugging/). If you want to run a production code component, you -need to do a couple of things. Let's assume you want to run the `keboola-test.ex-docs-tutorial` component and you have -already [prepared the data directory](#preparing-the-data-folder). - -The next step is to obtain the repository settings and credentials from the -[Developer Portal](https://components.keboola.com/). You can either use the [API](https://api.keboola.com/?service=developer-portal) or -the [CLI](https://github.com/keboola/developer-portal-cli-v2). The CLI is easier to use. First set your service account credentials -in the environment: - - export KBC_DEVELOPERPORTAL_USERNAME=keboola-test+ex_docs_tutorial_travis - export KBC_DEVELOPERPORTAL_PASSWORD=RFlYs3HnDkbzyXIUkdPFRMubiCK-FTjy5-tNXrdzRX3qEBLvDQjnxFtAJGzg6UO. - -or - - SET KBC_DEVELOPERPORTAL_USERNAME=keboola-test+ex_docs_tutorial_travis - SET KBC_DEVELOPERPORTAL_PASSWORD=RFlYs3HnDkbzyXIUkdPFRMubiCK-FTjy5-tNXrdzRX3qEBLvDQjnxFtAJGzg6UO. - -on Windows. Then run the command to obtain the component repository: - - docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD quay.io/keboola/developer-portal-cli-v2 ecr:get-repository vendor component-id - -for example: - - docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD quay.io/keboola/developer-portal-cli-v2 ecr:get-repository keboola-test keboola-test.ex-docs-tutorial - -You will receive the repository URI, e.g.: - - 147946154733.dkr.ecr.us-east-1.amazonaws.com/developer-portal-v2/keboola-test.ex-docs-tutorial - -Then call a command to obtain credentials for the component repository: - - docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD quay.io/keboola/developer-portal-cli-v2 ecr:get-login vendor component-id - -for example: - - docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD quay.io/keboola/developer-portal-cli-v2 ecr:get-login keboola-test keboola-test.ex-docs-tutorial - -You will receive a `docker login` command which will authorize you to fetch the repository: - - docker login -u AWS -p ey...ODAzOH0= 147946154733.dkr.ecr.us-east-1.amazonaws.com - -Then pull the image from the registry: - - docker pull 147946154733.dkr.ecr.us-east-1.amazonaws.com/developer-portal-v2/keboola-test.ex-docs-tutorial - -Or run it directly: - - docker run --volume=/user/johndoe/data/:/data --memory=4000m --net=bridge -e KBC_RUNID=123456789 -e KBC_PROJECTID=123 -e KBC_DATADIR=/data/ -e KBC_CONFIGID=test-123 147946154733.dkr.ecr.us-east-1.amazonaws.com/developer-portal-v2/keboola-test.ex-docs-tutorial - -The `/user/johndoe/data/` path refers to the contents of the data folder. - -**Note for Windows users:** -If you receive the error `The stub received bad data.`, you have to modify the `%userprofile%\.docker\config.json` to e.g.: -{% highlight json %} -{ - "auths": { - "https://index.docker.io/v1/": { - "email": "email@example.com" - } - } -} -{% endhighlight %} - -This is a known [bug in Docker](https://github.com/docker/for-win/issues/1306), see [the workaround](https://github.com/Azure/azure-cli/issues/4843). - -## Running Transformations -Both R and Python transformations are implemented as Docker components. They can be run -locally as well. Use the [Run Job API call in debug mode](/extend/component/running/#preparing-the-data-folder) to obtain the data directory. -In the [API call](https://api.keboola.com/?service=job-queue#post-/jobs), specify the full -configuration (using the `configData` node). See [examples](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb) -for both R and Python transformations. - -To run R transformations, use: - - docker run --volume=/user/johndoe/data/:/data --memory=4000m --net=bridge -e KBC_RUNID=123456789 -e KBC_PROJECTID=123 -e KBC_DATADIR=/data/ -e KBC_CONFIGID=test-123 [quay.io/keboola/r-transformation](https://quay.io/repository/keboola/r-transformation):latest - -To run [Python transformations](https://quay.io/repository/keboola/python-transformation), use: - - docker run --volume=/user/johndoe/data/:/data --memory=4000m --net=bridge -e KBC_RUNID=123456789 -e KBC_PROJECTID=123 -e KBC_DATADIR=/data/ -e KBC_CONFIGID=test-123 quay.io/keboola/python-transformation:latest - -The transformation will run automatically and produce results. If you want to get into -the container interactively, use the `--entrypoint` parameter. +--- +title: Running Components +permalink: /extend/component/running/ +redirect_from: + - /extend/docker/running/ + - /extend/common-interface/sandbox/ +redirect_to: https://help.keboola.com/extend/component/running/ +--- diff --git a/extend/component/running/input-configuration.png b/extend/component/running/input-configuration.png deleted file mode 100644 index 9d5f392e..00000000 Binary files a/extend/component/running/input-configuration.png and /dev/null differ diff --git a/extend/component/running/sandbox-data.png b/extend/component/running/sandbox-data.png deleted file mode 100644 index 417a8e6f..00000000 Binary files a/extend/component/running/sandbox-data.png and /dev/null differ diff --git a/extend/component/running/sandbox-progress.png b/extend/component/running/sandbox-progress.png deleted file mode 100644 index 58867dd5..00000000 Binary files a/extend/component/running/sandbox-progress.png and /dev/null differ diff --git a/extend/component/tutorial/component-configuration.png b/extend/component/tutorial/component-configuration.png deleted file mode 100644 index dac4cbb6..00000000 Binary files a/extend/component/tutorial/component-configuration.png and /dev/null differ diff --git a/extend/component/tutorial/component-deployed.png b/extend/component/tutorial/component-deployed.png deleted file mode 100644 index eb01d080..00000000 Binary files a/extend/component/tutorial/component-deployed.png and /dev/null differ diff --git a/extend/component/tutorial/component-generator.png b/extend/component/tutorial/component-generator.png deleted file mode 100644 index 23afcf89..00000000 Binary files a/extend/component/tutorial/component-generator.png and /dev/null differ diff --git a/extend/component/tutorial/configuration-1.png b/extend/component/tutorial/configuration-1.png deleted file mode 100644 index 9524e336..00000000 Binary files a/extend/component/tutorial/configuration-1.png and /dev/null differ diff --git a/extend/component/tutorial/configuration-2.png b/extend/component/tutorial/configuration-2.png deleted file mode 100644 index 500901af..00000000 Binary files a/extend/component/tutorial/configuration-2.png and /dev/null differ diff --git a/extend/component/tutorial/configuration-3.png b/extend/component/tutorial/configuration-3.png deleted file mode 100644 index bbe684a3..00000000 Binary files a/extend/component/tutorial/configuration-3.png and /dev/null differ diff --git a/extend/component/tutorial/configuration-4.png b/extend/component/tutorial/configuration-4.png deleted file mode 100644 index ec9a0506..00000000 Binary files a/extend/component/tutorial/configuration-4.png and /dev/null differ diff --git a/extend/component/tutorial/configuration-sample.png b/extend/component/tutorial/configuration-sample.png deleted file mode 100644 index fd9eb190..00000000 Binary files a/extend/component/tutorial/configuration-sample.png and /dev/null differ diff --git a/extend/component/tutorial/configuration.md b/extend/component/tutorial/configuration.md index 0cb9888c..42179e46 100644 --- a/extend/component/tutorial/configuration.md +++ b/extend/component/tutorial/configuration.md @@ -1,160 +1,5 @@ --- title: Configuration permalink: /extend/component/tutorial/configuration/ +redirect_to: https://help.keboola.com/extend/component/tutorial/configuration/ --- - -* TOC -{:toc} - -In this part of the [tutorial](/extend/component/tutorial/), you will see how to pass -arbitrary configuration parameters to your component. By this time, you probably noticed -that your component has a configuration field: - -{: .image-popup} -![Screenshot -- Configuration Empty](/extend/component/tutorial/configuration-1.png) - -You can use this field to pass arbitrary configuration parameters to your component. -The parameters will be available in the [/data/config.json](/extend/common-interface/config-file/) file provided to the -component when it is [run](/extend/job-queue/). - -Note: if you don't want to hard-code the [`/data/` directory](/extend/common-interface/folders/#root-folder-data) use the [`KBC_DATADIR`](/extend/common-interface/environment/#environment-variables) environment variable. - -Let's assume you want to make the [sample component](/extend/component/tutorial/output-mapping/) -add a given sound to each row a given number of times. For that you'll need two parameters: `sound` and `repeat`. - -## Modifying Source Code -To implement the above, you can change the [sample component](/extend/component/tutorial/output-mapping/) to: - -```python -import csv -import os - -# Load the Component library to process the config file -from keboola.component import CommonInterface - -# Rely on the KBC_DATADIR environment variable by default, -# alternatively provide a data folder path in the constructor (CommonInterface('data')) -ci = CommonInterface() -params = ci.configuration.parameters - -print("Hello world from python") - -csvlt = '\n' -csvdel = ',' -csvquo = '"' - -# get input table definition by name -in_table = ci.get_input_table_definition_by_name('source.csv') - -with open(in_table.full_path, mode='rt', encoding='utf-8') as in_file, \ - open(os.path.join(ci.tables_out_path, 'odd.csv'), mode='wt', encoding='utf-8') as odd_file, \ - open(os.path.join(ci.tables_out_path, 'even.csv'), mode='wt', encoding='utf-8') as even_file: - lazy_lines = (line.replace('\0', '') for line in in_file) - reader = csv.DictReader(lazy_lines, lineterminator=csvlt, delimiter=csvdel, - quotechar=csvquo) - - odd_writer = csv.DictWriter(odd_file, fieldnames=reader.fieldnames, - lineterminator=csvlt, delimiter=csvdel, - quotechar=csvquo) - odd_writer.writeheader() - - even_writer = csv.DictWriter(even_file, fieldnames=reader.fieldnames, - lineterminator=csvlt, delimiter=csvdel, - quotechar=csvquo) - even_writer.writeheader() - i = 0 - for row in reader: - if i % 2 == 0: - even_writer.writerow(row) - else: - newRow = {} - for key in reader.fieldnames: - newRow[key] = row[key] + ''.join([params['sound']] * params['repeat']) - odd_writer.writerow(newRow) - i = i + 1 - - -``` - -At the beginning, the [Keboola Python Component library](https://github.com/keboola/python-component) is imported and -initialized by reading the `data` directory (`CommonInterface()`). Its property `configuration.parameters` will provide the -configuration parameters as a dictionary. Apart from that the [Python library](https://github.com/keboola/python-component) - provides methods to handle input / output files and many more (see [here](/extend/component/implementation/python/) for more information). - -Similar library is currently available also for the [R language](https://github.com/keboola/r-docker-application). -It does no magic or rocket science, so you can read the [config file](/extend/common-interface/config-file/) directly if you wish. - -Commit and push the code in your repository and tag it with a [normal version tag](https://semver.org/#spec-item-2). -This will trigger a [build on Travis CI](https://docs.travis-ci.com/) and automatically -deploy the new version into Keboola. Keep in mind that after the deployment, it may take up to 5 minutes for the update to propagate to all Keboola instances. - -## Verifying -To verify that the parameters work, simply edit the component configuration in Keboola and paste in, for example: - -{% highlight json %} -{ - "sound": "Moo", - "repeat": 2 -} -{% endhighlight %} - -{: .image-popup} -![Screenshot -- Configuration Filled](/extend/component/tutorial/configuration-2.png) - -Run the component and examine the job results. In the `odd` result table, you should see that `Moo` was added twice to every value. - -{: .image-popup} -![Screenshot -- Table Results](/extend/component/tutorial/configuration-3.png) - -## Creating UI -Entering configuration parameters using JSON data is quite low-level. Therefore you should -provide a UI for the end user. The easiest option is to take advantage of the -[JSON editor](https://github.com/jdorn/json-editor) based on the -[configuration schema](/extend/component/ui-options/configuration-schema/). For the above -configuration, the following schema can be created: - -{% highlight json %} -{ - "title": "Person", - "type": "object", - "properties": { - "sound": { - "type": "string", - "title": "Sound:", - "default": "Boo", - "description": "The sound to make." - }, - "repeat": { - "type": "integer", - "title": "Repeat sound:", - "description": "Number of times to repeat the sound.", - "default": 2, - "minimum": 0, - "maximum": 10 - } - }, - "required": ["sound", "repeat"] -} -{% endhighlight %} - -In the schema the two properties `sound` and `repeat` are declared along with the specification -of their form input fields. -You can test the above schema [online](http://jeremydorn.com/json-editor/) ([alternative](https://mozilla-services.github.io/react-jsonschema-form/)) and verify that the -form generated from it produces the desired JSON structure. Once satisfied with the result, -simply paste the schema into the **Configuration schema** in your component properties in the -[Developer Portal](https://components.keboola.com/). - -Once the change propagates to your Keboola instance, you should see the form in the UI: - -{: .image-popup} -![Screenshot -- Configuration Form](/extend/component/tutorial/configuration-4.png) - -The end user can now configure your component without writing the JSON with parameters. For a very complex UI, the -JSON schema editor is not really suitable; contact us about available options. - -## Summary -Your component can now successfully read configuration parameters provided by the end user. You can read more about all the features of the -[configuration file](/extend/common-interface/config-file/). -Keep in mind that the code presented above is simplified as it does not use any validation of -end user parameters. The next part of the tutorial will show you -how to [configure processors](/extend/component/tutorial/processors/). diff --git a/extend/component/tutorial/create-component-1.png b/extend/component/tutorial/create-component-1.png deleted file mode 100644 index 5ab40e12..00000000 Binary files a/extend/component/tutorial/create-component-1.png and /dev/null differ diff --git a/extend/component/tutorial/create-component-2.png b/extend/component/tutorial/create-component-2.png deleted file mode 100644 index 3301f7e0..00000000 Binary files a/extend/component/tutorial/create-component-2.png and /dev/null differ diff --git a/extend/component/tutorial/debug-1.png b/extend/component/tutorial/debug-1.png deleted file mode 100644 index dadeb9ca..00000000 Binary files a/extend/component/tutorial/debug-1.png and /dev/null differ diff --git a/extend/component/tutorial/debug-2.png b/extend/component/tutorial/debug-2.png deleted file mode 100644 index 30f58d6c..00000000 Binary files a/extend/component/tutorial/debug-2.png and /dev/null differ diff --git a/extend/component/tutorial/debug-3.png b/extend/component/tutorial/debug-3.png deleted file mode 100644 index 2c589683..00000000 Binary files a/extend/component/tutorial/debug-3.png and /dev/null differ diff --git a/extend/component/tutorial/debug-4.png b/extend/component/tutorial/debug-4.png deleted file mode 100644 index 9b67d51b..00000000 Binary files a/extend/component/tutorial/debug-4.png and /dev/null differ diff --git a/extend/component/tutorial/debugging.md b/extend/component/tutorial/debugging.md index 76c2d699..86bb3c1c 100644 --- a/extend/component/tutorial/debugging.md +++ b/extend/component/tutorial/debugging.md @@ -1,225 +1,5 @@ ---- -title: Debugging -permalink: /extend/component/tutorial/debugging/ ---- - -* TOC -{:toc} - -Because all components [run in an isolated environment](/extend/job-queue/), it may be harder to debug them. There is no way to -examine the component while it is running. However, there are some options how the production environment can be -replicated locally, so that you can analyze what is happening if something is not right. - -## Checking Errors and Version -There are two [types of errors](/extend/common-interface/environment/#return-values). In the case of -application errors, you (or any other end user) will only see a generic error message in the job result: - - Internal Error Something is broken. Our developers were notified about this error and will let you know what went wrong. - -At the same moment, you should get a full error message on your vendor [channel for receiving errors](/extend/component/tutorial/#before-you-start) (typically a Slack -or email message). If you have not received a message or you don't have a channel for receiving errors, contact us to set it up. - -Also, if the component is misbehaving, please double check that you are running the correct version. This can be done in -the job detail in the section **Parameters & Results**, where you can see the tag used to execute the job: - -{: .image-popup} -![Screenshot -- Job Tags](/extend/component/tutorial/debug-1.png) - -You should be able to trace the tag to a specific version of your source code. - -## Running Locally - -### Step 1 -- Obtain Sample Data and Configuration -Data between Keboola and your Docker image are exchanged using [CSV files](/extend/common-interface/) in -designated [directories](/extend/common-interface/folders/); they will be -injected into the image when you [run it](/extend/job-queue/). To simulate this, download an archive containing the data files -and [configuration](/extend/common-interface/config-file/) in the exact same format you get in the production environment. - -Use the [Create Job API](https://api.keboola.com/?service=job-queue#post-/jobs). -You can see it in our [API request collection](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). -In the [API call](https://api.keboola.com/?service=job-queue#post-/jobs), either specify the -full configuration (using the `configData` node) or refer to an existing configuration -of the component (using the `config` node). See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). - -The Run Job API call in debug mode will prepare the data folder for the component, put it inside an archive and upload it to Keboola Storage. -When running the request with valid parameters, you should receive a response similar to this: - -{% highlight json %} -{ - "id": "176883685", - "status": "waiting", - "mode": "run", - "tag": "1.2.3", - "isFinished": false, - "url": "https://queue.keboola.com/jobs/176883685" -} -{% endhighlight %} - -This means an [asynchronous job](/integrate/jobs/) for preparing the archive has been created. -If curious, view the job progress under **Jobs** in Keboola. -When the job finishes, you'll see a `stage_0.zip` file uploaded to your project. - -{: .image-popup} -![Screenshot -- Job Tags](/extend/component/tutorial/debug-2.png) - -You can send the Run Job API call in debug mode with a reference to an existing configuration id, or you can also supply the configuration directly in -the API request. In such case, use the `configData` attribute in the request body, e.g.: - -{% highlight json %} -{ - "config": "my-test-config", - "configData": { - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.test", - "destination": "source.csv" - } - ] - }, - "output": { - "tables": [ - { - "source": "destination.csv", - "destination": "out.c-main.test" - } - ] - } - }, - "parameters": { - } - } -} -{% endhighlight %} - -The above request corresponds to the following setting in the UI: - -{: .image-popup} -![Configuration Screenshot](/extend/component/tutorial/configuration-sample.png) - -### Step 2 -- Build Image -Then you can build your component code locally: - - docker build path/to/component/code --tag=my-component - -or - - docker build . --tag=my-component - -in the component directory. It should produce an output similar to the one below: - -{: .image-popup} -![Screenshot -- Building](/extend/component/tutorial/debug-3.png) - -### Step 3 -- Run Component with Sample Data -Once you have prepared the data folder with sample data and configuration, inject it into the Docker image. -In addition to the options shown in the example, there are many [other options](/extend/common-interface/config-file/) available. - -When you run an image, a **container** is created in which the component is running isolated. -Use the following command to run the image: - - docker run --volume=physicalhostpath:/data/ imageTag - -An image tag is the tag you supplied in the `--tag` parameter for `docker build` (`my-component` in the above example). -The physical host path depends on the system you are running. In our example image with default Windows -installation of Docker, this would be: - - docker run --volume=C:\Users\JohnDoe\data\:/data/ my-component - -Where the contents of the sample data folder are put in the user's home directory. If you have set everything correctly, -you should see **Hello world from python**, and a `destination.csv` file will appear in the `data/out/tables/` folder. - -{: .image-popup} -![Screenshot -- Running](/extend/component/tutorial/debug-4.png) - -You can then examine what the component did and what files it produced in the `data/out` folder. You can -also read more in-depth information about [running images](/extend/component/running/). - -### Step 4 -- Debug -Chances are that you received an ugly error message or warning. In that case, you might want to check the -contents of the image; specifically, if all the files are where you expect -them to be. - -To work with the component container interactively, use the following command: - - docker run --volume=physicalhostpath:/data/ -i -t --entrypoint=/bin/bash imageTag - -For instance: - - docker run --volume=C:\Users\JohnDoe\data\:/data/ -i -t --entrypoint=/bin/bash my-component - -This will override the default command specified in the `Dockerfile` -- `CMD ["python", "-u", "/code/main.py"]` -to launch [Bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) instead. The [`-i` and `-t` flags](https://docs.docker.com/engine/reference/commandline/run/) -ensure that the container runs in an interactive mode. -You can then inspect the container contents: 'ls /data/'. - -### Step 4 -- Modify -Chances are that you want to modify the component code often. If you modify the component code, you have to rebuild the -docker image. To avoid the slow and tedious work, run the image with the following command: - - docker run --volume=physicalhostpathtodata:/data/ --volume=physicalhostpathtocode:/code/ -i -t my-component - -For instance: - - docker run --volume=C:\Users\JohnDoe\data\:/data/ --volume=D:\wwwroot\ex-docs-tutorial\:/code/ -i -t my-component - -This means that the directory with the component code will shadow the one inside the image (defined by the `COPY . /code/` -instruction in `Dockerfile`) and you will run the current code in the image environment. - -## Running Specific Tags -The Run Job API call in debug mode is very powerful but it always runs the production version of the component. There are cases where you might want to -run a test or development version of a component. In such situations, an alternative may be to run a specific image tag. - -Let's say that you need to list all files on input for some reason. Following the -[example component](/extend/component/tutorial/), you would have to add something like this -to the component code: - -{% highlight python %} -from os import listdir - -mypath = '/data/in/tables' -onlyfiles = [f for f in listdir(mypath)] -print(onlyfiles) -{% endhighlight %} - -Since you are debugging, it is not wise to add this for all customers. Therefore you can commit -the code and tag it with a **non-**[normal version tag](https://semver.org/#spec-item-2), for example, `0.0.7-test`. -Such a tag will be deployed as a Docker image, but it won't (automatically) update in the -Developer Portal. That means the previous tag will be still used for all jobs. However, you can -run the new tag manually, using the [Run Job API call](https://api.keboola.com/?service=job-queue#post-/jobs). Again, feel free to use our [collection](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). - -If you added the above debug code to the component `keboola-test.ex-docs-tutorial` and -tagged the release `0.0.7-test`, you can run the configuration `354678919` by issuing the -following API call: - - curl -X POST \ - https://queue.keboola.com/jobs \ - -H 'Content-Type: application/json' \ - -H 'X-StorageApi-Token: your-token' \ - -d '{ - "component": "keboola-test.ex-docs-tutorial", - "config": "354678919", - "tag": "0.0.7-test", - "mode": "run" - }' - -In the job detail -- under **Parameters & Results**, , you'll see that a specific tag was requested. In the job events, you can then -see that it was indeed used and that the script printed out all files in the `/data/in/tables/` folder. - -{: .image-popup} -![Screenshot -- Image Results](/extend/component/tutorial/debug-4.png) - -## Summary -You can find more information about running components in the corresponding part of the [documentation](/extend/component/running/). - -This concludes our development tutorial on the most important aspects of creating Keboola components. However, our platform offers a -lot more; we encourage you to read about other features in our documentation: - -- exchanging data in [data folders](/extend/common-interface/folders/) -- [manifest files](/extend/common-interface/manifest-files/) -- [OAuth support](/extend/common-interface/oauth/) -- or general information about the [common interface](/extend/common-interface/) -- [deployment settings](/extend/component/deployment/) -- [UI settings](/extend/component/ui-options/) -- [Running components locally](/extend/component/running/) +--- +title: Debugging +permalink: /extend/component/tutorial/debugging/ +redirect_to: https://help.keboola.com/extend/component/tutorial/debugging/ +--- diff --git a/extend/component/tutorial/gh-build-1.png b/extend/component/tutorial/gh-build-1.png deleted file mode 100644 index c090286c..00000000 Binary files a/extend/component/tutorial/gh-build-1.png and /dev/null differ diff --git a/extend/component/tutorial/gh-build-2.png b/extend/component/tutorial/gh-build-2.png deleted file mode 100644 index fceafd00..00000000 Binary files a/extend/component/tutorial/gh-build-2.png and /dev/null differ diff --git a/extend/component/tutorial/github-repository.png b/extend/component/tutorial/github-repository.png deleted file mode 100644 index 92cadbf6..00000000 Binary files a/extend/component/tutorial/github-repository.png and /dev/null differ diff --git a/extend/component/tutorial/hello-world.png b/extend/component/tutorial/hello-world.png deleted file mode 100644 index fe2568dc..00000000 Binary files a/extend/component/tutorial/hello-world.png and /dev/null differ diff --git a/extend/component/tutorial/index.md b/extend/component/tutorial/index.md index 91dfcb5f..b2cc9c3d 100644 --- a/extend/component/tutorial/index.md +++ b/extend/component/tutorial/index.md @@ -1,267 +1,8 @@ ---- -title: Component Quick Start -permalink: /extend/component/tutorial/ -redirect_from: - - /extend/docker/quick-start/ - - /extend/custom-science/quick-start/ ---- - -* TOC -{:toc} - -In this tutorial, you will create a simple "Hello, World!" component which runs in -Keboola. - -A Keboola component is a Docker image. When a job runs, the [Job Queue](/extend/job-queue/) executes the image's -`ENTRYPOINT` or `CMD`, so your image must define one of them. Component images are stored in Amazon AWS ECR -provisioned by the [Developer Portal](https://components.keboola.com/) --- the only supported registry --- and are -pushed there by the [deploy script](/extend/component/deployment/) in your CI pipeline. - -## Before You Start -You need to have a computer with working [Docker](https://www.docker.com/why-docker) to develop the Keboola component code. -To be able to create new components, you also need to have an account in the [Keboola Developer Portal](https://components.keboola.com/), -which manages the list of components available in Keboola. - -The Developer Portal uses different credentials than Keboola. [Creating an account](https://components.keboola.com/auth/create-account) is free; it requires a working email address -(to which a confirmation email will be sent) and a mobile phone for a mandatory two-factor authorization. - -When you log in to the Developer Portal, you have to join a **vendor** --- an organization of -developers. Every Keboola component has to have a vendor assigned. If you join an existing vendor, a -vendor administrator has to approve your request. If you do not work for a company, create a -vendor with your name (even a single developer has to be assigned to a vendor). When you join or create a vendor -you should also receive access to a development Keboola project. - -{: .image-popup} -![Screenshot -- Join a vendor](/extend/component/tutorial/join-vendor.png) - -In order to create a **new vendor**, a Keboola administrator has to approve your request, and you will -receive a [development project](/#development-project) in Keboola. In addition to that, you need to provide us -with a channel for receiving internal errors from your components. Anything supported -by [Papertrail notifications](https://help.papertrailapp.com/kb/how-it-works/alerts#supported-services) -is available, though e-mail or a Slack channel is most commonly used. - -When you are confirmed as a member of a vendor, you may proceed to creating your own component. -The example component is written in the Python language, but no knowledge of Python is required. -Before you continue with this tutorial, make sure you - -- can log in to the [Developer Portal](https://components.keboola.com/). -- can log in to one of the Keboola [stacks](https://help.keboola.com/overview/#stacks) -- have a [Github](https://github.com/) account. - -*Note: Even though the tutorial assumes using [GitHub](https://github.com/) + [Travis](https://travis-ci.org/) services, they are not required for extending Keboola. -We use them because we like them the most. The [deployment documentation](/extend/component/deployment/) shows how to configure, -for example, [Bitbucket](/extend/component/deployment/#bitbucket-integration) and [GitLab](/extend/component/deployment/#gitlab-integration) -integrations.* - -## Creating Component -To add a component, use the **Add a component** button on the main page, and fill in the component name and type: - -{: .image-popup} -![Screenshot -- Create component](/extend/component/tutorial/create-component-2.png) - -**Important:** Do **not** use the words 'extractor', 'writer', or 'application' in the component name. - -Choose the appropriate [component type](/extend/component/#component-types): - -- `extractor` -- brings data into Keboola -- `writer` -- sends data out of Keboola -- `transformation` -- does some transformation of the data, [read more](https://help.keboola.com/transformations/#new-transformations) -- `code pattern` -- generates code for transformation's component, [read more](/extend/component/code-patterns) -- `application` -- another arbitrary component - -The above does not mean technically that, for example, an extractor cannot send data out of Keboola -or an application cannot bring new data into Keboola. It is a matter of user perception, -so use your judgement to select the correct type. - -When you fill the form in, you will obtain a **component ID** (in the -form `vendor-id.component-name`, for instance, `keboola-test.ex-docs-tutorial`). Make a **note** of the ID. - -## Creating Deployment Account -To be able to deploy the component to Keboola, you will need **service credentials**. For security -reasons, we strongly advice against using your own credentials in any deployment service. To create -new deployment credentials, click the **Create a service account** button on the **Service accounts** page. - -{: .image-popup} -![Screenshot -- Create account](/extend/component/tutorial/service-account-1.png) - -Fill in a name (e.g., `ex_docs_tutorial_travis`) and description (e.g., `Travis deployment credentials`) and confirm: - -{: .image-popup} -![Screenshot -- Account details](/extend/component/tutorial/service-account-2.png) - -Take a note of the **username** and **password**. - -{: .image-popup} -![Screenshot -- Account credentials](/extend/component/tutorial/service-account-3.png) - -## Initializing Component -Once you have the **component ID** and the service account **username** and **password**, -you can create a component, the most convenient way is to use one of our templates. - -### Python - -**Prerequisite** - Install the cookiecutter tool: - -```bash -pip install cookiecutter -``` - -**Usage** - -1. Run ```bash -cookiecutter gh:keboola/cookiecutter-python-component``` -2. Fill in requested parameters: - - `template_variant` - Where is your empty repository (Github, Bitbucket) - - `repository_url` - (OPT) URL of your repository, if filled in the template git repository is initialised and remote set to your repository. - - `component_name` - Name of your component - - `repository_folder_name` - (OPT) Name of the destination folder. By default normalized component name. - - `component_short_description` - (OPT) short description that will be pushed to Developer Portal. May be edited in `component_config/component_short_description.md` later. - - `component_long_description` - (OPT) long description that will be pushed to Developer Portal. May be edited in `component_config/component_long_description.md` later. -3. Set up CI environment variables (see the [CI Setup section](## CI Setup)) -4. Navigate to newly created folder and run `git push`. The CI pipeline (action) should be now executed. If you add a tag to the commit, component will be pushed to your Developer Portal. -5. Modify the code in `src/component.py` and `src/configuration.py` as you like. - - You can set the configuration parameters in `data/config.json` - - You can execute the component via normal local environment without docker installed. - - Set any additional dependencies for your project in `requirements.txt` - - More information on the template [here](https://bitbucket.org/kds_consulting_team/kbc-python-template/src/master/README.md) - -## CI Setup - - Bitbucket: Enable [pipelines](https://confluence.atlassian.com/bitbucket/get-started-with-bitbucket-pipelines-792298921.html) in the repository. - - For Github: Check that the [workflows are enabled](https://docs.github.com/en/actions/managing-workflow-runs/disabling-and-enabling-a-workflow). - The actions are present in `.github/workflows/` folder. - - Set `KBC_DEVELOPERPORTAL_APP` env variable (dev portal app id) - - In case it is not set on the account level, set also other required dev portal env variables: - - - `KBC_DEVELOPERPORTAL_PASSWORD` - service account password - - `KBC_DEVELOPERPORTAL_USERNAME` - service account username - - `KBC_DEVELOPERPORTAL_VENDOR` - dev portal vendor - - `KBC_STORAGE_TOKEN` - (optional) in case you wish to run KBC automated tests - - -[https://github.com/keboola/cookiecutter-python-component]() - -### PHP, R or just CI pipeline for GitHub or Bitbucket - -Create an empty [Github](https://github.com/) repository. The name of the repository is -arbitrary, but using the component is probably a good idea to avoid confusion. - -{: .image-popup} -![Screenshot -- Github Repository](/extend/component/tutorial/github-repository.png) - -Checkout the repository on your local computer and execute the following from the command line: - - docker run -i -t --volume=/path/to/repository/:/code/ quay.io/keboola/component-generator - -Replace `/path/to/repository/` with an absolute local path to your empty repository. Follow -the on-screen instructions: - -{: .image-popup} -![Screenshot -- Component Generator](/extend/component/tutorial/component-generator.png) - -When done, you will have an initialized repository with a "Hello, World!" component. -In the above example, we chose the `simple-python` template, which contains the following: - -- template.md -- description of the template files -- main.py -- a "Hello, World!" Python script -- Dockerfile -- a Dockerfile defining the environment in which the script runs -- deploy.sh -- a Bash script to deploy the component to Keboola - -For Travis CI template contain: -- .travis.yml -- a configuration file for [Travis CI](https://docs.travis-ci.com/) to automate the deployment - -For GitHub Actions CI template contain: -- .github/workflows/push.yml -- a configuration file for [GitHub Actions CI](https://github.com/features/actions) to automate the deploy - -## Building Component -When done exploring, push to the repository. -This will automatically trigger a build on the Travis or GitHub Actions services; you can view the build -progress by visiting the provided link. In fact, two builds will be triggered: one -for the `master` branch, and one for the `0.1.0` tag. - -Travis: - -{: .image-popup} -![Screenshot -- Travis Build](/extend/component/tutorial/travis-build-1.png) - -GitHub Actions: - -{: .image-popup} -![Screenshot -- GitHub Actions Build](/extend/component/tutorial/gh-build-1.png) - -We are more interested in the latter because that is going to trigger the deployment to Keboola. - -Travis: - -{: .image-popup} -![Screenshot -- Travis Build Detail](/extend/component/tutorial/travis-build-2.png) - -GitHub Actions: - -{: .image-popup} -![Screenshot -- GitHub Actions Build Detail](/extend/component/tutorial/gh-build-2.png) - -If the deployment passes without errors, the component will become available in Keboola. You -can verify that in the component details (action Edit) in the Developer Portal: - -{: .image-popup} -![Screenshot -- Component Deployed](/extend/component/tutorial/component-deployed.png) - -This means that the component deployment is fully automated. If you change the component -source code, all you need to do is push the changes to the git repository and tag them -with the [normal version tag](https://semver.org/#spec-item-2). - -## Running Component -Once the component is deployed, it becomes available in Keboola. Note that it -takes **up to 5 minutes** for the changes to propagate to all Keboola instances. After that, -you can configure the component by visiting the following URL: - - https://connection.keboola.com/admin/projects/{DEFINED PROJECT_ID}/extractors/{YOUR COMPONENT_ID} - -On this URL, you can create a configuration and run it without any settings. - -{: .image-popup} -![Screenshot -- Component Configuration](/extend/component/tutorial/component-configuration.png) - -And you should see the "Hello, World" message in the events: - -{: .image-popup} -![Screenshot -- Component Events](/extend/component/tutorial/hello-world.png) - -When you create a component, it will have assigned a memory limit of **256MB** and -run timeout of **1 hour**. If you need to change those limits, please -[contact our support](mailto:support@keboola.com). - -## Component Repository -The component repository is a crucial part of the component setting because it -actually defines what Docker image will be used when running the component. -We offer free hosting of your Docker images in the **[Amazon Container Registry (AWS ECR)](https://aws.amazon.com/ecr/)** under our own account. -All repositories in AWS ECR are private. When you create your component using the method shown above, we -have just provisioned you with the Docker image hosting and you do not need to worry about it any more. - -AWS ECR is the **only supported registry** for Keboola components, providing reliable hosting for all Docker images. - -## Summary -You have just created your own Keboola component. Although it does not do much, it shows the easiest path -to bringing your own application logic to Keboola. You can now continue with other parts of the tutorial: - - - using [input](/extend/component/tutorial/input-mapping/) and - [output mapping](/extend/component/tutorial/output-mapping/) - - using [configuration parameters](/extend/component/tutorial/configuration/) - - [configuring a processor](/extend/component/tutorial/processors/) - - [debugging a component](/extend/component/tutorial/debugging/) - - [implementation notes](/extend/component/implementation/) for specific languages - -Although you rarely need all of the above parts (e.g., you do not need input mapping when building an extractor), -we suggest you go through all of them to gain a general overview of the available options. You can also read -all the details in the respective parts of the documentation: - -- general information about the [common interface](/extend/common-interface/) -- exchanging data in [data folders](/extend/common-interface/folders/) -- [manifest files](/extend/common-interface/manifest-files/) -- [OAuth support](/extend/common-interface/oauth/) -- [deployment settings](/extend/component/deployment/) (including [Bitbucket integration](/extend/component/deployment/#bitbucket-integration)) -- [UI settings](/extend/component/ui-options/) - +--- +title: Component Quick Start +permalink: /extend/component/tutorial/ +redirect_from: + - /extend/docker/quick-start/ + - /extend/custom-science/quick-start/ +redirect_to: https://help.keboola.com/extend/component/tutorial/ +--- diff --git a/extend/component/tutorial/input-mapping-1.png b/extend/component/tutorial/input-mapping-1.png deleted file mode 100644 index c06fb9a0..00000000 Binary files a/extend/component/tutorial/input-mapping-1.png and /dev/null differ diff --git a/extend/component/tutorial/input-mapping-2.png b/extend/component/tutorial/input-mapping-2.png deleted file mode 100644 index cffc1977..00000000 Binary files a/extend/component/tutorial/input-mapping-2.png and /dev/null differ diff --git a/extend/component/tutorial/input-mapping-3.png b/extend/component/tutorial/input-mapping-3.png deleted file mode 100644 index 8b45d117..00000000 Binary files a/extend/component/tutorial/input-mapping-3.png and /dev/null differ diff --git a/extend/component/tutorial/input-mapping-4.png b/extend/component/tutorial/input-mapping-4.png deleted file mode 100644 index b034be60..00000000 Binary files a/extend/component/tutorial/input-mapping-4.png and /dev/null differ diff --git a/extend/component/tutorial/input-mapping.md b/extend/component/tutorial/input-mapping.md index 867aa0b8..c5587872 100644 --- a/extend/component/tutorial/input-mapping.md +++ b/extend/component/tutorial/input-mapping.md @@ -1,89 +1,5 @@ --- title: Input Mapping permalink: /extend/component/tutorial/input-mapping/ +redirect_to: https://help.keboola.com/extend/component/tutorial/input-mapping/ --- - -* TOC -{:toc} - -In this part of the [tutorial](/extend/component/tutorial/), you will see how to use **input mapping**. Input mapping -defines what data your component receives from the end users' project. A component never -operates on data in a project's [Storage](https://help.keboola.com/storage/), -it always receives a copy of the selected data. - -Input mapping is therefore used when your component needs to read data from the customer -project -- these are typically **writers** and **applications**. Implementing the input mapping requires three steps: - -- adding input mapping to the component UI -- adding input mapping to the component source code -- verifying - -## Adding Input Mapping to UI -Adding the input mapping to the component UI is very simple. In the [Developer Portal](https://components.keboola.com/), edit the component -and add `genericDockerUI-tableInput` or `genericDockerUI-fileInput` (or both) [UI options](/extend/component/ui-options/). - -{: .image-popup} -![Screenshot -- Input Mapping Configuration](/extend/component/tutorial/input-mapping-1.png) - -Save the changes, and the corresponding UI elements will appear in your component configuration. Keep in mind that the changes -take up to 5 minutes to propagate to all Keboola instances. - -## Modifying Source Code -You also need to modify the source code so that it works with the input data. The data from input mapping will be available in the -[`/data/in/tables/`](/extend/common-interface/folders/#folder-dataintables) and -[`/data/in/files`](/extend/common-interface/folders/#folder-datainfiles) folders when the component is -[run](/extend/job-queue/). This is almost identical to writing -[Transformation code](https://help.keboola.com/transformations/) -- the only difference is that you need to -use absolute paths (or the [`KBC_DATADIR`](/extend/common-interface/environment/#environment-variables) environment variable). - -Therefore you can modify the [example component](/extend/component/tutorial/) code to, for instance, the -[following one](https://github.com/keboola/ex-docs-tutorial), which works with table input mapping. - -{% highlight python %} -import csv - -print("Hello, World! from python") - -csvlt = '\n' -csvdel = ',' -csvquo = '"' -with open('/data/in/tables/source.csv', mode='rt', encoding='utf-8') as in_file: - lazy_lines = (line.replace('\0', '') for line in in_file) - reader = csv.DictReader(lazy_lines, lineterminator=csvlt, delimiter=csvdel, quotechar=csvquo) - for row in reader: - # do something - print("The first row is ", row) - # we don't want to print the entire file to the output - exit() - -{% endhighlight %} - -Commit and push the code in your repository, and tag it with a [normal version tag](https://semver.org/#spec-item-2). -This will trigger a [build on Travis CI](https://docs.travis-ci.com/) and automatically -deploy the new version into Keboola. Keep in mind that after the deployment, it may take up to 5 minutes for the update to propagate to all Keboola instances. - -## Verifying -If you configured the UI correctly, you should see the corresponding control in the component -configuration page. - -{: .image-popup} -![Screenshot -- Component Configuration](/extend/component/tutorial/input-mapping-2.png) - -Add a **New Table Input**, select an arbitrary table from the project, and make sure to set **Destination** to `source.csv` -so that the final path matches `/data/in/tables/source.csv`, which is what you are expecting in the code. - -{: .image-popup} -![Screenshot -- Input Mapping Detail](/extend/component/tutorial/input-mapping-3.png) - -Now run the component and you should see the proper message in job events. - -{: .image-popup} -![Screenshot -- Input Mapping Events](/extend/component/tutorial/input-mapping-4.png) - -## Summary -Your component can now successfully read a provided table. You can learn -more about other input mapping options in the -[data folder specification](/extend/common-interface/folders/). Also, the -[manifest files](/extend/common-interface/manifest-files/) contain metadata which -might be useful in your component. The next part of the tutorial will show you -how to create [output mapping](/extend/component/tutorial/output-mapping/). diff --git a/extend/component/tutorial/join-vendor.png b/extend/component/tutorial/join-vendor.png deleted file mode 100644 index ac5bc139..00000000 Binary files a/extend/component/tutorial/join-vendor.png and /dev/null differ diff --git a/extend/component/tutorial/output-mapping-1.png b/extend/component/tutorial/output-mapping-1.png deleted file mode 100644 index d8ae0bd0..00000000 Binary files a/extend/component/tutorial/output-mapping-1.png and /dev/null differ diff --git a/extend/component/tutorial/output-mapping-2.png b/extend/component/tutorial/output-mapping-2.png deleted file mode 100644 index ad9c1089..00000000 Binary files a/extend/component/tutorial/output-mapping-2.png and /dev/null differ diff --git a/extend/component/tutorial/output-mapping.md b/extend/component/tutorial/output-mapping.md index f2847616..d7fdcce7 100644 --- a/extend/component/tutorial/output-mapping.md +++ b/extend/component/tutorial/output-mapping.md @@ -1,104 +1,5 @@ --- title: Output Mapping permalink: /extend/component/tutorial/output-mapping/ +redirect_to: https://help.keboola.com/extend/component/tutorial/output-mapping/ --- - -* TOC -{:toc} - -In this part of the [tutorial](/extend/component/tutorial/), you will see how to use **output mapping**. -Similarly to [input mapping](/extend/component/tutorial/input-mapping/), the output mapping -defines what data your component produces in the end users' project. A component cannot directly -write data to the project [Storage](https://help.keboola.com/storage/); -the produced data are stored in the end users' project when the component [finishes](/extend/job-queue/). - -Output mapping is therefore used when your application needs to send data to the customer -project -- these are typically **extractors** and **applications**. Implementing the input mapping requires three steps: - -- turning on the default bucket option (or adding output mapping to the component UI) -- adding output mapping to the component source code -- verifying - -Unlike [input mapping](/extend/component/tutorial/input-mapping/), the output mapping is fully optional. That means that -it can be configured similarly to the input mapping (using the [`genericDockerUI-tableOutput`](/extend/component/ui-options/) flag) or -using the [Default bucket](/extend/common-interface/folders/#default-bucket) option. The latter means that all tables produced in the -`/data/out/tables/` folder will be uploaded to a [Storage bucket](https://help.keboola.com/storage/buckets/) with a generated name. -This is almost identical to writing -[Transformation code](https://help.keboola.com/transformations/) -- the only difference is that you need to -use absolute paths (or the [`KBC_DATADIR`](/extend/common-interface/environment/#environment-variables) environment variable). - -## Configuring Default Bucket -Using the default bucket is the preferred option as it allows simpler configuration of the component. -To enable the default bucket for your component, simply go the [Developer Portal](https://components.keboola.com/) and tick the respective -checkbox. You also need to select the [stage](https://help.keboola.com/storage/buckets/) of the bucket. We recommend using `in` for -extractors and `out` for other components. - -{: .image-popup} -![Screenshot -- Default Bucket Configuration](/extend/component/tutorial/output-mapping-1.png) - -## Modifying Source Code -You can modify the [sample component](/extend/component/tutorial/) code to, for example, the one below. -The code takes a single arbitrary table on input and produces two tables -- one with -even rows and one with odd rows. - -{% highlight python %} -import csv - -print("Hello world from python") - -csvlt = '\n' -csvdel = ',' -csvquo = '"' -with open('/data/in/tables/source.csv', mode='rt', encoding='utf-8') as in_file, \ - open('/data/out/tables/odd.csv', mode='wt', encoding='utf-8') as odd_file, \ - open('/data/out/tables/even.csv', mode='wt', encoding='utf-8') as even_file: - lazy_lines = (line.replace('\0', '') for line in in_file) - reader = csv.DictReader(lazy_lines, lineterminator=csvlt, delimiter=csvdel, - quotechar=csvquo) - - even_writer = csv.DictWriter(odd_file, fieldnames=reader.fieldnames, - lineterminator=csvlt, delimiter=csvdel, - quotechar=csvquo) - even_writer.writeheader() - - odd_writer = csv.DictWriter(even_file, fieldnames=reader.fieldnames, - lineterminator=csvlt, delimiter=csvdel, - quotechar=csvquo) - odd_writer.writeheader() - i = 0 - for row in reader: - if i % 2 == 0: - even_writer.writerow(row) - else: - odd_writer.writerow(row) - i = i + 1 -{% endhighlight %} - -This script reads a CSV file line by line and checks whether it is odd or even. -Finally, the result is written to either `odd.csv` or `even.csv`. - -Commit and push the code in your repository and tag it with a [normal version tag](https://semver.org/#spec-item-2). -This will trigger a [build on Travis CI](/extend/component/tutorial/#building-the-component) and automatically -deploy the new version into Keboola. Keep in mind that after the deployment, it may take up to 5 minutes for the update to propagate to all Keboola instances. - -## Verifying -If you configured the default bucket option, nothing changes in the UI. If you followed the -[previous part of our tutorial](/extend/component/tutorial/input-mapping/#verifying), you don't have to do anything. -Just run the component and you should see that two tables were produced. - -{: .image-popup} -![Screenshot -- Output Mapping result](/extend/component/tutorial/output-mapping-2.png) - -If you happen to see the following error message: - - CSV file 'odd' file name is not a valid table identifier, either set output mapping for 'odd.csv' or make sure that the file name is a valid Storage table identifier. - -It means that you have not set the default bucket properly (thus no bucket is generated for the component, and we don't know where to put 'odd'). - -## Summary -Your component can now successfully write tables to a Keboola project. You can read -more about other output mapping options in the -[data folder specification](/extend/common-interface/folders/). Also, the -[manifest files](/extend/common-interface/manifest-files/) contain metadata which -you might want to set (e.g., primary key). The next part of the tutorial will show you -how to work with [configuration parameters](/extend/component/tutorial/configuration/). diff --git a/extend/component/tutorial/processors-1.png b/extend/component/tutorial/processors-1.png deleted file mode 100644 index 3af23105..00000000 Binary files a/extend/component/tutorial/processors-1.png and /dev/null differ diff --git a/extend/component/tutorial/processors.md b/extend/component/tutorial/processors.md index d6393d3b..45ea5109 100644 --- a/extend/component/tutorial/processors.md +++ b/extend/component/tutorial/processors.md @@ -1,114 +1,5 @@ ---- -title: Processors -permalink: /extend/component/tutorial/processors/ ---- - -* TOC -{:toc} - -[Processors](/extend/component/processors/) are an optional part of a component configuration. -While they are **not at all necessary** in the development of new components for Keboola, we think that you -should know about them; they can save you a lot of time in some cases. -To get a list of currently available processors, see the -[official component list](https://components.keboola.com/components). - -## Configuration -To be able to configure the processors in the Keboola UI, go to the -[Developer Portal](https://components.keboola.com/) and add the UI -flag `genericDockerUI-processors` to your component. You'll then see -a new UI element in the component configuration in Keboola: - -{: .image-popup} -![Screenshot -- Processors Empty](/extend/component/tutorial/processors-1.png) - -Taking the [example component](/extend/component/tutorial/), you might want to use the -**Add Row Number Column** processor in your component to add a sequential number to every -row of the table imported into Keboola. From the -[processor documentation](https://github.com/keboola/processor-add-row-number-column/blob/master/README.md#usage) -you can see that the processor is configured as: - -{% highlight json %} -{ - "definition": { - "component": "keboola.processor-add-row-number-column" - } -} -{% endhighlight %} - -You want the processor to execute on the output of your component; it means that the -above should be inserted into the `after` (after your component runs) section: - -{% highlight json %} -{ - "before": [], - "after": [ - { - "definition": { - "component": "keboola.processor-add-row-number-column" - } - } - ] -} -{% endhighlight %} - -## Chaining Processors -If you run the above configuration, you'll receive an error: - - Table odd.csv does not have a manifest file. - -This is expected because the [Add Row Number Column processor documentation](https://github.com/keboola/processor-add-row-number-column/blob/master/README.md#prerequisites) -clearly states that the processed CSV files must have -[manifests](/extend/common-interface/manifest-files/) and not headers. Since the example component is very simple and does -not generate manifests (or header-less CSV files), you have to add other processors to do that -for you: - -{% highlight json %} -{ - "before": [], - "after": [ - { - "definition": { - "component": "keboola.processor-create-manifest" - }, - "parameters": { - "columns_from": "header" - } - }, - { - "definition": { - "component": "keboola.processor-skip-lines" - }, - "parameters": { - "lines": 1 - } - }, - { - "definition": { - "component": "keboola.processor-add-row-number-column" - } - } - ] -} -{% endhighlight %} - -The `after` configuration is an array of three processors. The first one creates -[manifest files](/extend/common-interface/manifest-files/) for whatever data files were produced by your component. The manifest -files will contain a header read from the data files. The second processor removes the header -from the data files. The third processor adds the row number column. - -## Summary -Configuring processors is not part of the component development. However, processors -allow the end user to customize the input to the component and the output from it. That means -that they can be used to implement specific customer requests while keeping the component -code general. - -Choosing whether to implement a specific feature as a processor or as part of your -component may be difficult. A processor might be a good solution if the feature is - -- simple (one operation, contains no internal logic), -- optional (not all end users are interested in it), or -- universal (it is always applied to all input/output or none). - -Keep in mind, however, that the processors must be configured by the end user. You can read more about -[processors](/extend/component/processors/) or continue with the next part of the tutorial; -it will show you some [debugging tips](/extend/component/tutorial/debugging/). +--- +title: Processors +permalink: /extend/component/tutorial/processors/ +redirect_to: https://help.keboola.com/extend/component/tutorial/processors/ +--- diff --git a/extend/component/tutorial/service-account-1.png b/extend/component/tutorial/service-account-1.png deleted file mode 100644 index 9bc4dfdd..00000000 Binary files a/extend/component/tutorial/service-account-1.png and /dev/null differ diff --git a/extend/component/tutorial/service-account-2.png b/extend/component/tutorial/service-account-2.png deleted file mode 100644 index fb5e0cf9..00000000 Binary files a/extend/component/tutorial/service-account-2.png and /dev/null differ diff --git a/extend/component/tutorial/service-account-3.png b/extend/component/tutorial/service-account-3.png deleted file mode 100644 index ce143da5..00000000 Binary files a/extend/component/tutorial/service-account-3.png and /dev/null differ diff --git a/extend/component/tutorial/travis-build-1.png b/extend/component/tutorial/travis-build-1.png deleted file mode 100644 index 24cb081a..00000000 Binary files a/extend/component/tutorial/travis-build-1.png and /dev/null differ diff --git a/extend/component/tutorial/travis-build-2.png b/extend/component/tutorial/travis-build-2.png deleted file mode 100644 index 92d95d27..00000000 Binary files a/extend/component/tutorial/travis-build-2.png and /dev/null differ diff --git a/extend/component/ui-options/auth-0.png b/extend/component/ui-options/auth-0.png deleted file mode 100644 index 5fccc1bc..00000000 Binary files a/extend/component/ui-options/auth-0.png and /dev/null differ diff --git a/extend/component/ui-options/auth-1.png b/extend/component/ui-options/auth-1.png deleted file mode 100644 index a0fd4be8..00000000 Binary files a/extend/component/ui-options/auth-1.png and /dev/null differ diff --git a/extend/component/ui-options/configuration-schema-1.png b/extend/component/ui-options/configuration-schema-1.png deleted file mode 100644 index 0b727232..00000000 Binary files a/extend/component/ui-options/configuration-schema-1.png and /dev/null differ diff --git a/extend/component/ui-options/configuration-schema-2.png b/extend/component/ui-options/configuration-schema-2.png deleted file mode 100644 index e53c81a3..00000000 Binary files a/extend/component/ui-options/configuration-schema-2.png and /dev/null differ diff --git a/extend/component/ui-options/configuration-schema.md b/extend/component/ui-options/configuration-schema.md index c9c951f1..f7f649fa 100644 --- a/extend/component/ui-options/configuration-schema.md +++ b/extend/component/ui-options/configuration-schema.md @@ -3,280 +3,5 @@ title: Configuration Schema permalink: /extend/component/ui-options/configuration-schema/ redirect_from: - /extend/registration/configuration-schema/ +redirect_to: https://help.keboola.com/extend/component/ui-options/configuration-schema/ --- - -The default input for a component configuration is a JSON text area. - -{: .image-popup} -![Generic configuration screenshot](/extend/component/ui-options/configuration.png) - -If you define a JSON schema, we are able to display a nice form and -let the user to fill the JSON using a set of defined inputs. - -{: .image-popup} -![Configuration schema](/extend/component/ui-options/configuration-schema-1.png) - -Using the configuration schema also allows us to validate the user input on frontend. - -## Creating Schema - -JSON schemas are well documented on the [json-schema.org](https://json-schema.org/) website. - -We use [RJSF (React JSON Schema Form)](https://rjsf-team.github.io/react-jsonschema-form/) for rendering schemas -into forms. The schema supports standard JSON Schema properties as well as custom extensions documented -in [UI Element Examples](/extend/component/ui-options/configuration-schema/examples/) and -[Sync Action Examples](/extend/component/ui-options/configuration-schema/sync-action-examples/). - -### Supported Formats - -The following `format` values are supported in property definitions. The **Type** column shows the underlying JSON Schema `type`; italicized entries are descriptive notes, not literal `type` values. - -| Format | Type | Description | -|---|---|---| -| `password` | string | Masked password input with show/hide toggle | -| `textarea` | string | Multi-line text area | -| `editor` | string/object | CodeMirror code editor (JSON, SQL, Python, etc.) | -| `date` | string | Date picker input | -| `checkbox` | boolean | Checkbox toggle | -| `radio` | string | Radio button group (requires `enum`) | -| `trim` | string | Standard text input with automatic whitespace trimming | -| `grid` / `grid-strict` | object | Responsive grid layout for grouped fields | -| `tabs` / `tabs-top` / `categories` | object | Tabbed layout for grouped fields | -| `table` | array | Editable table for arrays of objects | -| `info` | *any JSON Schema type* | Static informational alert (uses `title` as message; Keboola UI extension) | -| `ssh-editor` | object | SSH key/form editor | -| `sync-action` | *Keboola UI button widget* | Action button triggering a sync action (not a JSON Schema `type`) | -| `test-connection` | *Keboola UI button widget* | Connection test button (not a JSON Schema `type`) | - -### Supported Options - -The following `options` keys can be used in property definitions: - -| Option | Description | -|---|---| -| `options.async` | Dynamic option loading via sync actions. See [Sync Action Examples](/extend/component/ui-options/configuration-schema/sync-action-examples/). | -| `options.dependencies` | Conditional field visibility based on other field values. See [Dynamic Options](/extend/component/ui-options/configuration-schema/examples/#conditionally-showing-fields-based-on-selection). | -| `options.tags` | Enable tag-style input for multi-select arrays | -| `options.creatable` | Allow user-created values in select dropdowns | -| `options.tooltip` | Help text displayed as a tooltip icon next to the field label. Supports Markdown syntax. | -| `options.documentation` | Documentation link rendered as a book icon next to the field label. Value: `{ "link": "https://...", "tooltip": "optional hover text" }` | -| `options.enum_titles` | Display labels for `enum` values | -| `options.hidden` | Hide the field from the UI | -| `options.collapsed` | Start object sections in collapsed state | -| `options.disable_collapse` | Prevent collapsing of object sections | -| `options.enabled` | Set to `false` to disable a field | -| `options.grid_columns` | Number of grid columns (1–12) in `grid`/`grid-strict` layouts | -| `options.grid_break` | Force a new row in grid layouts | -| `options.editor` | CodeMirror editor options: `mode`, `lineNumbers`, `lint`, `input_height` | -| `options.encryption_hint` | Set to `false` to hide the "properties prefixed with `#` will be encrypted" note under a JSON-mode `editor` field. See [Codemirror Editor](/extend/component/ui-options/configuration-schema/examples/#codemirror-jsonsqlpython-editor). | -| `options.input_height` | Height for textarea fields (e.g., `"100px"`) | -| `options.inputAttributes` | HTML input attributes (e.g., `placeholder`) | -| `options.only_keys` | SSH editor variant showing only key fields | -| `options.disable_array_add` | Disable adding items to arrays | -| `options.disable_array_delete` | Disable removing items from arrays | -| `options.disable_array_reorder` | Disable reordering items in arrays | - -## Developing and Previewing Schemas - -When building or iterating on a configuration schema, you need a way to see how your JSON schema renders as a form. -The recommended approach is to use the **live schema editor built into the Keboola UI**. - -### Live Schema Editor in Keboola UI (Recommended) - -The Keboola UI includes a built-in live schema editor that lets you edit a JSON schema and immediately see the rendered form -side by side. This is the fastest way to iterate on your schema during development. - -**How to use it:** - -1. Open any component configuration page in the Keboola UI (e.g., `https://connection.keboola.com/admin/projects//components//`). -2. Press **Ctrl+D** (or **Alt+D** / **Option+D** on Mac) while on the configuration page. -3. Use the **arrow button** in the panel toolbar to move the editor to the left or right side of the screen. -4. Click the **checkmark button** to apply the schema override for the current browser session. -5. Press **Ctrl+D** again (or click the **close button**) to hide the editor. - -This editor uses the same [RJSF](https://rjsf-team.github.io/react-jsonschema-form/) rendering library as the production UI, -so what you see in the editor is exactly what end users will see. The override is local to your browser session — it does not -change the schema registered in the Developer Portal. - -### Generating Schemas with AI Kit - -You can also use the **build-component-ui** skill from the -[Keboola AI Kit](https://github.com/keboola/ai-kit/tree/main/plugins/component-developer/skills/build-component-ui) -to generate or refine configuration schemas using AI. The skill is an expert agent specialized in: - -- Designing `configSchema.json` and `configRowSchema.json` structures -- Using `options.dependencies` for conditional field visibility -- Choosing the right UI elements and formats -- Setting up sync actions for dynamic dropdowns and test connection buttons - -To use it, install the [Keboola AI Kit](https://github.com/keboola/ai-kit) as a Claude Code plugin and invoke the -`@build-component-ui` agent. Describe your component's configuration needs and it will produce a complete schema -following Keboola conventions and best practices. - -### Example -Let's assume your component accepts the following configuration: - -{% highlight json %} - -{ - "username": "foo", - "#password": "baz", - "dateFrom": "yesterday", - "dateTo": "today" -} - -{% endhighlight %} - -This looks like an appropriate form: - -{: .image-popup} -![Configuration form](/extend/component/ui-options/form.png) - -The form above can be created using this JSON Schema: - -{% highlight json %} -{ - "title": "Parameters", - "type": "object", - "required": [ - "dateFrom", - "dateTo", - "username", - "#password" - ], - "properties": { - "username": { - "title": "Username", - "type": "string", - "minLength": 1, - "default": "", - "propertyOrder": 1 - }, - "#password": { - "title": "Password", - "type": "string", - "format": "password", - "minLength": 1, - "default": "", - "propertyOrder": 2 - }, - "dateFrom": { - "title": "Date from", - "type": "string", - "description": "Any date accepted by strtotime (https://www.php.net/manual/en/function.strtotime.php) function", - "minLength": 1, - "default": "", - "propertyOrder": 3 - }, - "dateTo": { - "title": "Date to", - "type": "string", - "description": "Any date accepted by strtotime (https://www.php.net/manual/en/function.strtotime.php) function", - "minLength": 1, - "default": "", - "propertyOrder": 4 - } - } -} -{% endhighlight %} - -### Links Example - -***Note:** The `links` feature is deprecated. Use `options.documentation` instead (see [Supported Options](#supported-options)).* - -If you want to provide links to external resources, use `options.documentation` to add a clickable documentation icon next to the field label: - -{% highlight json %} -{ - "dateFrom": { - "title": "Date from", - "type": "string", - "description": "Any date accepted by the strtotime function", - "options": { - "documentation": { - "link": "https://www.php.net/manual/en/function.strtotime.php", - "tooltip": "strtotime Documentation" - } - } - } -} -{% endhighlight %} - -#### Legacy links feature (deprecated) - -The old `links` property is **no longer supported**. If you have existing schemas using it, migrate to `options.documentation`: - -{% highlight json %} -{ - "title": "Parameters", - "type": "object", - "required": [ - "dateFrom", - "dateTo", - "username", - "#password" - ], - "properties": { - "username": { - "title": "Username", - "type": "string", - "minLength": 1, - "default": "", - "propertyOrder": 1 - }, - "#password": { - "title": "Password", - "type": "string", - "format": "password", - "minLength": 1, - "default": "", - "propertyOrder": 2 - }, - "dateFrom": { - "title": "Date from", - "type": "string", - "description": "Any date accepted by the strtotime function", - "minLength": 1, - "default": "", - "propertyOrder": 3, - "links": [ - { - "rel": "strtotime Documentation", - "href": "https://www.php.net/manual/en/function.strtotime.php" - } - ] - }, - "dateTo": { - "title": "Date to", - "type": "string", - "description": "Any date accepted by the strtotime function", - "minLength": 1, - "default": "", - "propertyOrder": 4, - "links": [ - { - "rel": "strtotime Documentation", - "href": "https://www.php.net/manual/en/function.strtotime.php" - } - ] - } - } -} -{% endhighlight %} - -Which renders like this: - -{: .image-popup} -![Configuration Schema with links](/extend/component/ui-options/configuration-schema-2.png) - -### Legacy Features - -The following features are still supported for backwards compatibility but have preferred alternatives: - -- **`enumSource` / `watch`** — Dynamic enum population based on other field values. For new schemas, prefer `options.async` with sync actions instead. See [Sync Action Examples](/extend/component/ui-options/configuration-schema/sync-action-examples/#autoload). - -### Deprecated Features - -The following features from the legacy JSON Editor library are **no longer supported**: - -- **`links`** — Clickable links on field descriptions. Use `options.documentation` with `link` and optional `tooltip` instead. diff --git a/extend/component/ui-options/default-configuration/developer-portal-01.png b/extend/component/ui-options/default-configuration/developer-portal-01.png deleted file mode 100644 index 0145e2c7..00000000 Binary files a/extend/component/ui-options/default-configuration/developer-portal-01.png and /dev/null differ diff --git a/extend/component/ui-options/default-configuration/index.md b/extend/component/ui-options/default-configuration/index.md index e6effeff..8b1dbb6c 100644 --- a/extend/component/ui-options/default-configuration/index.md +++ b/extend/component/ui-options/default-configuration/index.md @@ -1,107 +1,5 @@ --- title: Default Configuration permalink: /extend/component/ui-options/default-configuration/ +redirect_to: https://help.keboola.com/extend/component/ui-options/default-configuration/ --- - -* TOC -{:toc} - -To make configuring a component easier for users, you can provide a default configuration for it. -This can be done by defining either *Default Configuration* or *Default Row Configuration* -in [Keboola Developer Portal](https://components.keboola.com/): - -{: .image-popup} -![Setting Default Configuration in Developer Portal](/extend/component/ui-options/default-configuration/developer-portal-01.png) - -## Default Configuration - -If you define *Default Configuration* for your component, all new configurations -will be created with this configuration. - -Let's assume your component has the following JSON set as *Default Configuration*: - -{% highlight json %} - -{ - "parameters": { - "debug": true - } -} - -{% endhighlight %} - -Once the new configuration is created, the configuration JSON will look like this: - -{% highlight json %} - -{ - "changeDescription": "Configuration created", - "configuration": { - "parameters": { - "debug": true - } - }, - "created": "2021-06-17T12:14:50+0200", - "description": "", - "id": "719629255", - "name": "My DynamoDB Data Source", - "state": {}, - "version": 1 -} - -{% endhighlight %} - -## Default Row Configuration - -The same also applies to rows. If a component has *Default Row Configuration* -defined, e.g., like this: - -{% highlight json %} - -{ - "parameters": { - "verbose": false - } -} - -{% endhighlight %} - -Then adding a new row to the configuration will use the default values too, and the final configuration -will look like this: - -{% highlight json %} - -{ - "changeDescription": "Configuration created", - "configuration": { - "parameters": { - "debug": true - } - }, - "created": "2021-06-17T12:14:50+0200", - "description": "", - "id": "719629255", - "name": "My DynamoDB Data Source", - "state": {}, - "version": 2, - "rowsSortOrder": [], - "rows": [ - { - "id": "30645", - "name": "Test", - "description": "", - "isDisabled": false, - "version": 1, - "created": "2021-06-17T12:22:19+0200", - "changeDescription": "Create query Test", - "state": {}, - "configuration": { - "parameters": { - "verbose": false - } - } - } - ] -} - -{% endhighlight %} diff --git a/extend/component/ui-options/file-input-0.png b/extend/component/ui-options/file-input-0.png deleted file mode 100644 index 0d8cd558..00000000 Binary files a/extend/component/ui-options/file-input-0.png and /dev/null differ diff --git a/extend/component/ui-options/file-input-1.png b/extend/component/ui-options/file-input-1.png deleted file mode 100644 index a7dffe99..00000000 Binary files a/extend/component/ui-options/file-input-1.png and /dev/null differ diff --git a/extend/component/ui-options/file-input-2.png b/extend/component/ui-options/file-input-2.png deleted file mode 100644 index 602a0df3..00000000 Binary files a/extend/component/ui-options/file-input-2.png and /dev/null differ diff --git a/extend/component/ui-options/file-output-0.png b/extend/component/ui-options/file-output-0.png deleted file mode 100644 index 599c95b9..00000000 Binary files a/extend/component/ui-options/file-output-0.png and /dev/null differ diff --git a/extend/component/ui-options/file-output-1.png b/extend/component/ui-options/file-output-1.png deleted file mode 100644 index aaf8ff94..00000000 Binary files a/extend/component/ui-options/file-output-1.png and /dev/null differ diff --git a/extend/component/ui-options/file-output-2.png b/extend/component/ui-options/file-output-2.png deleted file mode 100644 index b4ae88ba..00000000 Binary files a/extend/component/ui-options/file-output-2.png and /dev/null differ diff --git a/extend/component/ui-options/form.png b/extend/component/ui-options/form.png deleted file mode 100644 index ad77b25b..00000000 Binary files a/extend/component/ui-options/form.png and /dev/null differ diff --git a/extend/component/ui-options/index.md b/extend/component/ui-options/index.md index 89bfb0d5..af5d9dd2 100644 --- a/extend/component/ui-options/index.md +++ b/extend/component/ui-options/index.md @@ -1,136 +1,5 @@ ---- -title: UI Options -permalink: /extend/component/ui-options/ ---- - -* TOC -{:toc} - -Each component needs to specify how its user interface (UI) will look. Otherwise the component cannot -be configured via the UI (it can still be configured using the API though). - -The most basic UI configuration is `genericDockerUI`. The generic UI will always show a text field for entering the -component configuration in JSON format. Other parts of the UI are turned on using other [UI options](/extend/component/ui-options/) -(for example, `genericDockerUI-tableInput`, `genericDockerUI-tableOutput`). All of the [UI options](/extend/component/ui-options/) may be combined freely. - -## genericDockerUI -This provides a basic text area for setting component parameters as a JSON; the text area has -JSON validation and syntax highlighting. - -{: .image-popup} -![Generic configuration screenshot](/extend/component/ui-options/configuration.png) - -The configuration provided in this input is available in the `parameters` section of the -[configuration file](/extend/common-interface/config-file/#configuration-file-structure). -Defining a [configuration schema](/extend/component/ui-options/configuration-schema/) will replace the JSON text area with a form. - -## genericDockerUI-tableInput -This flag provides a UI for setting the table input [mapping](https://help.keboola.com/transformations/mappings/). -You can set the following options: - -- *Source* --- the name of the table in Storage -- Destination *file name* --- the name of the .csv file passed to the component -- *Columns* --- select only some columns of the source table -- *Days* --- load only rows modified in the specified number of days; useful for incremental loads; set to 0 to load all data -- *Data filter* --- a simple filter for selecting specified rows only - -{: .image-popup} -![Table input screenshot](/extend/component/ui-options/table-input-0.png) - -{: .image-popup} -![Table input detail screenshot](/extend/component/ui-options/table-input-1.png) - -{: .image-popup} -![Table input result screenshot](/extend/component/ui-options/table-input-2.png) - -The configuration provided in this input is available in the `storage.input` section of the -[configuration file](/extend/common-interface/config-file/#configuration-file-structure). - -## genericDockerUI-tableOutput -This flag provides a UI for setting the table output [mapping](https://help.keboola.com/transformations/mappings/). This UI part **should not be used** -if the component is using the [default bucket](/extend/common-interface/folders/#default-bucket) setting. - -With this UI, you can set the following options: - -- *Source* --- the name of the .csv file retrieved from the component -- *Destination* --- the name of the table in Storage, the destination bucket should exist already -- *Incremental* --- if checked, the loaded data will be appended to the contents of the destination table -- *Primary key* --- set the primary key for your destination table --- multiple columns are allowed -- *Delete rows* --- delete some rows from the destination table using a simple filter - -{: .image-popup} -![Table output screenshot](/extend/component/ui-options/table-output-0.png) - -{: .image-popup} -![Table output detail screenshot](/extend/component/ui-options/table-output-1.png) - -{: .image-popup} -![Table output result screenshot](/extend/component/ui-options/table-output-2.png) - -The configuration provided in this input is available in the `storage.output` section of the -[configuration file](/extend/common-interface/config-file/#configuration-file-structure). - -## genericDockerUI-processors -This flag provides a UI for the [processor configuration](/extend/component/processors/). -It offers a basic text area for setting the processors and their parameters as a JSON; the text area has -JSON validation and syntax highlighting. - -{: .image-popup} -![Processors screenshot](/extend/component/ui-options/processors.png) - -## genericDockerUI-fileInput -This flag provides a UI for setting the file input mapping. You can set the following options: - -- *File tags* --- select files by the file tags listed in **File Uploads** -- *Query* --- [ElasticSearch query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) -to select files from **File Uploads** -- *Processed tags* --- used for [incremental processing](/extend/common-interface/config-file/#incremental-processing) - -{: .image-popup} -![File input screenshot](/extend/component/ui-options/file-input-0.png) - -{: .image-popup} -![File input detail screenshot](/extend/component/ui-options/file-input-1.png) - -{: .image-popup} -![File input result screenshot](/extend/component/ui-options/file-input-2.png) - -The configuration provided in this input is available in the `storage.input` section of the -[configuration file](/extend/common-interface/config-file/#configuration-file-structure). - -## genericDockerUI-fileOutput -This flag provides a UI for setting the file output mapping. You can set the following options: - -- *Source* --- the name of the file produced by the component -- *File tags* --- the file tags assigned to the produced file -- *Is public* --- the file is accessible to anyone knowing its URL -- *Is permanent* --- the file will not be deleted after 15 days - -{: .image-popup} -![File output screenshot](/extend/component/ui-options/file-output-0.png) - -{: .image-popup} -![File output detail screenshot](/extend/component/ui-options/file-output-1.png) - -{: .image-popup} -![File output result screenshot](/extend/component/ui-options/file-output-2.png) - -The configuration provided in this input is available in the `storage.output` section of the -[configuration file](/extend/common-interface/config-file/#configuration-file-structure). - -## genericDockerUI-authorization -This flag provides a UI for setting [OAuth2 Authorization](/extend/common-interface/oauth/). However, to -actually activate OAuth for your component, you have to [contact our support](mailto:support@keboola.com). - -{: .image-popup} -![Authorization screenshot](/extend/component/ui-options/auth-0.png) - -{: .image-popup} -![Authorization detail screenshot](/extend/component/ui-options/auth-1.png) - -The configuration provided in this input is available in the `authorization` section of the -[configuration file](/extend/common-interface/config-file/#configuration-file-structure). - -## genericTemplatesUI -This flag is used to provide a UI for components based on the [Generic Extractor](/extend/generic-extractor/). It allows the end user to select a -[Generic Extractor template](/extend/generic-extractor/publish/). +--- +title: UI Options +permalink: /extend/component/ui-options/ +redirect_to: https://help.keboola.com/extend/component/ui-options/ +--- diff --git a/extend/component/ui-options/processors.png b/extend/component/ui-options/processors.png deleted file mode 100644 index ac598b85..00000000 Binary files a/extend/component/ui-options/processors.png and /dev/null differ diff --git a/extend/component/ui-options/table-input-0.png b/extend/component/ui-options/table-input-0.png deleted file mode 100644 index e9888978..00000000 Binary files a/extend/component/ui-options/table-input-0.png and /dev/null differ diff --git a/extend/component/ui-options/table-input-1.png b/extend/component/ui-options/table-input-1.png deleted file mode 100644 index 03e4ffe3..00000000 Binary files a/extend/component/ui-options/table-input-1.png and /dev/null differ diff --git a/extend/component/ui-options/table-input-2.png b/extend/component/ui-options/table-input-2.png deleted file mode 100644 index 28c1d73a..00000000 Binary files a/extend/component/ui-options/table-input-2.png and /dev/null differ diff --git a/extend/component/ui-options/table-output-0.png b/extend/component/ui-options/table-output-0.png deleted file mode 100644 index 188d2c3d..00000000 Binary files a/extend/component/ui-options/table-output-0.png and /dev/null differ diff --git a/extend/component/ui-options/table-output-1.png b/extend/component/ui-options/table-output-1.png deleted file mode 100644 index 48e0e4aa..00000000 Binary files a/extend/component/ui-options/table-output-1.png and /dev/null differ diff --git a/extend/component/ui-options/table-output-2.png b/extend/component/ui-options/table-output-2.png deleted file mode 100644 index 343c5c00..00000000 Binary files a/extend/component/ui-options/table-output-2.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/checkbox.png b/extend/component/ui-options/ui-examples/checkbox.png deleted file mode 100644 index cc5b5f0c..00000000 Binary files a/extend/component/ui-options/ui-examples/checkbox.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/code_editor.png b/extend/component/ui-options/ui-examples/code_editor.png deleted file mode 100644 index c48ad8c7..00000000 Binary files a/extend/component/ui-options/ui-examples/code_editor.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/configuration-schema-examples.md b/extend/component/ui-options/ui-examples/configuration-schema-examples.md index cb6afe04..0b399dd4 100644 --- a/extend/component/ui-options/ui-examples/configuration-schema-examples.md +++ b/extend/component/ui-options/ui-examples/configuration-schema-examples.md @@ -1,703 +1,5 @@ --- title: UI Element Examples permalink: /extend/component/ui-options/configuration-schema/examples/ - +redirect_to: https://help.keboola.com/extend/component/ui-options/configuration-schema/examples/ --- - -* TOC -{:toc} - -[JSON schema](https://json-schema.org/) allows for design of some advanced UI elements. Some of these are often reused -in many components. This page contains a list of the commonly used UI elements and some advanced tips for UI design. - -### API Token & Secret Values - -Always prefix private parameters like passwords with `#` character. These will be automatically hashed and hidden from the view. -Use a textual input field with `"format":"password"` in the JsonSchema for these values to hide the content also during the typing. - -```json -{ - "#api_token": { - "type": "string", - "title": "API token", - "format": "password", - "propertyOrder": 1 - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![Password Screenshot](/extend/component/ui-options/ui-examples/password.png) - - -### Checkboxes - -```json -{ - "campaigns": { - "type": "boolean", - "title": "Download Campaigns", - "default": false, - "format": "checkbox", - "propertyOrder": 30 - }, - "segments": { - "type": "boolean", - "title": "Download Segments", - "default": false, - "format": "checkbox", - "propertyOrder": 40 - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![Checkboxes screenshot](/extend/component/ui-options/ui-examples/checkbox.png) - - -### Tooltips - -Additional description with optional links - -```json -{ - "test_tooltip": { - "type": "string", - "title": "Example tooltip", - "options": { - "tooltip": "custom tooltip, default is Open documentation" - }, - "description": "Test value.", - "propertyOrder": 1 - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![Tooltip screenshot](/extend/component/ui-options/ui-examples/tooltip_normal.png) - - -### Multi Selection - -```json -{ - "types": { - "type": "array", - "title": "Types", - "description": "Activity types", - "items": { - "enum": [ - "page", - "event", - "attribute_change", - "failed_attribute_change", - "stripe_event", - "drafted_email", - "failed_email", - "dropped_email", - "sent_email", - "spammed_email", - "bounced_email", - "delivered_email", - "triggered_email", - "opened_email" - ], - "type": "string" - }, - "format": "select", - "uniqueItems": true, - "propertyOrder": 360 - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![multiselect](/extend/component/ui-options/ui-examples/multi_select.png) - - -### Creatable Multi Select - -Multi select with user creatable values - -```json -{ - "test_creatable_multi_select": { - "propertyOrder": 50, - "type": "array", - "items": { - "type": "string" - }, - "format": "select", - "options": { - "tags": true - }, - "description": "Multi-select element with no enum => user creates arbitrary values. Comma-separated values are supported.", - "uniqueItems": true - } -} -``` - -The above code will create the following element: - -{: .image-popup} -![multiselect](/extend/component/ui-options/ui-examples/creatable_select.gif) - - -### Codemirror (json/sql/python..) Editor - -Allow inject Codemirror editor to a JSON schema based UI. -Allowed options: mode, placeholder, autofocus, lineNumbers lint -Available modes: `text/x-sfsql`, `text/x-sql`, `text/x-plsql`, `text/x-python`, `text/x-julia`, `text/x-rsrc`, `application/json` -JSON mode supports encryption. Default mode is `application/json` . You should set type base on mode (string or object). - -A JSON-mode editor stores the parsed JSON, so keys prefixed with `#` inside the field are -[encrypted](/overview/encryption/) on save. The UI states this in a note below the editor. If the field does not hold -Keboola configuration (a vendor query filter, a request payload template), that note advertises something the component -cannot use --- the platform would encrypt the key and the component would receive the ciphertext. Hide the note with -`options.encryption_hint: false`: - -```json -{ - "filters": { - "type": "object", - "title": "Filters", - "format": "editor", - "options": { - "editor": { - "mode": "application/json" - }, - "encryption_hint": false - } - } -} -``` - -The option only affects the note. Encryption itself is unchanged: a `#`-prefixed key is still encrypted on save. - -**JsonSchema examples:** - -```json -{ - "token": { - "type": "object", - "format": "editor" - } -} -``` - -```json -{ - "sql": { - "type": "string", - "format": "editor", - "options": { - "editor": { - "mode": "text/x-sql" - } - } - } -} -``` - -```json -{ - "json_properties": { - "type": "object", - "title": "User Parameters", - "format": "editor", - "default": { - "debug": false - }, - "options": { - "editor": { - "lint": true, - "mode": "application/json", - "lineNumbers": true, - "input_height": "100px" - } - }, - "description": "User parameters accessible, the result will be injected in standard data/config.json parameters property as in any other component", - "propertyOrder": 1 - } -} -``` - -The above code will create the following element: - -{: .image-popup} -![multiselect](/extend/component/ui-options/ui-examples/code_editor.png) - - -### Trimmed String - -Works only for simple string inputs. Value is trimmed before save. - -**JsonSchema example:** - -```json -"token": { - "type": "string", - "format": "trim" -} -``` - - - -### Date Range - -When a date range is applicable, it should be bounded by two parameters: *From Date* and *To Date*. -These should be the text fields that accept a particular date in a specified format or a string defining a relative -interval in [strtotime](https://www.php.net/manual/en/function.strtotime.php) manner. - -**Tip:** A convenient Python function for parsing such values and conversion to date can be found in the Keboola python-utils library -([parse_datetime_interval](https://github.com/keboola/python-utils#getting-converted-date-period-from-string)). - -```json -{ - "date_from": { - "propertyOrder": 5, - "type": "string", - "title": "From date [inclusive]", - "description": "Date from. Date in YYYY-MM-DD format or a string i.e. 5 days ago, 1 month ago, yesterday, etc. If left empty, all records are downloaded." - }, - "date_to": { - "propertyOrder": 7, - "type": "string", - "title": "To date [exclusive]", - "default": "now", - "description": "Date to. Date in YYYY-MM-DD format or a string i.e. 5 days ago, 1 month ago, yesterday, etc. If left empty, all records are downloaded." - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![Date period](/extend/component/ui-options/ui-examples/det_period.png) - - -### Loading Options (Incremental vs Full) - -This may be combined in [loading options block](/extend/component/ui-options/configuration-schema/examples/#example-1---object-blocks-loading-options). - -```json -{ - "incremental_output": { - "type": "number", - "enum": [ - 0, - 1 - ], - "options": { - "enum_titles": [ - "Full Load", - "Incremental Update" - ] - }, - "default": 1, - "title": "Load type", - "description": "If set to Incremental update, the result tables will be updated based on the primary key. Full load overwrites the destination table each time. NOTE: If you wish to remove deleted records, this needs to be set to Full load and the Period from attribute empty.", - "propertyOrder": 365 - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![Date period](/extend/component/ui-options/ui-examples/load_type.png) - -### Visual Separation of Sections - -It often happens that the configuration can be split into multiple sections. -It is advisable to split these visually using JSON Schema objects or arrays to achieve it using the generic UI. - -#### Example 1 – Object blocks (loading options) - -Loading options block: - -```json -{ - "loading_options": { - "type": "object", - "title": "Loading Options", - "propertyOrder": 400, - "format": "grid", - "required": [ - "incremental_output", - "date_since", - "date_to" - ], - "properties": { - "date_since": { - "type": "string", - "title": "Period from date [including].", - "default": "1 week ago", - "description": " Date in YYYY-MM-DD format or dateparser string, i.e., 5 days ago, 1 month ago, yesterday, etc. If left empty, all records are downloaded.", - "propertyOrder": 300 - }, - "date_to": { - "type": "string", - "title": "Period to date [excluding].", - "default": "now", - "description": " Date in YYYY-MM-DD format or dateparser string, i.e., 5 days ago, 1 month ago, yesterday, etc. If left empty, all records are downloaded.", - "propertyOrder": 400 - }, - "incremental_output": { - "type": "number", - "enum": [ - 0, - 1 - ], - "options": { - "enum_titles": [ - "Full Load", - "Incremental Update" - ] - }, - "default": 1, - "title": "Load type", - "description": "If set to Incremental update, the result tables will be updated based on the primary key. Full load overwrites the destination table each time. NOTE: If you wish to remove deleted records, this needs to be set to Full load and the Period from attribute empty.", - "propertyOrder": 450 - } - } - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![loading options block](/extend/component/ui-options/ui-examples/loading_options_block.png) - -#### Example 2 – Optional blocks using arrays - -Create an array with parameter `"maxItems": 1` to create optional blocks. - -```json -{ - "customers": { - "type": "array", - "title": "Customers", - "description": "Download Customers.", - "propertyOrder": 4000, - "maxItems": 1, - "items": { - "type": "object", - "title": "Setup", - "required": [ - "filters", - "attributes" - ], - "properties": { - "filters": { - "type": "string", - "title": "Filter", - "description": "Optional JSON filter, as defined in https://customer.io/docs/api-triggered-data-format#general-syntax. Example value: {\"and\":[{\"segment\":{\"id\":7}},{\"segment\":{\"id\":5}}]} If left empty, all users are downloaded", - "format": "textarea", - "propertyOrder": 1 - }, - "attributes": { - "type": "string", - "title": "Attributes", - "format": "textarea", - "options": { - "input_height": "100px" - }, - "description": "Comma-separated list of required customer attributes. Each customer may have different set of columns, this is to limit only to attributes you need. All attributes are downloaded if left empty.", - "uniqueItems": true, - "propertyOrder": 700 - } - } - } - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![optional block](/extend/component/ui-options/ui-examples/optional_block_array.gif) - - -### Conditionally Showing Fields Based on Selection - -In some cases, additional options should only be shown when a related setting is enabled, e.g., when a particular report feature is turned on. -JSON Schema UI options allow you to control field visibility based on the values of other fields. -This may be useful in configuration rows, where each row can expose extra options depending on how a user configures it. - -This can be achieved via `options.dependencies`. When the dependency conditions are met, the field is shown; otherwise it is hidden. - - -```json -{ - "type": "object", - "title": "extractor configuration", - "required": [ - "download_attachments" - - ], - "properties": { - "download_attachments": { - "type": "boolean", - "format": "checkbox", - "title": "Download Attachments", - "description": "When set to true, also the attachments will be downloaded. By default into the File Storage. Use processors to control the behaviour.", - "default": false, - "propertyOrder": 300 - }, - "attachment_pattern": { - "type": "string", - "title": "Attachment Pattern", - "description": "Regex pattern to filter particular attachments, e.g., to retrieve only pdf file types use: .+\\.pdf If left empty, all attachments are downloaded.", - "default": ".+\\.csv", - "options": { - "dependencies": { - "download_attachments": true - } - }, - "propertyOrder": 400 - } - } -} -``` - -The above code will create the following user interface: - -{: .image-popup} -![dynamic selection](/extend/component/ui-options/ui-examples/dynamic_sel.gif) - -You can also react on multiple array values or on multiple elements at the same time.: - -```json -"options": { - "dependencies": { - "endpoint": [ - "analytics_data_breakdown_by_content", "analytics_data_breakdown_by_object" - ], - "filtered": false - } -} -``` - -You can also reference nested fields using dot notation with a `root.` prefix: - -```json -"options": { - "dependencies": { - "root.credentials.#api_token": "" - } -} -``` - -This shows the field only when the `#api_token` field inside the `credentials` object is empty. - - -### Tooltips and Documentation Links - -Use `options.tooltip` to add a help icon next to the field label. The tooltip content supports **Markdown** syntax: - -```json -{ - "api_key": { - "type": "string", - "title": "API Key", - "options": { - "tooltip": "Your API key from the [dashboard](https://example.com/dashboard).\n\n**Note:** Keep this value secret." - } - } -} -``` - -Use `options.documentation` to add a clickable documentation link icon: - -```json -{ - "query": { - "type": "string", - "title": "SQL Query", - "options": { - "documentation": { - "link": "https://docs.example.com/sql-reference", - "tooltip": "Open SQL reference" - } - } - } -} -``` - -### Radio Buttons - -Use `format: "radio"` with an `enum` to render radio buttons instead of a dropdown: - -```json -{ - "output_format": { - "type": "string", - "title": "Output Format", - "enum": ["csv", "json", "parquet"], - "format": "radio", - "default": "csv", - "propertyOrder": 1 - } -} -``` - - -### Date Picker - -Use `format: "date"` for a date input with a calendar picker. Values are stored as `YYYY-MM-DD` strings: - -```json -{ - "start_date": { - "type": "string", - "title": "Start Date", - "format": "date", - "propertyOrder": 1 - } -} -``` - - -### Grid Layout - -Use `format: "grid"` or `format: "grid-strict"` on an object to arrange its properties in a responsive 12-column grid. -Each property can specify `options.grid_columns` (1–12) to control its width. Use `options.grid_break` to force a new row: - -```json -{ - "connection": { - "type": "object", - "title": "Connection", - "format": "grid-strict", - "properties": { - "host": { - "type": "string", - "title": "Hostname", - "propertyOrder": 1, - "options": { "grid_columns": 8 } - }, - "port": { - "type": "integer", - "title": "Port", - "propertyOrder": 2, - "options": { "grid_columns": 4 } - }, - "database": { - "type": "string", - "title": "Database", - "propertyOrder": 3, - "options": { "grid_break": true } - } - } - } -} -``` - - -### Collapsible Sections - -Use `options.collapsed` on an object to make it collapsible. Set `options.disable_collapse` to prevent the user from collapsing it: - -```json -{ - "advanced": { - "type": "object", - "title": "Advanced Settings", - "options": { - "collapsed": true - }, - "properties": { - "timeout": { "type": "integer", "title": "Timeout" } - } - } -} -``` - - -### Disabled Fields - -Use `options.enabled: false` to render a field as read-only: - -```json -{ - "locked_field": { - "type": "string", - "title": "Locked Value", - "default": "Cannot be changed", - "options": { - "enabled": false - } - } -} -``` - - -### Info Blocks - -Use `format: "info"` to display a static informational message. The `title` property is used as the message text: - -```json -{ - "notice": { - "title": "This component requires an API token to function.", - "format": "info", - "propertyOrder": 1 - } -} -``` - -#### Alert Type - -By default, info blocks render as an informational (blue) alert. Use `options.alert_type` to change the visual style. Supported values: `"info"` (default), `"warning"`, `"error"`, `"success"`. - -```json -{ - "missing_token_warning": { - "title": "WARNING: API token is required!", - "format": "info", - "options": { - "alert_type": "warning" - } - } -} -``` - -```json -{ - "config_error": { - "title": "Configuration is invalid.", - "description": "Please check the required fields below.", - "format": "info", - "options": { - "alert_type": "error" - } - } -} -``` - -Info blocks can be combined with `options.dependencies` to show warnings conditionally: - -```json -{ - "missing_token_warning": { - "title": "WARNING: API token is required!", - "format": "info", - "options": { - "alert_type": "warning", - "dependencies": { - "root.credentials.#api_token": "" - } - } - } -} -``` diff --git a/extend/component/ui-options/ui-examples/creatable_select.gif b/extend/component/ui-options/ui-examples/creatable_select.gif deleted file mode 100644 index 18860877..00000000 Binary files a/extend/component/ui-options/ui-examples/creatable_select.gif and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/det_period.png b/extend/component/ui-options/ui-examples/det_period.png deleted file mode 100644 index 9d9c52b5..00000000 Binary files a/extend/component/ui-options/ui-examples/det_period.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/dynamic_dropdown_multi.gif b/extend/component/ui-options/ui-examples/dynamic_dropdown_multi.gif deleted file mode 100644 index 699725d5..00000000 Binary files a/extend/component/ui-options/ui-examples/dynamic_dropdown_multi.gif and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/dynamic_sel.gif b/extend/component/ui-options/ui-examples/dynamic_sel.gif deleted file mode 100644 index 5e207a05..00000000 Binary files a/extend/component/ui-options/ui-examples/dynamic_sel.gif and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/generic-button.gif b/extend/component/ui-options/ui-examples/generic-button.gif deleted file mode 100644 index b1c4fe45..00000000 Binary files a/extend/component/ui-options/ui-examples/generic-button.gif and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/load_type.png b/extend/component/ui-options/ui-examples/load_type.png deleted file mode 100644 index b462eecf..00000000 Binary files a/extend/component/ui-options/ui-examples/load_type.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/loading_options_block.png b/extend/component/ui-options/ui-examples/loading_options_block.png deleted file mode 100644 index 3ef882f4..00000000 Binary files a/extend/component/ui-options/ui-examples/loading_options_block.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/multi_select.png b/extend/component/ui-options/ui-examples/multi_select.png deleted file mode 100644 index 934f920d..00000000 Binary files a/extend/component/ui-options/ui-examples/multi_select.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/optional_block_array.gif b/extend/component/ui-options/ui-examples/optional_block_array.gif deleted file mode 100644 index e78adaed..00000000 Binary files a/extend/component/ui-options/ui-examples/optional_block_array.gif and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/password.png b/extend/component/ui-options/ui-examples/password.png deleted file mode 100644 index da72d705..00000000 Binary files a/extend/component/ui-options/ui-examples/password.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/single-drop.gif b/extend/component/ui-options/ui-examples/single-drop.gif deleted file mode 100644 index f2a33161..00000000 Binary files a/extend/component/ui-options/ui-examples/single-drop.gif and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/status_icons.png b/extend/component/ui-options/ui-examples/status_icons.png deleted file mode 100644 index f725c487..00000000 Binary files a/extend/component/ui-options/ui-examples/status_icons.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/sync-action-examples.md b/extend/component/ui-options/ui-examples/sync-action-examples.md index b6644fe5..9008583c 100644 --- a/extend/component/ui-options/ui-examples/sync-action-examples.md +++ b/extend/component/ui-options/ui-examples/sync-action-examples.md @@ -1,273 +1,5 @@ --- title: Sync Action UI Elements Examples permalink: /extend/component/ui-options/configuration-schema/sync-action-examples/ - +redirect_to: https://help.keboola.com/extend/component/ui-options/configuration-schema/sync-action-examples/ --- - -* TOC -{:toc} - -Some UI elements use [sync actions](https://developers.keboola.com/extend/common-interface/actions/) to get some values dynamically -from the component code. This section provides a list of the elements currently supported. - -Each element specifies the `action` attribute, which relates to the name of the sync action registered in the Developer Portal. - -***Note:** Support for these elements is also abstracted in the official [Python Component library](https://github.com/keboola/python-component#framework-support).* - -### Dynamically Loaded Dropdowns - -Drop-down lists (values and labels) can be loaded by the component sync action. - -The sync action code has to return the following stdout: - -``` -[ - { label: 'Joe', value: 'joe' }, - { label: 'Doe', value: 'doe }, - { label: 'Jane', value: 'jane' } -] -``` - -The `label` value is optional. - -When used in Python, you can use the [SelectElement](https://github.com/keboola/python-component#selectelement) class as a return value. - -#### Dynamically loaded multi select - -```json -{ - "test_columns": { - "type": "array", - "propertyOrder": 10, - "description": "Element loaded by an arbitrary sync action.", - "items": { - "enum": [], - "type": "string" - }, - "format": "select", - "options": { - "async": { - "label": "Re-load test columns", - "action": "testColumns" - } - }, - "uniqueItems": true - } -} -``` - -The above code will create the following element, which triggers an action named `testColumns`: - -{: .image-popup} -![Screenshot](/extend/component/ui-options/ui-examples/dynamic_dropdown_multi.gif) - - -#### Dynamically loaded single select - -```json -{ - "test_columns_single": { - "propertyOrder": 40, - "type": "string", - "description": "Element loaded by an arbitrary sync action (single).", - "enum": [], - "format": "select", - "options": { - "async": { - "label": "Re-load test columns", - "action": "testColumns" - } - } - } -} -``` - -The above code will create the following element, which triggers the `testColumns` action: - -{: .image-popup} -![ Screenshot](/extend/component/ui-options/ui-examples/single-drop.gif) - - - -### Generic Validation Button - -This button can be used to return feedback from the component. The output supports Markdown. - -Example use cases are query testing, testing connection, report validation, etc. - -The sync action code has to return the following stdout (JSON string): - -```json -{ - "message": "###This is display text. \n\n It can contain **Markdown** notation. ", - "type": "info", - "status": "success" -} -``` - -**Available options:** -- `type`: possible values: success, info, warning, error, table -- `status`: possible values: success, error - -#### Custom Icons - -Markdown supports special status icons that are rendered in Keboola UI style. The following icons are supported: - -``` -![success]() -![warning]() -![error]() -``` - -![status_icons.png](/extend/component/ui-options/ui-examples/status_icons.png) - - - -When used in Python, you can use the [ValidationResult](https://github.com/keboola/python-component#validationresult) class as a return value. - -**NOTE** If status: error is used the message will always be displayed as an error message. - -#### Example - -```json -{ - "validation_button": { - "type": "button", - "format": "sync-action", - "propertyOrder": 10, - "options": { - "async": { - "label": "Validate", - "action": "validate_report" - } - } - } -} -``` - -The above code will create the following element, which triggers the `validate_report` action: - -{: .image-popup} -![screenshot](/extend/component/ui-options/ui-examples/generic-button.gif) - - -### Test Connection - -This button can be used for simple connection tests. - -The sync action code has to return the following stdout (JSON string) or error (exit code >0): - -```json -{ - "status": "success" // this is required and will never be other value than "success" -} -``` - -The name of this sync action **always has to be `testConnection`.** - -When used in Python, the method does not need to return anything, or it can just throw an exception. - -#### Example - -```json -{ - "test_connection": { - "type": "button", - "format": "test-connection", - "propertyOrder": 30 - } -} -``` - -The above code will create the following element, which triggers the `testConnection` action: - -{: .image-popup} -![multiselect](/extend/component/ui-options/ui-examples/test_connection.png) - - -### Autoload - -All sync action types (buttons, select, and multi-selects) can automatically trigger the sync action if not defined on the UI page load. - -#### Example - -```json -{ - "endpoint": { - "type": "string", - "title": "Endpoint", - "description": "Use the sync action to get a list of available endpoints.", - "propertyOrder": 1, - "options": { - "async": { - "label": "List Endpoints", - "action": "listEndpoints", - "autoload": [] - } - }, - "items": { - "enum": [], - "type": "string" - }, - "enum": [] - } -} -``` - -Additionally, a watch element can be set in an autoload array, which, when defined or changed, will trigger the sync action. - -#### Example - -```json -{ - "field_names": { - "type": "array", - "format": "select", - "title": "Fields (optional)", - "description": "List of field names to be downloaded", - "propertyOrder": 2, - "options": { - "async": { - "label": "List Fields", - "action": "listFields", - "autoload": [ - "parameters.endpoint" - ] - } - }, - "items": { - "enum": [], - "type": "string" - }, - "uniqueItems": true - } -} -``` - -The autoload option also enables caching loaded values by default, which can be disabled by setting the `autoload.cache` to false. - -#### Example - -```json -{ - "endpoint": { - "type": "string", - "title": "Endpoint", - "description": "Use a sync action to get a list of available endpoints.", - "propertyOrder": 1, - "options": { - "async": { - "label": "List Endpoints", - "action": "listEndpoints", - "autoload": [], - "cache": false - } - }, - "items": { - "enum": [], - "type": "string" - }, - "enum": [] - } -} -``` diff --git a/extend/component/ui-options/ui-examples/test_connection.png b/extend/component/ui-options/ui-examples/test_connection.png deleted file mode 100644 index a1020d38..00000000 Binary files a/extend/component/ui-options/ui-examples/test_connection.png and /dev/null differ diff --git a/extend/component/ui-options/ui-examples/tooltip_normal.png b/extend/component/ui-options/ui-examples/tooltip_normal.png deleted file mode 100644 index 3b129520..00000000 Binary files a/extend/component/ui-options/ui-examples/tooltip_normal.png and /dev/null differ diff --git a/extend/generic-extractor/configuration/api/authentication/index.md b/extend/generic-extractor/configuration/api/authentication/index.md index 9694e8fe..74858e50 100644 --- a/extend/generic-extractor/configuration/api/authentication/index.md +++ b/extend/generic-extractor/configuration/api/authentication/index.md @@ -69,5 +69,5 @@ OAuth --- 1.0 and 2.0. They are completely incompatible (both the authentication Generic Extractor supports both [OAuth 1.0](/extend/generic-extractor/configuration/api/authentication/oauth10/) and [OAuth 2.0](/extend/generic-extractor/configuration/api/authentication/oauth20/). If you are developing a new component using Generic Extractor [templates](/extend/generic-extractor/publish/#submission) and want to use and test OAuth authentication, -[inject the necessary credentials](/extend/common-interface/oauth/#credentials-injection) simply by passing them +[inject the necessary credentials](https://help.keboola.com/extend/common-interface/oauth/#credentials-injection) simply by passing them in the configuration's `authorization` property. diff --git a/extend/generic-extractor/configuration/api/authentication/oauth10.md b/extend/generic-extractor/configuration/api/authentication/oauth10.md index 6ff1728f..a77285af 100644 --- a/extend/generic-extractor/configuration/api/authentication/oauth10.md +++ b/extend/generic-extractor/configuration/api/authentication/oauth10.md @@ -117,4 +117,4 @@ The full configuration is, e.g.: {% endhighlight %} See [example [EX102]](https://github.com/keboola/generic-extractor/tree/master/doc/examples/102-oauth1) or -learn [more about Keboola-OAuth integration](/extend/common-interface/oauth). +learn [more about Keboola-OAuth integration](https://help.keboola.com/extend/common-interface/oauth/). diff --git a/extend/generic-extractor/configuration/api/authentication/oauth20.md b/extend/generic-extractor/configuration/api/authentication/oauth20.md index 5185fcb8..464ecbf0 100644 --- a/extend/generic-extractor/configuration/api/authentication/oauth20.md +++ b/extend/generic-extractor/configuration/api/authentication/oauth20.md @@ -74,7 +74,7 @@ has three child properties: Note that the properties `appKey` and `#appSecret` must exist even if not used by the API; set them to empty strings. For more information about OAuth 2, see the [official documentation](https://oauth.net/2/) -or learn [more about Keboola-OAuth integration](/extend/common-interface/oauth). +or learn [more about Keboola-OAuth integration](https://help.keboola.com/extend/common-interface/oauth/). ## Configuration Parameters The following configuration parameters are supported for the `oauth20` authentication type: diff --git a/extend/generic-extractor/configuration/config/index.md b/extend/generic-extractor/configuration/config/index.md index 8e348173..35fc1cd1 100644 --- a/extend/generic-extractor/configuration/config/index.md +++ b/extend/generic-extractor/configuration/config/index.md @@ -58,7 +58,7 @@ includes configuring the HTTP method and parameters. The `jobs` configuration is The `outputBucket` option defines the name of the [Storage Bucket](https://help.keboola.com/storage/buckets/) in which the extracted tables will be stored. The configuration is **required** unless the extractor is [published](/extend/generic-extractor/publish/) as a standalone component with the -[Default Bucket](/extend/common-interface/folders/#default-bucket) option. +[Default Bucket](https://help.keboola.com/extend/common-interface/folders/#default-bucket) option. The following configuration will make Generic Extractor put all extracted tables in the `ge-tutorial` bucket (the names of the tables are defined by the [`dataType`](/extend/generic-extractor/configuration/config/jobs/#dataType) setting): diff --git a/extend/generic-extractor/incremental.md b/extend/generic-extractor/incremental.md index 475d2cbc..8ce38991 100644 --- a/extend/generic-extractor/incremental.md +++ b/extend/generic-extractor/incremental.md @@ -78,7 +78,7 @@ If you run it every hour, it will extract the data modified within the last hour Should one of the runs fail or be skipped for any reason, the extraction will pick up where it ended the last time it was successful. See [example [EX107]](https://github.com/keboola/generic-extractor/tree/master/doc/examples/107-incremental-load). -The last successful time is stored in the [configuration state](/extend/common-interface/config-file/#state-file). +The last successful time is stored in the [configuration state](https://help.keboola.com/extend/common-interface/config-file/#state-file). If for some reason you need to reset it, [update the configuration via API](https://api.keboola.com/?service=storage#put-/v2/storage/branch/-branchId-/components/-componentId-/configs/-configurationId-). diff --git a/extend/generic-extractor/publish.md b/extend/generic-extractor/publish.md index bc3c4547..a1b3619f 100644 --- a/extend/generic-extractor/publish.md +++ b/extend/generic-extractor/publish.md @@ -14,7 +14,7 @@ This enables sharing the API extractor between various projects and simplifies i ## Configuration Considerations Before converting your configuration to a universally available component, consider what values in the configuration should be provided by the end-user (typically authentication values). -Then design a [configuration schema](/extend/component/ui-options/configuration-schema/) for setting +Then design a [configuration schema](https://help.keboola.com/extend/component/ui-options/configuration-schema/) for setting those values. You can [test the schema online](http://jeremydorn.com/json-editor/) ([alternative](https://mozilla-services.github.io/react-jsonschema-form/)). The values obtained from the end user will be stored in the [`config` property](/extend/generic-extractor/configuration/config/). Modify your configuration to read those values from there. @@ -25,7 +25,7 @@ Also, try to make the extractor [work incrementally](/extend/generic-extractor/i if possible. ## Publishing -To publish your Generic Extractor configuration, you need to [create a new component](/extend/component/tutorial/) in +To publish your Generic Extractor configuration, you need to [create a new component](https://help.keboola.com/extend/component/tutorial/) in the [Developer Portal](https://components.keboola.com/). Choose an appropriate name and the type `extractor`. Once you have created the component, edit it, and fill in the following details: @@ -41,7 +41,7 @@ we recommend that you configure your component with a specific tag and update it in future Generic Extractor releases. Because the UI is assumed to be `genericTemplatesUI`, provide a -[**configuration schema**](/extend/component/ui-options/configuration-schema/) and +[**configuration schema**](https://help.keboola.com/extend/component/ui-options/configuration-schema/) and a **template** to be used in conjunction with the schema. Optionally, the template UI may also contain an interface to negotiate [OAuth authentication](/extend/generic-extractor/configuration/api/authentication/#oauth). An example of the template UI is shown in the picture below. @@ -62,7 +62,7 @@ section. Other sections, such as [`api`](/extend/generic-extractor/configuration You can review existing templates in their [GitHub repository](https://github.com/keboola/kbc-ui-templates/tree/master/resources). If you feel confident, you can send a pull request with your templates, otherwise submit it when requesting the -[publication of your component](/extend/publish/). +[publication of your component](https://help.keboola.com/extend/publish/). ## Example Let's say you have the following working API configuration @@ -132,7 +132,7 @@ For each of the values, create a parameter of the appropriate type: The parameter names are completely arbitrary. However, they must not conflict with existing configuration properties of [Generic Extractor](/extend/generic-extractor/configuration/config/) (e.g., `jobs`, `mappings`). -Now create a [configuration schema](/extend/component/ui-options/configuration-schema/) for the four parameters. +Now create a [configuration schema](https://help.keboola.com/extend/component/ui-options/configuration-schema/) for the four parameters. {% highlight json %} { diff --git a/extend/index.md b/extend/index.md index 6c07b112..5c005ab8 100644 --- a/extend/index.md +++ b/extend/index.md @@ -5,9 +5,15 @@ permalink: /extend/ As an open system consisting of many built-in, interoperating components, such as Storage or Extractors, [Keboola](/overview/) can be easily extended. -We encourage you to [**build your own components**](/extend/component/tutorial), whether for +We encourage you to [**build your own components**](https://help.keboola.com/extend/component/tutorial/), whether for your own use or to be offered to other Keboola users and customers. +
+Note: Component development documentation — components, the common interface, Job Queue and publishing — +now lives at help.keboola.com/extend/. The old URLs here redirect there. +Generic Extractor and Generic Writer are still documented on this site. +
+ * TOC {:toc} @@ -34,7 +40,7 @@ Depending on your role, extending Keboola offers various advantages: - Let Keboola be in charge of the billing. ## Component -A [component](/extend/component/) can be used as: +A [component](https://help.keboola.com/extend/component/) can be used as: - **Extractor**, allowing customers to get data from new sources. It only processes input tables from external sources (usually API). - **Application**, further enriching the data or adding value in new ways. It processes input tables stored as CSV files or database tables and generates result tables as CSV files or database tables. @@ -42,11 +48,11 @@ A [component](/extend/component/) can be used as: - **Writer**, pushing data into new systems and consumption methods. It does not generate any data in Keboola projects. - **Processor**, adjusting the inputs or outputs of other components. It has to be run together with one of the above components. -All components are run using [Job Queue](/extend/job-queue/), a service that takes +All components are run using [Job Queue](https://help.keboola.com/extend/job-queue/), a service that takes care of their authentication, starting, stopping, isolation, reading data from and writing it to Keboola Storage. They must adhere to the -[common interface](/extend/common-interface/). Creating components requires an elementary knowledge of [Docker](https://www.docker.com/why-docker). +[common interface](https://help.keboola.com/extend/common-interface/). Creating components requires an elementary knowledge of [Docker](https://www.docker.com/why-docker). They can be implemented in virtually any programming language and be fully customized and tailored to anyone's needs. -They also support OAuth authorization. To get started with building a component, see our [**tutorial**](/extend/component/tutorial/). +They also support OAuth authorization. To get started with building a component, see our [**tutorial**](https://help.keboola.com/extend/component/tutorial/). ## Generic Extractor [Generic Extractor](/extend/generic-extractor/) is a Keboola component acting like a diff --git a/extend/job-queue/docker-runner.svg b/extend/job-queue/docker-runner.svg deleted file mode 100644 index 2968f7a7..00000000 --- a/extend/job-queue/docker-runner.svg +++ /dev/null @@ -1,2 +0,0 @@ - -
Job Queue
Job Queue
Isolated Container
Isolated Container
Storage API
Storage API
Project Storage
Project Storage
Job Configuration
[Not supported by viewer]
Storage Input
Storage Input
Storage Output
Storage Output
State
State
Parameters
Parameters
Input Tables
Input Tables
/data/in/tables/
/data/in/files/
[Not supported by viewer]
Output Tables
Output Tables
Output Files
Output Files
/data/out/tables/
/data/out/files/
[Not supported by viewer]
/data/config.json
/data/in/state.json
[Not supported by viewer]
Component Definition
Component Definition
Component Configuration
Component Configuration
Pull Docker Image
Pull Docker Image
stdout / stderr
stdout / stderr
Run Container
Run Container
/data/out/state.json
<div>/data/out/state.json</div>
Events
Events
Input Files
Input Files


<div><br></div><div><br></div>
Job Result
[Not supported by viewer]
Status
Status
Exit code
Exit code \ No newline at end of file diff --git a/extend/job-queue/index.md b/extend/job-queue/index.md index 7fdfc55c..5ce08203 100644 --- a/extend/job-queue/index.md +++ b/extend/job-queue/index.md @@ -3,90 +3,5 @@ title: Job Queue permalink: /extend/job-queue/ redirect_from: - /extend/docker-runner/ +redirect_to: https://help.keboola.com/extend/job-queue/ --- - -* TOC -{:toc} - -Job Queue is a core Keboola Service, which -provides an interface for running Keboola components. Every component in Keboola is -represented by a Docker image. -Running a component means creating and executing an [asynchronous job](/integrate/jobs/). - -Developing functionality in [Docker](https://www.docker.com/) allows you to focus only on the application logic; all communication -with the [Storage API](https://api.keboola.com/?service=storage) will be handled by Job Queue. You can encapsulate any application into a Docker image -following a set of rules that will allow you to integrate the application into Keboola. - -There is a [predefined interface](/extend/common-interface/) with Job Queue, consisting -mainly of a [folder structure](/extend/common-interface/folders/) and a [serialized configuration file](/extend/common-interface/config-file/). -All [components](/extend/component/), including our internal R and Python Transformations, are run using Job Queue. - -## Workflow -The Job Queue functionality can be described in the following steps: - -- Download and build the specified Docker image. -- Download all [tables](/extend/common-interface/folders/#dataintables-folder) and [files](/extend/common-interface/folders/#datainfiles-folder) specified in the input mapping from Storage. -- Create a [configuration file](/extend/common-interface/config-file/). -- Run [before processors](/extend/component/processors/) if there are any. -- Run the Docker image (create a Docker container). -- Run [after processors](/extend/component/processors/) if there are any. -- Upload all [tables](/extend/common-interface/folders/#dataouttables-folder) and -[files](/extend/common-interface/folders/#dataoutfiles-folder) in the output mapping to Storage. -- Delete the container and all temporary files. - -When the component execution is finished, Job Queue automatically collects the exit code and the content of STDOUT and STDERR. -The following schema illustrates the workflow of running a dockerized component. - -![Docker Workflow](/extend/job-queue/docker-runner.svg) - -### Features -The component is responsible for these processes: - -- Reading the configuration and source tables in CSV format and files (if specified) -- Writing the results to the predefined folders and files -- Proper handling of success/error results by setting an appropriate exit code - -Job Queue is responsible for the following processes: - -- **Authentication:** Job Queue makes sure the component is run by authorized users/tokens. -It is not possible to run a component anonymously. The component does not have an access to the Keboola token -itself, and it receives only limited information about the project and the end-user. -- **Starting and stopping** the component: Job Queue will boot a Docker container which contains the -component. This ensures the component runs in a precisely defined environment, which is guaranteed to -be the same for each component run. No component state is preserved (with the exception of the -[state file](/extend/common-interface/config-file/#state-file). -- **Reading and writing data** to Keboola Storage: Job Queue ensures a custom component -cannot access arbitrary data in the project. It will only receive the input mapping defined by the end user; -and only those outputs defined in the output mapping by the end user will be written to the project. -- **Component isolation**: Each component is run in its own Docker container, which is isolated from other -containers; the component cannot be affected by other running components. It may also be limited -to have no network access. - -## API -The [Job Queue API](https://api.keboola.com/?service=job-queue) has API calls to - -- run a [component](/extend/component/). -- [encrypt values](/overview/encryption/). -- [prepare the data folder](/extend/component/running/#preparing-the-data-folder). -- run [component actions](/extend/common-interface/actions/). -- run a [component](/extend/component/) with a [specified Docker image tag](https://api.keboola.com/?service=job-queue#post-/jobs), usable for [testing images](/extend/component/deployment/#test-live-configurations). - -## Configuration -Components executed by Job Queue store their configurations in -[Storage API components configurations](https://api.keboola.com/?service=storage#tag--Component-Configurations). - -When creating the configuration, use -[this JSON schema](https://github.com/keboola/docker-bundle/blob/master/Resources/schemas/configuration.json) -to validate the configuration before storing it. The configuration contains the following nodes, -all of them are optional: - -- `parameters` --- an arbitrary object passed to the dockerized application itself -- `storage` --- configuration of [input and output mapping](/extend/common-interface/folders/); specific options correspond to the options of the -[unload data](https://keboola.docs.apiary.io/#reference/tables/unload-data-asynchronously) and -[load data](https://api.keboola.com/?service=storage#post-/v2/storage/branch/-branchId-/tables/-id-/import-async) API calls. -- `runtime` --- [runtime settings](/integrate/jobs/#job-runtime-configuration) (`tag`, `backend`, `parallelism`); most notably `runtime.tag` -pins the Docker image tag that jobs of this configuration run, which is the usual way of testing a development build of a component -- `processors` --- configuration of [Processors](/extend/component/processors/) -- `authorization` --- OAuth authorization [injected to the configuration](/extend/common-interface/oauth/); not stored in the component configuration -- `image_parameters` --- an arbitrary object passed from the [component](/extend/component/); not stored in the component configuration -- `action` --- an [action](/extend/common-interface/actions/) being executed; not stored in the component configuration diff --git a/extend/publish/approve.png b/extend/publish/approve.png deleted file mode 100644 index 3b9e2bc7..00000000 Binary files a/extend/publish/approve.png and /dev/null differ diff --git a/extend/publish/checklist.md b/extend/publish/checklist.md index 3a1b6fd5..6683fa22 100644 --- a/extend/publish/checklist.md +++ b/extend/publish/checklist.md @@ -1,46 +1,5 @@ ---- -title: Checklist -permalink: /extend/publish/checklist/ ---- - -* TOC -{:toc} - -This checklist is used for the last check of the component before sending the publish request. -See [Publish Component tutorial](/extend/publish/) for details. - -**Developer Portal** -- The component name doesn't contain words like `extractor`, `application`, and `writer`. -- The component icon is representative and has reasonable quality. It is in `PNG` format and without background. -- The short description describes the **service**, NOT `This extractor extracts ...`. -- Licensing information is valid, and the vendor description is current. -- License and documentation URLs are publicly accessible, no link to a private repository. -- The tag is set to the expected value and uses [semantic versioning](https://semver.org/). -- The correct [data flow is set in UI options](/extend/publish/#component-name-and-description). - - -**Component Configuration** -- Sensitive values [use encryption](/overview/encryption/). -- [Configuration and Row schema](/extend/publish/#component-configuration) - - Titles are short and without a colon, period, etc. - - Required properties are listed in the field `required`. - - Each property has defined `propertyOrder`. - - Properties have an explanatory `description` if they are not trivial. -- Configuration description (if used) - - Contains only level 3 `###` and level 4 `####` headers. - - Doesn't repeat what is obvious from Configuration and Row schema. - - -**Component Internals** -- Job exits with an understandable [UserError](/extend/common-interface/environment/#return-values) if: - - Empty configuration. - - Invalid credentials. - - Wrong data type used (e.g., string instead of array). - - Missing required property. - - Random/invalid data typed to the configuration properties. - - External server/service is down. - - An expected error occurs (e.g., not found, too many requests, ...). -- Internal messages (e.g., stack trace) with no meaning for the user are not logged. - -**Publication Request** -- A link to the pull request with changes in the [documentation](https://help.keboola.com/) is included (if any). +--- +title: Checklist +permalink: /extend/publish/checklist/ +redirect_to: https://help.keboola.com/extend/publish/checklist/ +--- diff --git a/extend/publish/index.md b/extend/publish/index.md index a1894808..f29dcc67 100644 --- a/extend/publish/index.md +++ b/extend/publish/index.md @@ -1,127 +1,8 @@ ---- -title: Publish Component -permalink: /extend/publish/ -redirect_from: - - /extend/registration/checklist/ - - /extend/registration/ ---- - -* TOC -{:toc} - -As described in the [architecture overview](/overview/), Keboola consists of many different components. -Only those components that are published in our **Component List** are generally available in Keboola. -The list can be found in our [Storage Component API](https://api.keboola.com/?service=storage#get-/v2/storage) in the dedicated [Components section](https://api.keboola.com/?service=storage#get-/v2/storage). -The list of components is managed using the Keboola [Developer Portal](https://components.keboola.com/). - -That being said, any Keboola user can use any component, unless - -- the Keboola user (or their token) has a [limited access to the component](https://help.keboola.com/storage/tokens/). -- the component itself limits where it can run (in what projects and for which users). - -If you have not yet created your component, please go through the [tutorial](/extend/component/tutorial/), which will -navigate you through creating an account in the [Developer Portal](https://components.keboola.com/) and -[initializing the component](/extend/component/tutorial/). - -## Publishing Component -A non-published component can be used without limitations, but it is not offered in the Keboola UI. It can only be used via -the [API](https://api.keboola.com/?service=storage#tag--Component-Configurations) or by directly visiting a link with the -specific component ID: - - https://connection.keboola.com/admin/projects/{PROJECT_ID}/extractors/{COMPONENT_ID} - -This way you can fully test your component before requesting its publication. Also, unpublished -components are not part of our [list of public components](https://components.keboola.com/components). -An existing configuration of a non-public component is accessible the same way as a configuration of any other component. - -**Important:** Changes made in the Developer Portal take up to 5 minutes to propagate to all Keboola instances in all regions. - -Before your component can be published, it must be approved by Keboola. Request the approval from the component list in -the [Developer Portal](https://components.keboola.com/). We will review your component and either publish it or contact you -with required changes. - -{: .image-popup} -![Approval screenshot](/extend/publish/approve.png) - -## Component Review -The goal of the component review is to maintain reasonable end-user experience and component reliability. Before -applying for component registration, make sure the same component does not already exist. If there is a similar one -(e.g., an extractor for the same service), clearly state the differences in the new component's description. During our -component review, the best practices in the next sections are followed. - -### Component Name and Description -Before you name and describe your component, check out our YouTube, Facebook Pages, Dark Sky, and ECB Currency Rates -components for inspiration. - -- Names should not contain words like `extractor`, `application`, and `writer`. -
OK: *Cloudera Impala* -
WRONG: *Cloudera Extractor* -- The short description describes the **service** (helping the user find it) rather than the component. -Obviously for large services like Facebook or Gmail, describe the part of the service relevant to the component. -
OK: *Native analytic database for Apache Hadoop* -
WRONG: *This extractor extracts data from Cloudera Impala* -
OK: *Facebook Pages connect your business with people. Facebook Insights help you get good at it.* -
WRONG: *Facebook connects you with friends, family and other people you know, allows you to share photos and videos, send messages and get updates.* -- The long description provides **additional information about the extracted/written data**: -What will the end user get? What must the end user provide? Is the data going to be imported incrementally? Are there links to -available resources?
Configuration instructions should not be included in the long description, because the long description -is displayed before the end user starts configuring the component. However, if there are any special requirements (external approval, -specific account setting), they should be stated. -
OK: *This component allows you to extract currency exchange rates as published by the European Central Bank (ECB). The -exchange rates are available from a base currency (USD, EUR) to 30 destination currencies (AUD, BGN, BRL, CAD, CNY, -CZK, EUR, GBP, HKD, HRK, HUF, CHF, IDR, ILS, INR, JPY, KRW, MXN, MYR, NOK, NZD, PHP, PLN, RON, RUB, SEK, SGD, THB, TRY, -ZAR). The rates are available for all working days from 4 January 1999 up to present.* -- Component icons must be of representative and reasonable quality. Make sure the icon license allows you to use it. -- Components must correctly state the data flow --- [UI options](/extend/component/ui-options/). Use -`appInfo.dataOut` and `appInfo.dataIn` for this purpose: - - Use `appInfo.dataIn` for extractors, which bring data into a Keboola project (omit `appInfo.dataOut` for extractors). - - Use `appInfo.dataOut` for writers, which send data outside (omit `appInfo.dataIn` for writers). - - Use `appInfo.dataIn` and/or `appInfo.dataOut` for applications. -- Use `appInfo.beta` in [UI options](/extend/component/ui-options/) if you suspect changes to the component behavior. -- Licensing information must be valid, and the vendor description must be current. - -### Component Icon - -- Use a PNG image that is at least 256x256px large and has transparent background. - -### Component Configuration - -- Use only the necessary [UI options](/extend/component/ui-options/) (i.e., if there are no output files, do not use `genericDockerUI-fileOutput`). -- For extractors, always use the [default bucket](/extend/common-interface/folders/#default-bucket) --- do not use the `genericDockerUI-tableOutput` flag. -- Use [encryption](/overview/encryption/) to store sensitive values. No plain-text passwords! -- Use a [configuration schema](/extend/component/ui-options/configuration-schema/). - - List all properties in the `required` field. - - Always use `propertyOrder` to explicitly define the order of the fields in the form. - - Use your short `title` without a colon, period, etc. - - Use `description` to provide an explanatory sentence if needed. -
OK: ![Good Schema](/extend/publish/schema-good.png) -
WRONG: ![Bad Schema](/extend/publish/schema-bad.png) -- Use a configuration description only if the configuration is not trivial/self-explanatory. Provide **links to resources** -(for instance, when creating an Elastic extractor, not everyone is familiar with the ElasticSearch query syntax). The -configuration description supports markdown. Your markdown should not start with a header and should use only level 3 and -level 4 headers (level 2 header is prepended before the configuration description).
OK:
some introduction text

### Input -Description
-description of input tables
-
#### First Table
-some other text
-WRONG:
## Configuration Description
-some introduction text
-
#### Input Description
-description of input tables - - -### Component Internals - -- Make sure that the amount of consumed **memory does not depend** on the amount of processed data. Use streaming or -processing in chunks to maintain a limited amount of consumed memory. If not possible, state the expected usage in -the **Component Limits**. -- The component must distinguish between [user and application errors](/extend/common-interface/environment/#return-values). -- The component must [validate](/extend/common-interface/config-file/#validation) its parameters; an invalid configuration must result in a user error. User error messages must clearly state what's wrong and what the user should do to fix the issue. E.g., `Invalid configuration.` is wrong, `Login failed, check your credentials.` is better. -- The events produced must be reasonable. Provide status messages if possible and with a reasonable frequency. Avoid internal messages with no meaning to the end user. Also avoid flooding the event log or sending data files in the event log. -- Set up [continuous deployment](/extend/component/deployment/) so that you can keep the component up to date. -- Use [semantic versioning](http://semver.org/) to mark and deploy versions of your component. Using other tags (e.g., -`latest`, `master`) in production is not allowed. - -### Checklist - -Before requesting to publish a component, please check all rules using [this checklist](/extend/publish/checklist). +--- +title: Publish Component +permalink: /extend/publish/ +redirect_from: + - /extend/registration/checklist/ + - /extend/registration/ +redirect_to: https://help.keboola.com/extend/publish/ +--- diff --git a/extend/publish/schema-bad.png b/extend/publish/schema-bad.png deleted file mode 100644 index ed245075..00000000 Binary files a/extend/publish/schema-bad.png and /dev/null differ diff --git a/extend/publish/schema-good.png b/extend/publish/schema-good.png deleted file mode 100644 index 8082071b..00000000 Binary files a/extend/publish/schema-good.png and /dev/null differ diff --git a/index.md b/index.md index 77df2d87..e2267094 100644 --- a/index.md +++ b/index.md @@ -26,7 +26,7 @@ In this documentation, we will show you how to - Control whole data processing pipeline within Keboola from the [outside](/integrate/). - Leverage natural language to build data pipelines and interact with your Keboola project using the [Model Context Protocol (MCP)](https://help.keboola.com/ai/mcp-server/). - [**Extend Keboola by building your own components**](/extend/) for your own use or for other Keboola users and customers. - - [Extend Keboola with arbitrary Docker images](/extend/component/). + - [Extend Keboola with arbitrary Docker images](https://help.keboola.com/extend/component/). - Build your own [extractors](/extend/generic-extractor/) for services we do not support yet. - [**Automate your processes**](/automate/) to run any component in specified intervals or at specified times of the day. - Control any component of Keboola [programmatically](/integrate/jobs/) (for example, you can trigger data load when something happens in your system). @@ -48,7 +48,7 @@ Under the following conditions: - You will use the project fairly (not abuse it or use it for production). - You will remain active in the development. -Note that once you [register](/extend/component/tutorial/#before-you-start) (and join a vendor) in +Note that once you [register](https://help.keboola.com/extend/component/tutorial/#before-you-start) (and join a vendor) in our [Developer portal](https://components.keboola.com/), you will gain access to a development project. If you don't have it, or need a development project for other reasons, [send us an email](mailto:support@keboola.com). Not into creative writing? Feel free to use our template: diff --git a/integrate/artifacts/index.md b/integrate/artifacts/index.md index 5bc79317..dee1b5b4 100644 --- a/integrate/artifacts/index.md +++ b/integrate/artifacts/index.md @@ -8,7 +8,7 @@ permalink: /integrate/artifacts/ *Note: This is a preview feature and as such may change considerably in the future. The project must have an `artifacts` feature enabled.* -**Artifacts** are additional files that can be produced or consumed by a [component](/extend/component). +**Artifacts** are additional files that can be produced or consumed by a [component](https://help.keboola.com/extend/component/). See [Tutorial](/integrate/artifacts/tutorial) for step-by-step example. diff --git a/integrate/jobs/index.md b/integrate/jobs/index.md index a1fd8ef5..beee92b0 100644 --- a/integrate/jobs/index.md +++ b/integrate/jobs/index.md @@ -214,7 +214,7 @@ Such functionality is implemented in specific components (e.g. Orchestrator) or ### Job Configuration To create a job, you must provide the [configuration](https://help.keboola.com/components/) to run. A configuration is always tied to a specific -[component](/extend/component/). +[component](https://help.keboola.com/extend/component/). A configuration can be provided in multiple ways. The easiest is to provide a reference to a stored configuration ID using the `config` field as shown above. Configurations can be stored and listed using the @@ -233,13 +233,13 @@ has to be provided in the request. If you are retrieving a note that the configuration data is the contents of the `configuration` node and not the entire response. When using the `configData` field, the `configRowIds` and `branchId` values are ignored. When using the `configData` field the `config` field is ignored for the purpose of reading the configuration, but may still be required in case the component is using -[Default Bucket](https://developers.keboola.com/extend/component/tutorial/output-mapping/#configuring-default-bucket). In that case, the +[Default Bucket](https://help.keboola.com/extend/component/tutorial/output-mapping/#configuring-default-bucket). In that case, the configuration referenced in `config` is used to generate the name of the output bucket. It still holds that configuration data is not read from it. That means that `configData` always fully overrides the `config` field. ### Job Mode When creating a job, you need to provide `mode`. This can be one of `run`, `forceRun` and `debug`. The basic `mode` choice is `run`. -Use the `forceRun` mode to run a configuration that is disabled. The `debug` can be used during [Component Development & Debugging](https://developers.keboola.com/extend/component/tutorial/debugging/). +Use the `forceRun` mode to run a configuration that is disabled. The `debug` can be used during [Component Development & Debugging](https://help.keboola.com/extend/component/tutorial/debugging/). ### Job Runtime configuration You may provide runtime settings for a job. Runtime settings do not affect what the job does, they affect how the job does it. The available runtime settings are: @@ -268,7 +268,7 @@ not inside it. For example, to pin all jobs of a configuration to a specific ima Jobs of this configuration then run the given image tag (the job detail shows the resolved value in its `tag` field) until the `runtime.tag` key is removed from the configuration. This is the usual way of testing a development build of a component in a single project without affecting -other projects --- the tag in the [Developer Portal](/extend/publish/) stays untouched. Do not forget to remove the key when done; a pinned +other projects --- the tag in the [Developer Portal](https://help.keboola.com/extend/publish/) stays untouched. Do not forget to remove the key when done; a pinned configuration keeps running the old image even after new versions of the component are released. ### Job Type @@ -465,7 +465,7 @@ to one of the finished states or until `isFinished` is true. ### Run a Debug job To run a debug job, use `debug` for the mode. Optionally you can provide the component version which should run -to [live test](/extend/component/deployment/#test-live-configurations) an image. +to [live test](https://help.keboola.com/extend/component/deployment/#test-live-configurations) an image. ```json { @@ -477,7 +477,7 @@ to [live test](/extend/component/deployment/#test-live-configurations) an image. ``` The debug mode creates a job that prepares the data folder including the serialized configuration files. Then it compresses the -[data folder](/extend/component/running/#preparing-the-data-folder) and uploads it to your project's Files in Storage. This way you will get a snapshot +[data folder](https://help.keboola.com/extend/component/running/#preparing-data-folder) and uploads it to your project's Files in Storage. This way you will get a snapshot of what the data folder looked like before the component started. If processors are used, a snapshot of the data folder is created before each processor. After the entire component finishes, another snapshot is made. For example, if you run component A with processor B and C in the after section, you will receive: - `stage_0` file with contents of the data folder before component A was run diff --git a/integrate/storage/api/configurations.md b/integrate/storage/api/configurations.md index 45e0552d..8e4fd494 100644 --- a/integrate/storage/api/configurations.md +++ b/integrate/storage/api/configurations.md @@ -197,22 +197,22 @@ configuration of the `keboola.ex-aws-s3` component. You will receive a response The actual component configuration is split into three parts: - `configuration` node, containing an arbitrary component configuration -- `state` node, containing a component [state file](/extend/common-interface/config-file/#state-file) +- `state` node, containing a component [state file](https://help.keboola.com/extend/common-interface/config-file/#state-file) - `rows` node, containing iterations of `configuration` and `state` The important part is the ID of the configuration you want to work with. In the following examples, we will use `364479526`. ### Configuration -The `configuration` node maps to the [configuration file](/extend/common-interface/config-file/#configuration-file-structure). +The `configuration` node maps to the [configuration file](https://help.keboola.com/extend/common-interface/config-file/#configuration-file-structure). It can contain the `storage`, `parameters`, `processors` and `authorization` child nodes (the `image_parameters` and `action` nodes found in the config file are injected at runtime and are not stored in the configuration). The `authorization` node is set in the configuration only when -[credentials injection](/extend/common-interface/oauth/#credentials-injection) should be used, otherwise it is also set during the runtime. -The `processors` node defines the [processors and their configuration](/extend/component/processors/). +[credentials injection](https://help.keboola.com/extend/common-interface/oauth/#credentials-injection) should be used, otherwise it is also set during the runtime. +The `processors` node defines the [processors and their configuration](https://help.keboola.com/extend/component/processors/). The most common sub-nodes stored in the `configuration` node are therefore `parameters` (containing an arbitrary component configuration) -and `storage` (containing [input](/extend/component/tutorial/input-mapping/) and [output mapping](/extend/component/tutorial/output-mapping/)). +and `storage` (containing [input](https://help.keboola.com/extend/component/tutorial/input-mapping/) and [output mapping](https://help.keboola.com/extend/component/tutorial/output-mapping/)). Both are transferred to the -configuration file without modification; that means that the [`storage` configuration](/extend/common-interface/config-file/#configuration-file-structure) +configuration file without modification; that means that the [`storage` configuration](https://help.keboola.com/extend/common-interface/config-file/#configuration-file-structure) is directly usable in the `configuration` node. The `parameters` node is fully dependent on the component and has no universal specification or rules. In the above example, the `configuration` node contains the following: @@ -235,7 +235,7 @@ there are rows. For each row, the `configuration` node from `root` and the `conf the latter overwriting the former in the case of conflict. Given the above configuration, the **effective configuration** passed to the component -[configuration file](/extend/common-interface/config-file/#configuration-file-structure) will be as follows: +[configuration file](https://help.keboola.com/extend/common-interface/config-file/#configuration-file-structure) will be as follows: {% highlight json %} { @@ -279,7 +279,7 @@ iterable part which is repeated many times. Keep in mind that configurations hea in the UI**. ### State -The `state` node contains the content of the [state file](/extend/common-interface/config-file/#state-file). The +The `state` node contains the content of the [state file](https://help.keboola.com/extend/common-interface/config-file/#state-file). The `state` is read from the state file and then supplied to the state file on the next run. In the above configuration, the state is: @@ -307,7 +307,7 @@ which will return all the configuration details. This means - the configuration itself (`configuration`) --- [section on configuration](#modifying-a-configuration) follows; - configuration rows (`rows`) --- additional data of the configuration; and -- configuration state (`state`) --- [component state](/extend/common-interface/config-file/#state-file). +- configuration state (`state`) --- [component state](https://help.keboola.com/extend/common-interface/config-file/#state-file). Please note that the contents of the `configuration`, `rows` and `state` sections depend purely on the component itself. See an [example](https://documenter.getpostman.com/view/3086797/kbc-samples/77h845D?version=latest#9b9f3e7b-de3b-4c90-bad6-a8760e3852eb). diff --git a/integrate/storage/api/tde-exporter.md b/integrate/storage/api/tde-exporter.md index f2048288..be82c40d 100644 --- a/integrate/storage/api/tde-exporter.md +++ b/integrate/storage/api/tde-exporter.md @@ -16,7 +16,7 @@ as an orchestration task. After the exporter finishes, the resulting TDE files w *Storage* --- *File uploads* section where you can download them via UI or [API](/integrate/storage/api/import-export/). ## Running the Component -The TDE Exporter is a Keboola [component](/extend/component/) supporting both +The TDE Exporter is a Keboola [component](https://help.keboola.com/extend/component/) supporting both [stored](/integrate/storage/api/configurations/) and custom configurations supplied directly in the `run` request. @@ -35,7 +35,7 @@ Then [create a job](/integrate/jobs/) with the specified configuration: ### Custom Configuration You can specify the entire configuration in the API call. The JSON configuration conforms -to the [general configuration format](/extend/common-interface/config-file/). The specific part +to the [general configuration format](https://help.keboola.com/extend/common-interface/config-file/). The specific part is only the `parameters` section. A sample request to the `in.c-main.old-table` export table would look like this: {% highlight json %} diff --git a/integrate/variables/index.md b/integrate/variables/index.md index 1372217e..7dfcd12c 100644 --- a/integrate/variables/index.md +++ b/integrate/variables/index.md @@ -188,8 +188,8 @@ See an [example](https://documenter.getpostman.com/view/3086797/77h845D?version= ### Step 3 -- Create Main Configuration Now it is time to create the actual configuration which will contain a Python transformation. -Use the following configuration body. The `storage` section describes the standard [input](/extend/common-interface/config-file/#input-mapping--basic) -and [output](/extend/common-interface/config-file/#output-mapping--basic) mapping. +Use the following configuration body. The `storage` section describes the standard [input](https://help.keboola.com/extend/common-interface/config-file/#input-mapping--basic) +and [output](https://help.keboola.com/extend/common-interface/config-file/#output-mapping--basic) mapping. {% highlight json %} {% raw %} @@ -315,7 +315,7 @@ values, e.g.: |Italybatman|4139387742| |Romaniabatman|654126042| -The events of the job will contain the contents of the [configuration file](/extend/common-interface/config-file/) +The events of the job will contain the contents of the [configuration file](https://help.keboola.com/extend/common-interface/config-file/) where you can verify that the variables were replaced.
diff --git a/integrate/variables/tutorial.md b/integrate/variables/tutorial.md index 4f580ece..3cc975c3 100644 --- a/integrate/variables/tutorial.md +++ b/integrate/variables/tutorial.md @@ -18,7 +18,7 @@ First, store the [API endpoints](/overview/api/) as environment variables, so we We will need: - [Storage API](/integrate/storage/api/) to store the variable definitions and the extractor configuration - -- [Job Queue API](/extend/job-queue/) to run the extractor job from the configuration. +- [Job Queue API](https://help.keboola.com/extend/job-queue/) to run the extractor job from the configuration. The host names depend on your [stack](/overview/api/#stacks-and-endpoints): diff --git a/overview/api/index.md b/overview/api/index.md index 4c301540..107dc3bf 100644 --- a/overview/api/index.md +++ b/overview/api/index.md @@ -27,18 +27,18 @@ Using a portal for a different stack than your token's stack will result in