diff --git a/NOTICE.txt b/NOTICE.txt index f2d139a8f..8f1a273ad 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5655,8 +5655,8 @@ MIT License multidict -6.5.1 -Apache Software License +6.6.3 +Apache License 2.0 Copyright 2016 Andrew Svetlov and aio-libs contributors Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/connectors/connectors_cli.py b/connectors/connectors_cli.py index ffccc9aba..16d426e3b 100644 --- a/connectors/connectors_cli.py +++ b/connectors/connectors_cli.py @@ -187,7 +187,7 @@ def interactive_service_type_prompt(): @click.option( "--index-name", prompt=f"{click.style('?', fg='green')} Index name", - help="Name of the index. If the connector will be native, `search-` will be prepended to the index name.", + help="Name of the index. For managed connectors, index name should be prefixed with `content-`. For self-managed connectors, no prefix is required.", ) @click.option( "--service-type", diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 9963fc99e..71f422828 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -55,22 +55,20 @@ connectors: - connector_id: 1234 service_type: api_key: - # Used to write data to the `search-*` index associated with connector 1234 + # Used to write data to the content index associated with connector 1234 # You may have multiple connectors in your config file! - connector_id: 5678 service_type: api_key: - # Used to write data to the `search-*` index associated with connector 5678 + # Used to write data to the content index associated with connector 5678 - connector_id: abcd service_type: # No explicit api key specified, so this connector will used ``` -## Connector service on Elastic Cloud -When you have an Enterprise Search deployment on Elastic Cloud (post 8.5.0), the connector service is **automatically deployed**. -The connector service runs in native mode on Elastic Cloud, and the Elasticsearch connection configurations (i.e. `host`, `username`, `password`) are automatically configured. -A special Cloud user (`cloud-internal-enterprise_search-server`) is used for the Elasticsearch connection. -This user has proper privileges on the connector index (`.elastic-connectors`), the connector job index (`.elastic-connectors-sync-jobs`) and the connector content indices (`search-*`). +## Managed connectors on Elastic Cloud -To run self-managed connector clients, you will need to run the connector service on-prem, as described in the first section. +Elastic managed connectors are now available via agentless deployment on Elastic Cloud. The connector service runs using a Fleet Server account that has the `manage_connector` permission and access to the connector indices (`.elastic-connectors`, `.elastic-connectors-sync-jobs`) and the connector content indices (`content-*`). + +The Elasticsearch connection configurations (i.e. `host`, `api_key`) are automatically configured by the Fleet Server infrastructure. diff --git a/resources/connectors_api/Connectors API.postman_collection.json b/resources/connectors_api/Connectors API.postman_collection.json index a87650ad1..34f71fe73 100644 --- a/resources/connectors_api/Connectors API.postman_collection.json +++ b/resources/connectors_api/Connectors API.postman_collection.json @@ -40,7 +40,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"index_name\": \"search-test-index\",\n \"name\": \"my-connector\",\n \"is_native\": false,\n \"service_type\": \"google_drive\"\n}", + "raw": "{\n \"index_name\": \"test-index\",\n \"name\": \"my-connector\",\n \"is_native\": false,\n \"service_type\": \"google_drive\"\n}", "options": { "raw": { "language": "json" @@ -67,7 +67,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"index_name\": \"search-test-index\",\n \"name\": \"my-connector\",\n \"is_native\": false,\n \"service_type\": \"google_drive\"\n}", + "raw": "{\n \"index_name\": \"test-index\",\n \"name\": \"my-connector\",\n \"is_native\": false,\n \"service_type\": \"google_drive\"\n}", "options": { "raw": { "language": "json" @@ -1296,7 +1296,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"index_name\": \"search-test-index-2\"\n}", + "raw": "{\n \"index_name\": \"test-index-2\"\n}", "options": { "raw": { "language": "json" @@ -1442,4 +1442,4 @@ "type": "string" } ] -} \ No newline at end of file +} diff --git a/scripts/testing/README.md b/scripts/testing/README.md index e6bf6b79b..c16257044 100644 --- a/scripts/testing/README.md +++ b/scripts/testing/README.md @@ -2,10 +2,10 @@ ### Installation -Automated testing CLI depends on `gcloud` and `vault` CLIs so you need to install them. +Automated testing CLI depends on `gcloud` and `vault` CLIs so you need to install them. #### Google cloud CLI -1. Follow the [offical installation](https://cloud.google.com/sdk/docs/install) guide. Once installed, run `gcloud auth login` and authorize using SSO. +1. Follow the [offical installation](https://cloud.google.com/sdk/docs/install) guide. Once installed, run `gcloud auth login` and authorize using SSO. 2. Set the correct Google cloud project by running this command `gcloud config set project connectors-sources`. #### Vault @@ -19,10 +19,10 @@ Run `make clean install` to generate executable files in `./bin` folder ### Usage -Run `.venv/bin/test-connectors --help` or `.venv/bin/test-connectors {command name} --help` to get more information about the cli. +Run `.venv/bin/test-connectors --help` or `.venv/bin/test-connectors {command name} --help` to get more information about the cli. #### Running test with Elastic cloud deployment -If you want to run your test suite using a cloud Elasticsearch deployment follow the next steps: +If you want to run your test suite using a cloud Elasticsearch deployment follow the next steps: 1. Create a cloud deployment 2. Download a credentials file (or create a new user) 3. Run `.venv/bin/test-connectors run-test my-testing-environment-name --es-host {host} --es-username {user name} --es-password {password} --test-case {path to the test case file}` @@ -32,25 +32,25 @@ If you want to run your tests with local Elasticsearch you need to specify `--es `.venv/bin/test-connectors run-test my-testing-environment-name --es-version 8.12-SNAPSHOT --test-case {path to the test case file}` -In this case, the cli will deploy an Elasticsearch instance in the same VM where the connector service will be running. +In this case, the cli will deploy an Elasticsearch instance in the same VM where the connector service will be running. #### Running test with a specific Connector service version -You can use any git reference such as commit sha, a tag, or a branch name. The cli will pull the defined git reference and run `make clean install`. +You can use any git reference such as commit sha, a tag, or a branch name. The cli will pull the defined git reference and run `make clean install`. Example: `.venv/bin/test-connectors run-test my-testing-environment-name --es-version 8.12-SNAPSHOT --connectors-ref 8.12 --test-case {path to the test case file}` #### Keeping your VM running when the tests passed -Sometimes it's useful to get access to the logs or make some changes in the code and run tests again. The CLI will print a list of useful commands you can use to access the VM resources like: +Sometimes it's useful to get access to the logs or make some changes in the code and run tests again. The CLI will print a list of useful commands you can use to access the VM resources like: `Access logs: gcloud compute ssh {my-testing-environment-name} --zone {VM zone} --command "tail -f ~/service.log"` -To automatically delete the VM you need to use `--delete` option. +To automatically delete the VM you need to use `--delete` option. `.venv/bin/test-connectors run-test my-testing-environment-name --es-version 8.12-SNAPSHOT --connectors-ref 8.12 --test-case {path to the test case file} --delete` #### Using different machine type -All new VMs are based on a predefined image which is in turn based on `ubuntu-2204-lts` with python3 and docker installed. Custome images are not supported. You can change a machine type by providing `--vm-type`. Visit [the official GCP documentation](https://cloud.google.com/compute/docs/general-purpose-machines) to get more information. +All new VMs are based on a predefined image which is in turn based on `ubuntu-2204-lts` with python3 and docker installed. Custome images are not supported. You can change a machine type by providing `--vm-type`. Visit [the official GCP documentation](https://cloud.google.com/compute/docs/general-purpose-machines) to get more information. #### Adding a new test case @@ -59,7 +59,7 @@ A test case should be present as a YAML file. ```YAML --- scenarios: - - index_name: search-demo-index-001 + - index_name: demo-index-001 connector_name: spo service_type: sharepoint_online index_language: en @@ -76,11 +76,11 @@ scenarios: match: { status: 'completed'} ``` -Each test case can contain multiple scenarios which will be executed one by one. Each scenario can have multiple test cases which will be executed one by one too. Currently, only one type of match is supported (job status). +Each test case can contain multiple scenarios which will be executed one by one. Each scenario can have multiple test cases which will be executed one by one too. Currently, only one type of match is supported (job status). To add a new test case you need to create a new YAML file and describe your test case. Give it a thoughtful name. Also, you need to define `connector_configuration` that points to a connector configuration file. -Example: +Example: ```JSON { @@ -102,11 +102,11 @@ Example: Make sure that all the required fields are present in the secret. The CLI will get all the secrets and upload them to the VM. -Consider adding new secrets only for new connector sources. Reach out to the Search Productivity team if you need any help with Vault secrets. +Consider adding new secrets only for new connector sources. Reach out to the Search Productivity team if you need any help with Vault secrets. ### Can I use gcloud? Yes, it's already installed on your machine and the cli does not limit your use. Visit [the official documentation page](https://cloud.google.com/sdk/docs) for more information. ### Hygiene of use -Since the CLI creates virtual machines in GCP it's recommended to keep them running only when you need them. Let's save some trees 🌳🌳🌳. +Since the CLI creates virtual machines in GCP it's recommended to keep them running only when you need them. Let's save some trees 🌳🌳🌳. In case of pipeline errors check if your VM is still running `gcloud compute instances list` and then delete it manually `gcloud compute instances delete `