CH-291 Harmonize yaml libraries and formatting

libraries/models/test-requirements.txt

@@ -3,4 +3,4 @@ pytest-cov>=2.8.1
 pytest-randomly>=3.12.0
 mypy>=1.4.1
 types-python-dateutil>=2.8.19
-oyaml
+ruamel.yaml

libraries/models/test/test_backwards_compatibility.py

@@ -1,7 +1,7 @@
 
 from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig, CDCEvent
 from os.path import join, dirname as dn, realpath
-import oyaml as yaml
+from ruamel.yaml import YAML
 
 HERE = dn(realpath(__file__))
 
@@ -37,7 +37,7 @@ def test_dict_behaviour():
 
 def test_usages():
     with open(join(HERE, "resources/values.yaml")) as f:
-        values = yaml.safe_load(f)
+        values = YAML(typ="safe").load(f)
         v = HarnessMainConfig.from_dict(values)
     assert v.apps["accounts"].harness.database
     assert v.apps["accounts"].client.id

libraries/models/test/test_deserialize.py

@@ -1,5 +1,5 @@
 from os.path import join, dirname as dn, realpath
-import oyaml as yaml
+from ruamel.yaml import YAML
 
 from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig, CDCEvent, ApplicationTestConfig, DatabaseConfig, GatekeeperConf
 
@@ -8,7 +8,7 @@
 
 def test_helm_values_deserialize():
     with open(join(HERE, "resources/values.yaml")) as f:
-        values = yaml.safe_load(f)
+        values = YAML(typ="safe").load(f)
     v = HarnessMainConfig.from_dict(values)
 
     assert v.domain

libraries/models/test/test_serialize.py

@@ -2,7 +2,7 @@
 import json
 from os.path import join, dirname as dn, realpath
 
-import oyaml as yaml
+from ruamel.yaml import YAML
 
 from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig
 from cloudharness_model.encoder import CloudHarnessJSONEncoder
@@ -11,7 +11,7 @@
 
 def test_json_serialize():
     with open(join(HERE, "resources/values.yaml")) as f:
-        values = yaml.safe_load(f)
+        values = YAML(typ="safe").load(f)
     v = HarnessMainConfig.from_dict(values)
     dumped = json.dumps(v, cls=CloudHarnessJSONEncoder)
     cloned = json.loads(dumped)

tools/deployment-cli-tools/ch_cli_tools/codefresh.py

@@ -6,15 +6,15 @@
 import logging
 from cloudharness_model.models.api_tests_config import ApiTestsConfig
 
-import oyaml as yaml
+from ruamel.yaml.scalarstring import SingleQuotedScalarString
 
 from cloudharness_utils.testing.util import get_app_environment
 from .models import HarnessMainConfig, ApplicationTestConfig, ApplicationHarnessConfig
 from cloudharness_utils.constants import *
 from .configurationgenerator import KEY_APPS, KEY_TASK_IMAGES, KEY_TEST_IMAGES
 from .secrets import is_cloudharness_managed, is_secret_config, secret_value
 from .utils import check_image_exists_in_registry, find_dockerfiles_paths, get_app_relative_to_base_path, guess_build_dependencies_from_dockerfile, \
-    get_template, dict_merge, app_name_from_path, clean_path, strip_registry_tag, get_image_source
+    get_template, dict_merge, app_name_from_path, clean_path, strip_registry_tag, get_image_source, yaml, yaml_rt
 from cloudharness_utils.testing.api import get_api_filename, get_schemathesis_command, get_urls_from_api_file
 
 logging.getLogger().setLevel(logging.INFO)
@@ -44,19 +44,6 @@ def _to_codefresh_path(path: str) -> str:
         return '/'.join([CLOUD_HARNESS_PATH] + parts[i + 1:])
     return rel
 
-# Codefresh variables may need quotes: adjust yaml dump accordingly
-
-
-def literal_presenter(dumper, data):
-    if isinstance(data, str) and "\n" in data:
-        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
-    if isinstance(data, str) and data.startswith('${{'):
-        return dumper.represent_scalar('tag:yaml.org,2002:str', data, style="'")
-    return dumper.represent_scalar('tag:yaml.org,2002:str', data)
-
-
-yaml.add_representer(str, literal_presenter)
-
 
 def clean_step_key(s: str) -> str:
     """Normalize a string to a valid Codefresh step key (alphanumeric + underscore)."""
@@ -236,7 +223,7 @@ def codefresh_app_build_spec(app_name, full_image_name, app_context_path, docker
                 logging.info("Specific build template found: %s" %
                              (specific_build_template_path))
                 with open(specific_build_template_path) as f:
-                    build_specific = yaml.safe_load(f)
+                    build_specific = yaml.load(f)
 
                 build_specific.pop(
                     'build_arguments') if 'build_arguments' in build_specific else []
@@ -551,8 +538,6 @@ def adjust_build_steps(index):
         codefresh_dir = dirname(codefresh_abs_path)
         if not exists(codefresh_dir):
             os.makedirs(codefresh_dir)
-        from ruamel.yaml.scalarstring import SingleQuotedScalarString
-
         deployment_step = codefresh.get("steps", {}).get("deployment", {})
         arguments = deployment_step.get("arguments", {})
         if "custom_values" in arguments:
@@ -561,11 +546,10 @@ def adjust_build_steps(index):
                 for v in arguments["custom_values"]
             ]
 
-        from ruamel.yaml import YAML
-        ryaml = YAML()
-        ryaml.default_flow_style = False
+        # Round trip handler: the codefresh steps are deliberately ordered by stage,
+        # a sorting representer would scramble them.
         with open(codefresh_abs_path, 'w') as f:
-            ryaml.dump(codefresh, f)
+            yaml_rt.dump(codefresh, f)
     return codefresh
 
 

tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py

@@ -3,7 +3,6 @@
 """
 from typing import List, Union
 import copy
-import yaml
 import os
 import shutil
 import logging
@@ -18,7 +17,7 @@
     DEPLOYMENT_CONFIGURATION_PATH, BASE_IMAGES_PATH, STATIC_IMAGES_PATH
 from .utils import get_cluster_ip, env_variable, get_dockerfile_baseimg_args, get_sub_paths, guess_build_dependencies_from_dockerfile, image_name_from_dockerfile_path, \
     get_template, merge_configuration_directories, dict_merge, app_name_from_path, \
-    find_dockerfiles_paths, get_git_commit_hash
+    find_dockerfiles_paths, get_git_commit_hash, yaml
 from .secrets import secret_definition_error
 
 
@@ -111,7 +110,7 @@ def __init_deployment(self):
     def _adjust_missing_values(self, helm_values):
         if 'name' not in helm_values:
             with open(self.helm_chart_path) as f:
-                chart_idx_content = yaml.safe_load(f)
+                chart_idx_content = yaml.load(f)
             helm_values['name'] = chart_idx_content['name'].lower()
 
     def _process_applications(self, helm_values, base_image_name=None):
@@ -481,7 +480,7 @@ def collect_helm_values(deployment_root, env=()):
             logging.info(
                 "Specific environment values template found: " + specific_template_path)
             with open(specific_template_path) as f:
-                values_env_specific = yaml.safe_load(f)
+                values_env_specific = yaml.load(f)
             values = dict_merge(values, values_env_specific)
     return values
 

tools/deployment-cli-tools/ch_cli_tools/dockercompose.py

@@ -3,8 +3,6 @@
 """
 from pathlib import Path
 from typing import Union
-import yaml
-from ruamel.yaml import YAML
 import os
 import logging
 import subprocess
@@ -13,7 +11,8 @@
 
 from cloudharness_utils.constants import VALUES_MANUAL_PATH, COMPOSE
 from .utils import get_cluster_ip, image_name_from_dockerfile_path, get_template, \
-    merge_to_yaml_file, dict_merge, app_name_from_path, find_dockerfiles_paths, find_file_paths
+    merge_to_yaml_file, dict_merge, app_name_from_path, find_dockerfiles_paths, find_file_paths, \
+    yaml, yaml_rt
 
 from .models import HarnessMainConfig
 
@@ -142,8 +141,7 @@ def __post_process_multiple_document_docker_compose(self, yaml_document):
             logging.warning("Something went wrong during the docker-compose.yaml generation, cannot post-process it")
             return
 
-        yaml_handler = YAML()
-        documents = yaml_handler.load_all(yaml_document)
+        documents = yaml_rt.load_all(yaml_document)
 
         main_document = None
         for document in documents:
@@ -161,7 +159,7 @@ def __post_process_multiple_document_docker_compose(self, yaml_document):
                 # so if we modify it while looping on "documents"
                 # the output will be affected (probably truncated for some outputs)
                 main_document = document  # we need to save the main document later
-        yaml_handler.dump(main_document, yaml_document)
+        yaml_rt.dump(main_document, yaml_document)
 
     def __get_default_helm_values_with_secrets(self, helm_values):
         helm_values = copy.deepcopy(helm_values)
@@ -276,7 +274,7 @@ def create_app_values_spec(self, app_name: str, app_path: Path, base_image_name:
                 logging.info(
                     f"Specific environment values template found: {specific_template_path}")
                 with open(specific_template_path) as f:
-                    values_env_specific = yaml.safe_load(f)
+                    values_env_specific = yaml.load(f)
                 values = dict_merge(values, values_env_specific)
 
         if KEY_HARNESS in values and 'name' in values[KEY_HARNESS] and values[KEY_HARNESS]['name']:
@@ -362,7 +360,7 @@ def load_app_values(self, app_name, app_path, helm_values={}):
                 logging.info(
                     f"Specific environment values template found: {specific_template_path}")
                 with open(specific_template_path) as f:
-                    values_env_specific = yaml.safe_load(f)
+                    values_env_specific = yaml.load(f)
                 values = dict_merge(values, values_env_specific)
 
         if KEY_HARNESS in values and 'name' in values[KEY_HARNESS] and values[KEY_HARNESS]['name']:

tools/deployment-cli-tools/ch_cli_tools/helm.py

@@ -3,7 +3,6 @@
 """
 from pathlib import Path
 from typing import Union
-import yaml
 import os
 import logging
 from hashlib import sha1
@@ -12,7 +11,7 @@
 from cloudharness_utils.constants import VALUES_MANUAL_PATH, HELM_CHART_PATH
 from .utils import get_cluster_ip, get_dockerfile_baseimg_args, get_git_commit_hash, get_image_name, image_name_from_dockerfile_path, \
     get_template, merge_to_yaml_file, dict_merge, app_name_from_path, \
-    find_dockerfiles_paths
+    find_dockerfiles_paths, yaml
 
 from .models import HarnessMainConfig
 
@@ -352,7 +351,7 @@ def create_app_values_spec(self, app_name: str, app_path: Path, base_image_name:
             if specific_template_path.exists():
                 logging.info(f"Specific environment values template found: {specific_template_path}")
                 with specific_template_path.open("r") as f:
-                    values_env_specific = yaml.safe_load(f)
+                    values_env_specific = yaml.load(f)
                 values = dict_merge(values, values_env_specific)
 
         if KEY_HARNESS in values and 'name' in values[KEY_HARNESS] and values[KEY_HARNESS]['name']:
@@ -426,7 +425,7 @@ def load_app_values(self, app_name, app_path, helm_values={}):
                 logging.info(
                     "Specific environment values template found: " + specific_template_path)
                 with open(specific_template_path) as f:
-                    values_env_specific = yaml.safe_load(f)
+                    values_env_specific = yaml.load(f)
                 values = dict_merge(values, values_env_specific)
 
         if KEY_HARNESS in values and 'name' in values[KEY_HARNESS] and values[KEY_HARNESS]['name']:

tools/deployment-cli-tools/ch_cli_tools/openapi.py

@@ -17,7 +17,7 @@
 from ch_cli_tools.manifest import get_manifest
 
 from . import HERE
-from .utils import confirm, copymergedir, replace_in_file, replaceindir, to_python_module, get_apps_paths
+from .utils import confirm, copymergedir, replace_in_file, replaceindir, to_python_module, get_apps_paths, yaml
 
 CODEGEN = os.path.join(HERE, 'bin', 'openapi-generator-cli.jar')
 APPLICATIONS_SRC_PATH = os.path.join('applications')
@@ -147,7 +147,6 @@ def generate_ts_client(openapi_file, app_name=""):
 
 
 def json2yaml(json_filename, yaml_file=None):
-    import yaml
     if yaml_file is None:
         yaml_file = str(json_filename).replace('.json', '.yaml')
     with open(json_filename, 'r') as json_filename:

tools/deployment-cli-tools/ch_cli_tools/utils.py

@@ -23,7 +23,18 @@
     APPS_PATH, EXCLUDE_PATHS
 from . import CH_ROOT
 
+# Single YAML handler for the whole toolchain: ruamel.yaml.
+# `yaml` loads/dumps plain Python types; block style is forced so that generated
+# files never come out as inline `{a: 1, b: 2}` flow mappings.
 yaml = YAML(typ='safe')
+yaml.default_flow_style = False
+
+# Round trip handler, for the cases where the key order of the source document
+# (or of the dumped dictionary) must be preserved: the safe representer above
+# sorts mapping keys alphabetically.
+yaml_rt = YAML()
+yaml_rt.default_flow_style = False
+
 BASE_TEMPLATES_PATH = CH_ROOT
 
 

tools/deployment-cli-tools/requirements.txt

@@ -1,7 +1,6 @@
 docker
 six
 ruamel.yaml
-oyaml
 cloudharness_model
 cloudharness_utils
 dirhash

tools/deployment-cli-tools/setup.py

@@ -22,7 +22,6 @@
 
 REQUIREMENTS = [
     'ruamel.yaml',
-    'oyaml',
     'docker',
     'six',
     'cloudharness_model',

tools/deployment-cli-tools/tests/test_dockercompose.py

@@ -4,7 +4,7 @@
 import pytest
 import shutil
 import subprocess
-import yaml
+from ch_cli_tools.utils import yaml
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 RESOURCES = os.path.join(HERE, 'resources')
@@ -110,7 +110,7 @@ def test_compose_gatekeeper_native_configuration_rendering(tmp_path):
     compose_path = out_folder / COMPOSE_PATH
     values_path = compose_path / 'values.yaml'
     with open(values_path, 'r') as values_file:
-        values = yaml.safe_load(values_file)
+        values = yaml.load(values_file)
 
     values['apps']['samples']['harness']['proxy']['gatekeeper']['configuration'] = {
         'same-site-cookie': 'None',
@@ -126,18 +126,18 @@ def test_compose_gatekeeper_native_configuration_rendering(tmp_path):
 
     def render_proxy_config():
         with open(values_path, 'w') as values_file:
-            yaml.safe_dump(values, values_file)
+            yaml.dump(values, values_file)
         completed = subprocess.run(
             ['helm', 'template', str(compose_path)],
             check=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
         )
-        for document in yaml.safe_load_all(completed.stdout):
+        for document in yaml.load_all(completed.stdout):
             metadata = (document or {}).get('cloudharness-metadata', {})
             if metadata.get('path') == 'resources/generated/samples-gk/proxy.yml':
-                return yaml.safe_load(document['data'])
+                return yaml.load(document['data'])
         raise AssertionError('Could not find the samples Gatekeeper proxy configuration')
 
     tls_config = render_proxy_config()