Propagate directory upload errors without progress - #48242

1ES PT Non-Blocking Error: This pipeline is using the deprecated BinSkim v1.9.5 which will be retired and upgraded to the latest version. Please address this issue to resolve the corresponding S360 item. For 1ES PT users, please visit https://aka.ms/1espt/binskim195 for the instructions to avoid any breaks

Check failure on line 10005 in Build log

azure-pipelines / python - pullrequest

Build log #L10005

PowerShell exited with code '1'.

Check failure on line 11132 in Build log

azure-pipelines / python - pullrequest

Build log #L11132

PowerShell exited with code '1'.

Check failure on line 1 in test_automl_node_in_pipeline_image_multilabel_classification[single]

azure-pipelines / python - pullrequest

test_automl_node_in_pipeline_image_multilabel_classification[single]

TypeError: '<=' not supported between instances of 'Mock' and 'float'

Raw output


            self = <pipeline_job.unittests.test_pipeline_job_entity.TestPipelineJobEntity object at 0x10a933150>
mock_machinelearning_client = MLClient(credential=<Mock spec_set='DefaultAzureCredential' id='4733832784'>,
         subscription_id=test_subscription,
         resource_group_name=test_resource_group,
         workspace_name=test_workspace_name)
mocker = <pytest_mock.plugin.MockerFixture object at 0x11b33e550>
run_type = 'single'
tmp_path = PosixPath('/private/var/folders/pd/2_nlvl1s4k121pdk4d5_2c8m0000gn/T/pytest-of-runner/pytest-1/test_automl_node_in_pipeline_i3')

    @pytest.mark.parametrize("run_type", ["single", "sweep", "automode"])
    def test_automl_node_in_pipeline_image_multilabel_classification(
        self,
        mock_machinelearning_client: MLClient,
        mocker: MockFixture,
        run_type: str,
        tmp_path: Path,
    ):
        test_path = "./tests/test_configs/pipeline_jobs/jobs_with_automl_nodes/onejob_automl_image_multilabel_classification.yml"
    
        test_config = load_yaml(test_path)
        if (run_type == "single") or (run_type == "automode"):
            # Remove search_space and sweep sections from the config
            del test_config["jobs"]["hello_automl_image_multilabel_classification"]["search_space"]
            del test_config["jobs"]["hello_automl_image_multilabel_classification"]["sweep"]
    
        test_yaml_file = tmp_path / "job.yml"
        dump_yaml_to_file(test_yaml_file, test_config)
        job = load_job(source=test_yaml_file)
        assert isinstance(job, PipelineJob)
        node = next(iter(job.jobs.values()))
        assert isinstance(node, ImageClassificationMultilabelJob)
        mocker.patch(
            "azure.ai.ml.operations._operation_orchestrator.OperationOrchestrator.get_asset_arm_id",
            return_value="xxx",
        )
        mocker.patch(
            "azure.ai.ml.operations._job_operations._upload_and_generate_remote_uri",
            return_value="yyy",
        )
>       mock_machinelearning_client.jobs._resolve_arm_id_or_upload_dependencies(job)

tests/pipeline_job/unittests/test_pipeline_job_entity.py:649: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_job_operations.py:1342: in _resolve_arm_id_or_upload_dependencies
    self._resolve_arm_id_or_azureml_id(job, self._orchestrators.get_asset_arm_id)
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_job_operations.py:1633: in _resolve_arm_id_or_azureml_id
    job = self._resolve_arm_id_for_pipeline_job(job, resolver)
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_job_operations.py:1822: in _resolve_arm_id_for_pipeline_job
    self._component_operations._resolve_dependencies_for_pipeline_component_jobs(
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_component_operations.py:1182: in _resolve_dependencies_for_pipeline_component_jobs
    client_key=self._get_client_key(),
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_component_operations.py:1133: in _get_client_key
    self._client_key = "workspace/" + self._get_workspace_key()
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_component_operations.py:1095: in _get_workspace_key
    workspace_rest = self._workspace_operations._operation.get(
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/core/tracing/decorator.py:119: in wrapper_use_tracer
    return func(*args, **kwargs)
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/_restclient/arm_ml_service/operations/_operations.py:7073: 
... [The stack trace has been truncated as it exceeded the maximum allowed size. Please refer to the complete log available in the Test Run attachments for full details.]

Check failure on line 1 in test_automl_node_in_pipeline_image_multilabel_classification[sweep]

azure-pipelines / python - pullrequest

test_automl_node_in_pipeline_image_multilabel_classification[sweep]

TypeError: '<=' not supported between instances of 'Mock' and 'float'

Raw output


            self = <pipeline_job.unittests.test_pipeline_job_entity.TestPipelineJobEntity object at 0x10a9332d0>
mock_machinelearning_client = MLClient(credential=<Mock spec_set='DefaultAzureCredential' id='4930676304'>,
         subscription_id=test_subscription,
         resource_group_name=test_resource_group,
         workspace_name=test_workspace_name)
mocker = <pytest_mock.plugin.MockerFixture object at 0x125e8fc90>
run_type = 'sweep'
tmp_path = PosixPath('/private/var/folders/pd/2_nlvl1s4k121pdk4d5_2c8m0000gn/T/pytest-of-runner/pytest-1/test_automl_node_in_pipeline_i4')

    @pytest.mark.parametrize("run_type", ["single", "sweep", "automode"])
    def test_automl_node_in_pipeline_image_multilabel_classification(
        self,
        mock_machinelearning_client: MLClient,
        mocker: MockFixture,
        run_type: str,
        tmp_path: Path,
    ):
        test_path = "./tests/test_configs/pipeline_jobs/jobs_with_automl_nodes/onejob_automl_image_multilabel_classification.yml"
    
        test_config = load_yaml(test_path)
        if (run_type == "single") or (run_type == "automode"):
            # Remove search_space and sweep sections from the config
            del test_config["jobs"]["hello_automl_image_multilabel_classification"]["search_space"]
            del test_config["jobs"]["hello_automl_image_multilabel_classification"]["sweep"]
    
        test_yaml_file = tmp_path / "job.yml"
        dump_yaml_to_file(test_yaml_file, test_config)
        job = load_job(source=test_yaml_file)
        assert isinstance(job, PipelineJob)
        node = next(iter(job.jobs.values()))
        assert isinstance(node, ImageClassificationMultilabelJob)
        mocker.patch(
            "azure.ai.ml.operations._operation_orchestrator.OperationOrchestrator.get_asset_arm_id",
            return_value="xxx",
        )
        mocker.patch(
            "azure.ai.ml.operations._job_operations._upload_and_generate_remote_uri",
            return_value="yyy",
        )
>       mock_machinelearning_client.jobs._resolve_arm_id_or_upload_dependencies(job)

tests/pipeline_job/unittests/test_pipeline_job_entity.py:649: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_job_operations.py:1342: in _resolve_arm_id_or_upload_dependencies
    self._resolve_arm_id_or_azureml_id(job, self._orchestrators.get_asset_arm_id)
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_job_operations.py:1633: in _resolve_arm_id_or_azureml_id
    job = self._resolve_arm_id_for_pipeline_job(job, resolver)
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_job_operations.py:1822: in _resolve_arm_id_for_pipeline_job
    self._component_operations._resolve_dependencies_for_pipeline_component_jobs(
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_component_operations.py:1182: in _resolve_dependencies_for_pipeline_component_jobs
    client_key=self._get_client_key(),
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_component_operations.py:1133: in _get_client_key
    self._client_key = "workspace/" + self._get_workspace_key()
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/operations/_component_operations.py:1095: in _get_workspace_key
    workspace_rest = self._workspace_operations._operation.get(
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/core/tracing/decorator.py:119: in wrapper_use_tracer
    return func(*args, **kwargs)
../../../.venv/azure-ai-ml/.venv_sdist/lib/python3.11/site-packages/azure/ai/ml/_restclient/arm_ml_service/operations/_operations.py:7073: i
... [The stack trace has been truncated as it exceeded the maximum allowed size. Please refer to the complete log available in the Test Run attachments for full details.]

Check failure on line 1 in test_serialize_patch_no_op

azure-pipelines / python - pullrequest

test_serialize_patch_no_op

FileNotFoundError: [Errno 2] No such file or directory: '/mnt/vss/_work/1/s/sdk/ml/azure-ai-ml/dummy_file2.txt'

Raw output


            self = <job_common.unittests.test_local_job_invoker.TestLocalJobInvoker object at 0x7f2853a64430>

    def test_serialize_patch_no_op(self):
        # Validate that the previously escaped string does not modify if funciton is run
        dummy_file = Path("./dummy_file2.txt").resolve()
        expected_out = (
            """unread strings
        everythng before are ignored
        continue to ignore --snapshots"""
            + ' "[{\\"Id\\": \\"abc-123\\"}]" '
            + """Everything after is ignored"""
        )
        dummy_file.write_text(expected_out)
        # Should change nothing if correctly serialized
        patch_invocation_script_serialization(dummy_file)
    
        assert dummy_file.read_text() == expected_out
>       dummy_file.unlink()

tests/job_common/unittests/test_local_job_invoker.py:60: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = PosixPath('/mnt/vss/_work/1/s/sdk/ml/azure-ai-ml/dummy_file2.txt')
missing_ok = False

    def unlink(self, missing_ok=False):
        """
        Remove this file or link.
        If the path is a directory, use rmdir() instead.
        """
        try:
>           self._accessor.unlink(self)
E           FileNotFoundError: [Errno 2] No such file or directory: '/mnt/vss/_work/1/s/sdk/ml/azure-ai-ml/dummy_file2.txt'

/opt/hostedtoolcache/Python/3.10.20/x64/lib/python3.10/pathlib.py:1206: FileNotFoundError

Check failure on line 1 in test_create_and_get

azure-pipelines / python - pullrequest

test_create_and_get

azure.core.exceptions.ResourceNotFoundError: Operation returned an invalid status 'Not Found'
ErrorCode:None
Content: {"Message":"Unable to find a record for the request PUT https://Sanitized.blob.core.windows.net/azureml-blobstore-5f4ab075-734b-4c1e-b5f9-5434c32651b9/LocalUpload/00000000000000000000000000000000/feature_set/.amlignore\nMethod doesn\u0027t match, request \u003CPUT\u003E record \u003CHEAD\u003E\nUri doesn\u0027t match:\n    request \u003Chttps://Sanitized.blob.core.windows.net/azureml-blobstore-5f4ab075-734b-4c1e-b5f9-5434c32651b9/LocalUpload/00000000000000000000000000000000/feature_set/.amlignore\u003E\n    record  \u003Chttps://Sanitized.blob.core.windows.net/azureml-blobstore-5f4ab075-734b-4c1e-b5f9-5434c32651b9/LocalUpload/00000000000000000000000000000000/feature_set/FeatureSetSpec.yaml\u003E\nHeader differences:\nBody differences:\nRequest has body but record doesn\u0027t\nRemaining Entries:\n0: https://Sanitized.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets?api-version=2023-04-01-preview\n1: https://Sanitized.blob.core.windows.net/azureml-blobstore-5f4ab075-734b-4c1e-b5f9-5434c32651b9/LocalUpload/00000000000000000000000000000000/feature_set/FeatureSetSpec.yaml\n2: https://Sanitized.blob.core.windows.net/azureml-blobstore-5f4ab075-734b-4c1e-b5f9-5434c32651b9/az-ml-artifacts/00000000000000000000000000000000/feature_set/FeatureSetSpec.yaml\n3: https://Sanitized.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/featuresets/e2etest_test_764360407128/versions/1?api-version=2023-10-01\n4: https://Sanitized.azure.com/subscriptions/00000000-0000-0000-0000-000000000/providers/Microsoft.MachineLearningServices/locations/westus/mfeOperationsStatus/fs:5f4ab075-734b-4c1e-b5f9-5434c32651b9:25578f00-beb9-5af8-8e48-cf052ec731d1?api-version=2023-10-01\u0026t=638378907932966530\u0026c=MIIHADCCBeigAwIBAgITfAQMTI1h2_N6jbL4IQAABAxMjTANB

Raw output


            self = <feature_set.e2etests.test_feature_set.TestFeatureSet object at 0x7f6a39e14cd0>
feature_store_client = MLClient(credential=<devtools_testutils.fake_credentials.FakeTokenCredential object at 0x7f6a0eaf8530>,
         subscription_id=00000000-0000-0000-0000-000000000,
         resource_group_name=00000,
         workspace_name=00000)
tmp_path = PosixPath('/tmp/pytest-of-cloudtest/pytest-0/test_create_and_get1')
randstr = <function randstr.<locals>.generate_random_string at 0x7f6a0ea99e40>

    def test_create_and_get(self, feature_store_client: MLClient, tmp_path: Path, randstr: Callable[[], str]) -> None:
        fset_name = f"e2etest_{randstr('fset_name')}"
        fset_description = "Feature set description"
        fs_entity_name = "e2etest_fs_entity"
        version = "1"
    
        params_override = [
            {"name": fset_name},
            {"version": version},
            {"description": fset_description},
        ]
    
        def feature_set_validation(fset: FeatureSet):
            fset.entities = [f"azureml:{fs_entity_name}:{version}"]
            fset_poller = feature_store_client.feature_sets.begin_create_or_update(featureset=fset)
            assert isinstance(fset_poller, LROPoller)
            fset = fset_poller.result()
            assert isinstance(fset, FeatureSet)
            assert fset.name == fset_name
            assert fset.description == fset_description
    
>       fset = verify_entity_load_and_dump(
            load_feature_set,
            feature_set_validation,
            "./tests/test_configs/feature_set/feature_set_e2e.yaml",
            params_override=params_override,
        )[0]

tests/feature_set/e2etests/test_feature_set.py:47: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_utilities/utils.py:256: in verify_entity_load_and_dump
    entity_validation_function(file_entity)
tests/feature_set/e2etests/test_feature_set.py:40: in feature_set_validation
    fset_poller = feature_store_client.feature_sets.begin_create_or_update(featureset=fset)
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/core/tracing/decorator.py:119: in wrapper_use_tracer
    return func(*args, **kwargs)
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/ai/ml/_telemetry/activity.py:288: in wrapper
    return f(*args, **kwargs)
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/ai/ml/operations/_feature_set_operations.py:170: in begin_create_or_update
    featureset_copy, _ = _check_and_upload_path(
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/ai/ml/_artifacts/_artifact_utilities.py:515: in _check_and_upload_path
    uploaded_artifact = _upload_to_datastore(
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/ai/ml/_artifacts/_artifact_utilities.py:393: in _upload_to_datastore
    artifact = upload_artifact(
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/ai/ml/_artifacts/_artifact_utilities.py:252: in upload_artifact
    artifact_info = storage_client.upload(
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/ai/ml/_artifacts/_blob_storage_helper.py:122: in upload
    upload_directory(
../../../.venv/azure-ai-ml/.venv_whl/lib/python3.13/site-packages/azure/ai/ml/_utils/_asset_utils.py:720: in upload_directory
    future.result()  # access result to propagate any exceptions
/opt/hostedtoolcache/Python/3.13.14/x64/lib/python3.13/concurrent/futures/_base.py:453: in result
    return self.__get_result()
/opt/hostedtoolcache/Python/3.13.14/x64/lib/python3.13/concurrent/futures/_base.py:402: in __get_result
    raise self._exception
/opt/hostedtoolcache/Python/3.13.14/x64/lib/python3.13/concurrent/futures/thread.py:59: in run
    result = self.fn
... [The stack trace has been truncated as it exceeded the maximum allowed size. Please refer to the complete log available in the Test Run attachments for full details.]

View more details on Azure Pipelines

Navigation Menu

Uh oh!

Propagate directory upload errors without progress - #48242