diff --git a/.changes/next-release/145405938520458-feature-architecture-89098.json b/.changes/next-release/145405938520458-feature-architecture-89098.json new file mode 100644 index 000000000..63918493f --- /dev/null +++ b/.changes/next-release/145405938520458-feature-architecture-89098.json @@ -0,0 +1,5 @@ +{ + "type": "feature", + "category": "architecture", + "description": "Add support for Arm64 processor ([1808](https://github.com/aws/chalice/issues/1808)]" +} diff --git a/chalice/awsclient.py b/chalice/awsclient.py index 2cf2b3129..5de446753 100644 --- a/chalice/awsclient.py +++ b/chalice/awsclient.py @@ -400,6 +400,7 @@ def create_function( security_group_ids: OptStrList = None, subnet_ids: OptStrList = None, layers: OptStrList = None, + architecture: OptStr = None, ) -> str: # pylint: disable=too-many-locals kwargs: Dict[str, Any] = { @@ -426,6 +427,8 @@ def create_function( ) if layers is not None: kwargs['Layers'] = layers + if architecture is not None: + kwargs['Architectures'] = [architecture] arn, state = self._create_lambda_function(kwargs) # Avoid the GetFunctionConfiguration call unless # we're not immediately active. @@ -906,6 +909,7 @@ def update_function( subnet_ids: OptStrList = None, security_group_ids: OptStrList = None, layers: OptStrList = None, + architecture: OptStr = None, ) -> Dict[str, Any]: """Update a Lambda function's code and configuration. @@ -914,7 +918,9 @@ def update_function( the targeted lambda function. """ return_value = self._update_function_code( - function_name=function_name, zip_contents=zip_contents + function_name=function_name, + zip_contents=zip_contents, + architecture=architecture, ) self._update_function_config( environment_variables=environment_variables, @@ -933,12 +939,21 @@ def update_function( return return_value def _update_function_code( - self, function_name: str, zip_contents: str + self, + function_name: str, + zip_contents: str, + architecture: OptStr = None, ) -> Dict[str, Any]: lambda_client = self._client('lambda') + kwargs: Dict[str, Any] = { + 'FunctionName': function_name, + 'ZipFile': zip_contents + } + if architecture is not None: + kwargs['Architectures'] = [architecture] try: result = lambda_client.update_function_code( - FunctionName=function_name, ZipFile=zip_contents + **kwargs ) except _REMOTE_CALL_ERRORS as e: context = LambdaErrorContext( diff --git a/chalice/config.py b/chalice/config.py index 6f7d571ea..20e0571f8 100644 --- a/chalice/config.py +++ b/chalice/config.py @@ -273,6 +273,16 @@ def lambda_timeout(self) -> int: varies_per_chalice_stage=True, varies_per_function=True) + @property + def lambda_architecture(self) -> str: + value = self._chain_lookup( + 'lambda_architecture', + varies_per_chalice_stage=True, + varies_per_function=True) + if value is None: + return 'x86_64' + return value + @property def automatic_layer(self) -> bool: v = self._chain_lookup('automatic_layer', diff --git a/chalice/deploy/appgraph.py b/chalice/deploy/appgraph.py index 4572fffb2..734c55206 100644 --- a/chalice/deploy/appgraph.py +++ b/chalice/deploy/appgraph.py @@ -573,6 +573,7 @@ def _build_lambda_function( subnet_ids=subnet_ids, reserved_concurrency=config.reserved_concurrency, layers=lambda_layers, + architecture=config.lambda_architecture, managed_layer=self._get_managed_lambda_layer(config), xray=config.xray_enabled, ) diff --git a/chalice/deploy/deployer.py b/chalice/deploy/deployer.py index de1c17cf7..ae85a38d6 100644 --- a/chalice/deploy/deployer.py +++ b/chalice/deploy/deployer.py @@ -451,7 +451,9 @@ def handle_deploymentpackage(self, config, resource): # type: (Config, models.DeploymentPackage) -> None if isinstance(resource.filename, models.Placeholder): zip_filename = self._packager.create_deployment_package( - config.project_dir, config.lambda_python_version) + config.project_dir, + config.lambda_python_version, + architecture=config.lambda_architecture) resource.filename = zip_filename @@ -472,7 +474,9 @@ def handle_lambdafunction(self, config, resource): if isinstance(resource.deployment_package.filename, models.Placeholder): zip_filename = self._lambda_packager.create_deployment_package( - config.project_dir, config.lambda_python_version + config.project_dir, + config.lambda_python_version, + architecture=config.lambda_architecture ) resource.deployment_package.filename = zip_filename if resource.managed_layer is not None and \ @@ -489,7 +493,9 @@ def handle_lambdalayer(self, config, resource): models.Placeholder): try: zip_filename = self._layer_packager.create_deployment_package( - config.project_dir, config.lambda_python_version + config.project_dir, + config.lambda_python_version, + architecture=config.lambda_architecture, ) resource.deployment_package.filename = zip_filename except EmptyPackageError: diff --git a/chalice/deploy/models.py b/chalice/deploy/models.py index 887f5a120..b25daef32 100644 --- a/chalice/deploy/models.py +++ b/chalice/deploy/models.py @@ -203,6 +203,7 @@ class LambdaFunction(ManagedModel): reserved_concurrency: int # These are customer created layers. layers: List[str] + architecture: str managed_layer: Opt[LambdaLayer] = None log_group: Opt[LogGroup] = None diff --git a/chalice/deploy/packager.py b/chalice/deploy/packager.py index b2df2ae25..c16924d0c 100644 --- a/chalice/deploy/packager.py +++ b/chalice/deploy/packager.py @@ -91,7 +91,10 @@ def __init__( self._ui = ui def create_deployment_package( - self, project_dir: str, python_version: str + self, + project_dir: str, + python_version: str, + architecture: str = 'x86_64', ) -> str: raise NotImplementedError("create_deployment_package") @@ -116,7 +119,10 @@ def _add_vendor_files( zipped.write(full_path, zip_path) def deployment_package_filename( - self, project_dir: str, python_version: str + self, + project_dir: str, + python_version: str, + architecture: str = 'x86_64', ) -> str: # Computes the name of the deployment package zipfile # based on a hash of the requirements file. @@ -126,10 +132,18 @@ def deployment_package_filename( # to the end of the filename since the the dependencies may not change # but if the python version changes then the dependencies need to be # re-downloaded since they will not be compatible. - return self._deployment_package_filename(project_dir, python_version) + return self._deployment_package_filename( + project_dir, + python_version, + architecture=architecture + ) def _deployment_package_filename( - self, project_dir: str, python_version: str, prefix: str = '' + self, + project_dir: str, + python_version: str, + prefix: str = '', + architecture: str = 'x86_64', ) -> str: requirements_filename = self._get_requirements_filename(project_dir) hash_contents = self._hash_project_dir( @@ -137,7 +151,9 @@ def _deployment_package_filename( self._osutils.joinpath(project_dir, self._VENDOR_DIR), project_dir, ) - filename = '%s%s-%s.zip' % (prefix, hash_contents, python_version) + filename = '%s%s-%s-%s.zip' % ( + prefix, hash_contents, python_version, architecture + ) deployment_package_filename = self._osutils.joinpath( project_dir, '.chalice', 'deployments', filename ) @@ -289,11 +305,15 @@ def _build_python_dependencies( python_version: str, requirements_filepath: str, site_packages_dir: str, + architecture: str = 'x86_64', ) -> None: try: abi = self._RUNTIME_TO_ABI[python_version] self._dependency_builder.build_site_packages( - abi, requirements_filepath, site_packages_dir + abi, + requirements_filepath, + site_packages_dir, + architecture=architecture, ) except MissingDependencyError as e: missing_packages = '\n'.join([p.identifier for p in e.missing]) @@ -302,13 +322,16 @@ def _build_python_dependencies( class LambdaDeploymentPackager(BaseLambdaDeploymentPackager): def create_deployment_package( - self, project_dir: str, python_version: str + self, + project_dir: str, + python_version: str, + architecture: str = 'x86_64' ) -> str: msg = "Creating deployment package." self._ui.write("%s\n" % msg) logger.debug(msg) package_filename = self.deployment_package_filename( - project_dir, python_version + project_dir, python_version, architecture=architecture ) if self._osutils.file_exists(package_filename): self._ui.write("Reusing existing deployment package.\n") @@ -319,7 +342,10 @@ def create_deployment_package( project_dir ) self._build_python_dependencies( - python_version, requirements_filepath, site_packages_dir=tmpdir + python_version, + requirements_filepath, + site_packages_dir=tmpdir, + architecture=architecture, ) with self._osutils.open_zip( package_filename, 'w', self._osutils.ZIP_DEFLATED @@ -334,7 +360,10 @@ def create_deployment_package( class AppOnlyDeploymentPackager(BaseLambdaDeploymentPackager): def create_deployment_package( - self, project_dir: str, python_version: str + self, + project_dir: str, + python_version: str, + architecture: str = 'x86_64', ) -> str: msg = "Creating app deployment package." self._ui.write("%s\n" % msg) @@ -353,15 +382,23 @@ def create_deployment_package( return package_filename def deployment_package_filename( - self, project_dir: str, python_version: str + self, + project_dir: str, + python_version: str, + architecture: str = 'x86_64', ) -> str: return self._deployment_package_filename( - project_dir, python_version, prefix='appcode-' + project_dir, + python_version, + prefix='appcode-', + architecture=architecture ) def _deployment_package_filename( - self, project_dir: str, python_version: str, prefix: str = '' + self, project_dir: str, python_version: str, prefix: str = '', + architecture: str = 'x86_64' ) -> str: + # architecture is unused; this zip only contains app source code. h = hashlib.md5(b'') for filename, _ in self._iter_app_filenames(project_dir): with self._osutils.open(filename, 'rb') as f: @@ -384,13 +421,16 @@ class LayerDeploymentPackager(BaseLambdaDeploymentPackager): _PREFIX = 'python/lib/%s/site-packages' def create_deployment_package( - self, project_dir: str, python_version: str + self, + project_dir: str, + python_version: str, + architecture: str = 'x86_64', ) -> str: msg = "Creating shared layer deployment package." self._ui.write("%s\n" % msg) logger.debug(msg) package_filename = self.deployment_package_filename( - project_dir, python_version + project_dir, python_version, architecture=architecture ) self._create_output_dir_if_needed(package_filename) if self._osutils.file_exists(package_filename): @@ -403,7 +443,10 @@ def create_deployment_package( project_dir ) self._build_python_dependencies( - python_version, requirements_filepath, site_packages_dir=tmpdir + python_version, + requirements_filepath, + site_packages_dir=tmpdir, + architecture=architecture, ) with self._osutils.open_zip( package_filename, 'w', self._osutils.ZIP_DEFLATED @@ -439,14 +482,23 @@ def _check_valid_package(self, package_filename: str) -> None: raise EmptyPackageError(package_filename) def deployment_package_filename( - self, project_dir: str, python_version: str + self, project_dir: str, + python_version: str, + architecture: str = 'x86_64' ) -> str: return self._deployment_package_filename( - project_dir, python_version, prefix='managed-layer-' + project_dir, + python_version, + prefix='managed-layer-', + architecture=architecture ) def _deployment_package_filename( - self, project_dir: str, python_version: str, prefix: str = '' + self, + project_dir: str, + python_version: str, + prefix: str = '', + architecture: str = 'x86_64', ) -> str: requirements_filename = self._get_requirements_filename(project_dir) if not self._osutils.file_exists(requirements_filename): @@ -463,7 +515,9 @@ def _deployment_package_filename( if self._osutils.directory_exists(vendor_dir): self._hash_vendor_dir(vendor_dir, h) hash_contents = h.hexdigest() - filename = '%s%s-%s.zip' % (prefix, hash_contents, python_version) + filename = '%s%s-%s-%s.zip' % ( + prefix, hash_contents, python_version, architecture + ) deployment_package_filename = self._osutils.joinpath( project_dir, '.chalice', 'deployments', filename ) @@ -483,7 +537,6 @@ class DependencyBuilder(object): packager. """ - _ADDITIONAL_COMPATIBLE_PLATFORM = {'any', 'linux_x86_64'} _MANYLINUX_LEGACY_MAP = { 'manylinux1_x86_64': 'manylinux_2_5_x86_64', 'manylinux2010_x86_64': 'manylinux_2_12_x86_64', @@ -522,13 +575,16 @@ def __init__( self._pip = pip_runner def _is_compatible_wheel_filename( - self, expected_abi: str, filename: str + self, expected_abi: str, filename: str, architecture: str = 'x86_64', ) -> bool: wheel = filename[:-4] all_compatibility_tags = self._iter_all_compatibility_tags(wheel) for implementation, abi, platform in all_compatibility_tags: # Verify platform is compatible - if not self._is_compatible_platform_tag(expected_abi, platform): + if not self._is_compatible_platform_tag( + expected_abi, + platform, + architecture=architecture): continue # Verify that the ABI is compatible with lambda. Either none or the # correct type for the python version cp27mu for py27 and cp36m for @@ -547,7 +603,7 @@ def _is_compatible_wheel_filename( return False def _is_compatible_platform_tag( - self, expected_abi: str, platform: str + self, expected_abi: str, platform: str, architecture: str = 'x86_64', ) -> bool: # From PEP 600, the new manylinux tag is # manylinux_${GLIBCMAJOR}_${GLIBCMINOR}_${ARCH} @@ -556,7 +612,9 @@ def _is_compatible_platform_tag( # legacy manylinux formats to the new perennial format. # Then we verify that the glibc version is compatible with the version # on the Lambda runtime (from _RUNTIME_GLIBC). - if platform in self._ADDITIONAL_COMPATIBLE_PLATFORM: + arch = 'aarch64' if architecture == 'arm64' else 'x86_64' + compatible_platforms = {'any', 'linux_%s' % arch} + if platform in compatible_platforms: logger.debug("Found compatible platform tag: %s", platform) return True elif platform.startswith('manylinux'): @@ -567,6 +625,9 @@ def _is_compatible_platform_tag( if m is None: return False tag_major, tag_minor = [int(x) for x in m.groups()[:2]] + tag_arch = m.group(3) + if tag_arch != arch: + return False runtime_major, runtime_minor = self._RUNTIME_GLIBC.get( expected_abi, self._DEFAULT_GLIBC ) @@ -632,7 +693,8 @@ def _download_all_dependencies( return deps def _download_binary_wheels( - self, abi: str, packages: Set[Package], directory: str + self, abi: str, packages: Set[Package], directory: str, + architecture: str = 'x86_64', ) -> None: # Try to get binary wheels for each package that isn't compatible. logger.debug("Downloading manylinux wheels: %s", packages) @@ -640,22 +702,25 @@ def _download_binary_wheels( abi, [pkg.identifier for pkg in packages], directory, - self._get_pip_platforms(abi), + self._get_pip_platforms(abi, architecture=architecture), ) - def _get_pip_platforms(self, abi: str) -> List[str]: + def _get_pip_platforms( + self, abi: str, + architecture: str = 'x86_64') -> List[str]: # Pip treats --platform as a literal tag and does not auto-include # lower manylinux_X_Y versions, so we enumerate every glibc minor up # to the runtime's. The trailing manylinux2014_x86_64 alias engages # pip's legacy compatibility hierarchy (manylinux1, manylinux2010). + arch = 'aarch64' if architecture == 'arm64' else 'x86_64' runtime_major, runtime_minor = self._RUNTIME_GLIBC.get( abi, self._DEFAULT_GLIBC ) platforms = [ - 'manylinux_%s_%s_x86_64' % (runtime_major, minor) + 'manylinux_%s_%s_%s' % (runtime_major, minor, arch) for minor in range(17, runtime_minor + 1) ] - platforms.append('manylinux2014_x86_64') + platforms.append('manylinux2014_%s' % arch) return platforms def _download_sdists(self, packages: Set[Package], directory: str) -> None: @@ -687,7 +752,7 @@ def _build_sdists( self._pip.build_wheel(path_to_sdist, directory, compile_c) def _categorize_wheel_files( - self, abi: str, directory: str + self, abi: str, directory: str, architecture: str = 'x86_64', ) -> Tuple[Set[Package], Set[Package]]: final_wheels = [ Package(directory, filename) @@ -697,13 +762,20 @@ def _categorize_wheel_files( compatible_wheels, incompatible_wheels = set(), set() for wheel in final_wheels: - if self._is_compatible_wheel_filename(abi, wheel.filename): + if self._is_compatible_wheel_filename( + abi, + wheel.filename, + architecture=architecture): compatible_wheels.add(wheel) else: incompatible_wheels.add(wheel) return compatible_wheels, incompatible_wheels - def _categorize_deps(self, abi: str, deps: Set[Package]) -> Any: + def _categorize_deps( + self, + abi: str, + deps: Set[Package], + architecture: str = 'x86_64') -> Any: compatible_wheels = set() incompatible_wheels = set() sdists = set() @@ -711,15 +783,20 @@ def _categorize_deps(self, abi: str, deps: Set[Package]) -> Any: if package.dist_type == 'sdist': sdists.add(package) else: - if self._is_compatible_wheel_filename(abi, package.filename): + if self._is_compatible_wheel_filename( + abi, + package.filename, + architecture=architecture): compatible_wheels.add(package) else: incompatible_wheels.add(package) return sdists, compatible_wheels, incompatible_wheels def _download_dependencies( - self, abi: str, directory: str, requirements_filename: str - ) -> Tuple[Set[Package], Set[Package]]: + self, abi: str, + directory: str, + requirements_filename: str, + architecture: str = 'x86_64') -> Tuple[Set[Package], Set[Package]]: # Download all dependencies we can, letting pip choose what to # download. # deps should represent the best effort we can make to gather all the @@ -739,7 +816,7 @@ def _download_dependencies( # wheel file may not be compatible with lambda. Pure python wheels # still will be compatible because they have no platform dependencies. sdists, compatible_wheels, incompatible_wheels = self._categorize_deps( - abi, deps + abi, deps, architecture=architecture ) logger.debug("Compatible wheels for Lambda: %s", compatible_wheels) logger.debug( @@ -752,7 +829,11 @@ def _download_dependencies( # For these packages we need to explicitly try to download a # compatible wheel file. missing_wheels = sdists.union(incompatible_wheels) - self._download_binary_wheels(abi, missing_wheels, directory) + self._download_binary_wheels( + abi, + missing_wheels, + directory, + architecture=architecture) # Re-count the wheel files after the second download pass. Anything # that has an sdist but not a valid wheel file is still not going to @@ -764,7 +845,7 @@ def _download_dependencies( # incompatible wheel file and no sdist. So we need to get any missing # sdists before we can build them. compatible_wheels, incompatible_wheels = self._categorize_wheel_files( - abi, directory + abi, directory, architecture=architecture ) # The self._download_binary_wheels() can now introduce duplicate # entries. For example, if we download a macOS whl at first but @@ -791,7 +872,7 @@ def _download_dependencies( # building the package and trying to sever its ability to find a C # compiler. compatible_wheels, incompatible_wheels = self._categorize_wheel_files( - abi, directory + abi, directory, architecture=architecture ) logger.debug( "compatible after building wheels (C compiling): %s", @@ -805,7 +886,7 @@ def _download_dependencies( # can do about any missing wheel files. We tried downloading a # compatible version directly and building from source. compatible_wheels, incompatible_wheels = self._categorize_wheel_files( - abi, directory + abi, directory, architecture=architecture ) logger.debug( "compatible after building wheels (no C compiling): %s", @@ -875,12 +956,14 @@ def _install_wheels( self._install_purelib_and_platlib(wheel, dst_dir) def build_site_packages( - self, abi: str, requirements_filepath: str, target_directory: str + self, abi: str, requirements_filepath: str, target_directory: str, + architecture: str = 'x86_64', ) -> None: if self._has_at_least_one_package(requirements_filepath): with self._osutils.tempdir() as tempdir: wheels, packages_without_wheels = self._download_dependencies( - abi, tempdir, requirements_filepath + abi, tempdir, requirements_filepath, + architecture=architecture, ) self._install_wheels(tempdir, target_directory, wheels) if packages_without_wheels: diff --git a/chalice/deploy/planner.py b/chalice/deploy/planner.py index a22072584..0b0f18e2b 100644 --- a/chalice/deploy/planner.py +++ b/chalice/deploy/planner.py @@ -491,7 +491,8 @@ def _plan_lambdafunction(self, resource): 'memory_size': resource.memory_size, 'security_group_ids': resource.security_group_ids, 'subnet_ids': resource.subnet_ids, - 'layers': layers + 'layers': layers, + 'architecture': resource.architecture, } api_calls.extend([ @@ -523,7 +524,8 @@ def _plan_lambdafunction(self, resource): 'memory_size': resource.memory_size, 'security_group_ids': resource.security_group_ids, 'subnet_ids': resource.subnet_ids, - 'layers': layers + 'layers': layers, + 'architecture': resource.architecture, } api_calls.extend([ (models.APICall( diff --git a/chalice/deploy/validate.py b/chalice/deploy/validate.py index 003e0f258..0e5d539e0 100644 --- a/chalice/deploy/validate.py +++ b/chalice/deploy/validate.py @@ -1,7 +1,7 @@ import sys import warnings -from typing import Dict, List, Set, Iterator, Optional, Any # noqa +from typing import Dict, List, Set, Iterator, Optional, Any, Tuple # noqa from chalice import app # noqa from chalice.config import Config # noqa @@ -50,6 +50,7 @@ def validate_configuration(config): validate_resource_policy(config) validate_sqs_configuration(config.chalice_app) validate_environment_variables_type(config) + validate_lambda_architecture(config) def validate_resource_policy(config): @@ -277,3 +278,23 @@ def _validate_environment_variables(environment_variables): raise ValueError("Environment variable values must be strings, " "got 'type' %s for key '%s'" % ( type(value).__name__, key)) + + +def validate_lambda_architecture(config): + # type: (Config) -> None + valid_architectures = ('x86_64', 'arm64') + _check_architecture( + config.lambda_architecture, valid_architectures) + for name in _get_all_function_names(config.chalice_app): + _check_architecture( + config.scope(config.chalice_stage, name).lambda_architecture, + valid_architectures) + + +def _check_architecture(architecture, valid_architectures): + # type: (str, Tuple[str, ...]) -> None + if architecture not in valid_architectures: + raise ValueError( + "Invalid lambda_architecture value: '%s'. " + "Must be one of: %s" % ( + architecture, ", ".join(valid_architectures))) diff --git a/chalice/package.py b/chalice/package.py index 06c4e01d6..f37ac0d94 100644 --- a/chalice/package.py +++ b/chalice/package.py @@ -282,6 +282,9 @@ def _generate_lambdafunction(self, resource, template): } # type: Dict[str, Any] lambdafunction_definition['Properties'].update(layers_config) + architectures_config = {'Architectures': [resource.architecture]} + lambdafunction_definition['Properties'].update(architectures_config) + if resource.log_group is not None: num_days = resource.log_group.retention_in_days log_name = self._register_cfn_resource_name( @@ -1253,6 +1256,7 @@ def _generate_lambdafunction(self, resource, template): func_definition = { 'function_name': resource.function_name, 'runtime': resource.runtime, + 'architectures': [resource.architecture], 'handler': resource.handler, 'memory_size': resource.memory_size, 'tags': resource.tags, diff --git a/tests/functional/test_awsclient.py b/tests/functional/test_awsclient.py index 1a69fb1e4..30654dce8 100644 --- a/tests/functional/test_awsclient.py +++ b/tests/functional/test_awsclient.py @@ -1716,7 +1716,7 @@ def test_create_function_succeeds_first_try(self, stubbed_session): Runtime='python2.7', Code={'ZipFile': b'foo'}, Handler='app.app', - Role='myarn' + Role='myarn', ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -1748,7 +1748,7 @@ def test_create_function_wait_for_active_state(self, stubbed_session, Runtime='python2.7', Code={'ZipFile': b'foo'}, Handler='app.app', - Role='myarn' + Role='myarn', ).returns({'FunctionArn': 'arn:12345:name', 'State': 'Pending'}) client.get_function_configuration( FunctionName='name', @@ -1771,7 +1771,7 @@ def test_create_function_with_environment_variables(self, stubbed_session): Code={'ZipFile': b'foo'}, Handler='app.app', Role='myarn', - Environment={'Variables': {'FOO': 'BAR'}} + Environment={'Variables': {'FOO': 'BAR'}}, ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -1788,7 +1788,7 @@ def test_create_function_with_tags(self, stubbed_session): Code={'ZipFile': b'foo'}, Handler='app.app', Role='myarn', - Timeout=240 + Timeout=240, ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -1804,7 +1804,7 @@ def test_create_function_with_timeout(self, stubbed_session): Code={'ZipFile': b'foo'}, Handler='app.app', Role='myarn', - Tags={'mykey': 'myvalue'} + Tags={'mykey': 'myvalue'}, ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -1820,7 +1820,7 @@ def test_create_function_with_memory_size(self, stubbed_session): Code={'ZipFile': b'foo'}, Handler='app.app', Role='myarn', - MemorySize=256 + MemorySize=256, ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -1839,7 +1839,7 @@ def test_create_function_with_vpc_config(self, stubbed_session): VpcConfig={ 'SecurityGroupIds': ['sg1', 'sg2'], 'SubnetIds': ['sn1', 'sn2'] - } + }, ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -1858,7 +1858,7 @@ def test_create_function_with_layers(self, stubbed_session): Code={'ZipFile': b'foo'}, Handler='app.app', Role='myarn', - Layers=layers + Layers=layers, ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -2121,6 +2121,22 @@ def test_raises_large_deployment_error_for_too_large_unzip( 'app.app') stubbed_session.verify_stubs() + def test_create_function_with_arm64_architecture(self, stubbed_session): + stubbed_session.stub('lambda').create_function( + FunctionName='name', + Runtime='python3.12', + Code={'ZipFile': b'foo'}, + Handler='app.app', + Role='myarn', + Architectures=['arm64'], + ).returns(self.SUCCESS_RESPONSE) + stubbed_session.activate_stubs() + awsclient = TypedAWSClient(stubbed_session) + assert awsclient.create_function( + 'name', 'myarn', b'foo', runtime='python3.12', + handler='app.app', architecture='arm64') == 'arn:12345:name' + stubbed_session.verify_stubs() + class TestUpdateLambdaFunction(object): @@ -2131,7 +2147,9 @@ class TestUpdateLambdaFunction(object): def test_always_update_function_code(self, stubbed_session): lambda_client = stubbed_session.stub('lambda') lambda_client.update_function_code( - FunctionName='name', ZipFile=b'foo').returns(self.SUCCESS_RESPONSE) + FunctionName='name', + ZipFile=b'foo', + ).returns(self.SUCCESS_RESPONSE) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) awsclient.update_function('name', b'foo') @@ -2222,6 +2240,18 @@ def test_update_function_with_layers_config(self, stubbed_session): ) stubbed_session.verify_stubs() + def test_update_function_with_arm64_architecture(self, stubbed_session): + lambda_client = stubbed_session.stub('lambda') + lambda_client.update_function_code( + FunctionName='name', + ZipFile=b'foo', + Architectures=['arm64'] + ).returns(self.SUCCESS_RESPONSE) + stubbed_session.activate_stubs() + awsclient = TypedAWSClient(stubbed_session) + awsclient.update_function('name', b'foo', architecture='arm64') + stubbed_session.verify_stubs() + def test_update_function_with_adding_tags(self, stubbed_session): function_arn = 'arn' diff --git a/tests/functional/test_deployer.py b/tests/functional/test_deployer.py index 1bfd81c6c..655c7a037 100644 --- a/tests/functional/test_deployer.py +++ b/tests/functional/test_deployer.py @@ -267,7 +267,7 @@ def test_py_deps_in_layer_package(tmpdir, layer_packager): extra_package = vendor.mkdir('mypackage') extra_package.join('__init__.py').write('# Test package') name = packager.create_deployment_package( - str(appdir), 'python3.11') + str(appdir), 'python3.11', architecture='x86_64') assert os.path.basename(name).startswith('managed-layer-') with zipfile.ZipFile(name) as f: prefix = 'python/lib/python3.11/site-packages' @@ -276,8 +276,9 @@ def test_py_deps_in_layer_package(tmpdir, layer_packager): _assert_not_in_zip('%s/chalicelib/__init__.py' % prefix, f) _assert_not_in_zip('%s/app.py' % prefix, f) deps_builder.build_site_packages.assert_called_with( - 'cp311', str(appdir.join('requirements.txt')), mock.ANY - ) + 'cp311', + str(appdir.join('requirements.txt')), + mock.ANY, architecture='x86_64') def test_empty_layer_package_raises_error(tmpdir, layer_packager): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index e109bb415..33650928c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -245,6 +245,7 @@ def lambda_function(): security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, xray=None, ) diff --git a/tests/unit/deploy/test_appgraph.py b/tests/unit/deploy/test_appgraph.py index 0a2a023e0..bc4bebf33 100644 --- a/tests/unit/deploy/test_appgraph.py +++ b/tests/unit/deploy/test_appgraph.py @@ -132,6 +132,7 @@ def test_can_build_single_lambda_function_app(self, security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, managed_layer=None, xray=None, @@ -165,6 +166,7 @@ def test_can_build_single_lambda_function_app_with_log_retention( security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, managed_layer=None, xray=None, @@ -201,6 +203,7 @@ def test_can_build_single_lambda_function_app_with_managed_layer( security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', managed_layer=models.LambdaLayer( resource_name='managed-layer', layer_name='lambda-only-dev-managed-layer', @@ -257,6 +260,7 @@ def test_can_build_lambda_function_with_layers(self, security_group_ids=[], subnet_ids=[], layers=layers, + architecture='x86_64', reserved_concurrency=None, xray=None, ) @@ -316,6 +320,7 @@ def foo(event, context): security_group_ids=['sg1', 'sg2'], subnet_ids=['sn1', 'sn2'], layers=[], + architecture='x86_64', reserved_concurrency=None, xray=None, ) @@ -378,6 +383,7 @@ def test_can_build_lambda_function_app_with_reserved_concurrency( security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=5, xray=None, ) diff --git a/tests/unit/deploy/test_deployer.py b/tests/unit/deploy/test_deployer.py index c1398f4f7..f3309dc89 100644 --- a/tests/unit/deploy/test_deployer.py +++ b/tests/unit/deploy/test_deployer.py @@ -285,6 +285,7 @@ def create_function_resource(name): security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, ) @@ -580,6 +581,7 @@ def test_inject_when_values_are_none(self): security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, ) config = Config.create() @@ -610,6 +612,7 @@ def test_no_injection_when_values_are_set(self): security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, ) config = Config.create() @@ -747,14 +750,16 @@ def test_can_generate_layer_package(self): p.handle(config, function) assert function.deployment_package.filename == 'package.zip' lambda_packager.create_deployment_package.assert_called_with( - '.', config.lambda_python_version - ) + '.', + config.lambda_python_version, + architecture=config.lambda_architecture) assert function.managed_layer.deployment_package.filename == ( 'package-layer.zip' ) layer_packager.create_deployment_package.assert_called_with( - '.', config.lambda_python_version - ) + '.', + config.lambda_python_version, + architecture=config.lambda_architecture) def test_layer_package_not_generated_if_filename_populated(self): generator = mock.Mock(spec=packager.BaseLambdaDeploymentPackager) diff --git a/tests/unit/deploy/test_planner.py b/tests/unit/deploy/test_planner.py index d0d934b63..72a334190 100644 --- a/tests/unit/deploy/test_planner.py +++ b/tests/unit/deploy/test_planner.py @@ -46,6 +46,7 @@ def create_function_resource(name, function_name=None, security_group_ids=[], subnet_ids=[], layers=layers, + architecture='x86_64', reserved_concurrency=None, managed_layer=managed_layer, ) @@ -606,6 +607,7 @@ def test_can_create_function(self): 'security_group_ids': [], 'subnet_ids': [], 'layers': [], + 'architecture': 'x86_64', }, ), models.APICall( @@ -657,7 +659,8 @@ def test_create_function_with_layers(self): 'memory_size': 128, 'security_group_ids': [], 'subnet_ids': [], - 'layers': [Variable('layer_version_arn')] + layers + 'layers': [Variable('layer_version_arn')] + layers, + 'architecture': 'x86_64' }, ), models.APICall( @@ -693,6 +696,7 @@ def test_can_update_lambda_function_code(self): 'security_group_ids': [], 'subnet_ids': [], 'layers': [], + 'architecture': 'x86_64', } expected_params = dict(memory_size=256, **existing_params) expected = [models.APICall( @@ -758,6 +762,7 @@ def test_can_create_function_with_reserved_concurrency(self): 'security_group_ids': [], 'subnet_ids': [], 'layers': [], + 'architecture': 'x86_64', }, ), models.APICall( @@ -799,6 +804,13 @@ def test_can_set_variables_when_needed(self): assert isinstance(role_arn, Variable) assert role_arn.name == 'myrole-dev_role_arn' + def test_can_create_function_with_arm64(self): + function = create_function_resource('function_name') + function.architecture = 'arm64' + self.remote_state.declare_no_resources_exists() + plan = self.determine_plan(function) + assert plan[0].params['architecture'] == 'arm64' + class TestPlanS3Events(BasePlannerTests): def test_can_plan_s3_event(self): diff --git a/tests/unit/deploy/test_validate.py b/tests/unit/deploy/test_validate.py index 3c5b01178..c2c2990a7 100644 --- a/tests/unit/deploy/test_validate.py +++ b/tests/unit/deploy/test_validate.py @@ -15,6 +15,7 @@ from chalice.deploy.validate import validate_feature_flags from chalice.deploy.validate import validate_endpoint_type from chalice.deploy.validate import validate_resource_policy +from chalice.deploy.validate import validate_lambda_architecture from chalice.deploy.validate import ExperimentalFeatureError @@ -387,3 +388,40 @@ def foo(event, context): ) with pytest.raises(ValueError): validate_configuration(config) + + +def test_can_validate_lambda_architecture(sample_app): + config = Config.create( + chalice_app=sample_app, lambda_architecture='aarch64') + with pytest.raises(ValueError): + validate_lambda_architecture(config) + + config = Config.create( + chalice_app=sample_app, lambda_architecture='arm64') + validate_lambda_architecture(config) + + config = Config.create( + chalice_app=sample_app, lambda_architecture='x86_64') + validate_lambda_architecture(config) + + +def test_validate_lambda_architecture_per_function(sample_app): + @sample_app.lambda_function() + def myfunc(event, context): + pass + + config = Config( + chalice_stage='dev', + config_from_disk={ + 'stages': { + 'dev': { + 'lambda_functions': { + 'myfunc': {'lambda_architecture': 'aarch64'} + } + } + } + }, + user_provided_params={'chalice_app': sample_app}, + ) + with pytest.raises(ValueError): + validate_lambda_architecture(config) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 19c8f7d4d..fe942d32b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -700,3 +700,40 @@ def test_can_upgrade_pre10_lambda_functions(self): 'name': 'foo', 'resource_type': 'lambda_function', } + + +class TestLambdaArchitecture: + def test_lambda_architecture_defaults_to_x86_64(self): + c = Config(chalice_stage='dev', config_from_disk={}) + assert c.lambda_architecture == 'x86_64' + + def test_lambda_architecture_from_config(self): + c = Config(chalice_stage='dev', config_from_disk={ + 'lambda_architecture': 'arm64' + }) + assert c.lambda_architecture == 'arm64' + + def test_lambda_architecture_per_stage(self): + c = Config(chalice_stage='prod', config_from_disk={ + 'stages': { + 'prod': { + 'lambda_architecture': 'arm64' + } + } + }) + assert c.lambda_architecture == 'arm64' + + def test_lambda_architecture_per_function(self): + c = Config(chalice_stage='dev', config_from_disk={ + 'stages': { + 'dev': { + 'lambda_functions': { + 'myfunction': { + 'lambda_architecture': 'arm64' + } + } + } + } + }) + new_config = c.scope(chalice_stage='dev', function_name='myfunction') + assert new_config.lambda_architecture == 'arm64' diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py index aed1698ee..a9e310637 100644 --- a/tests/unit/test_package.py +++ b/tests/unit/test_package.py @@ -13,6 +13,9 @@ from chalice.deploy.swagger import SwaggerGenerator from chalice.package import PackageOptions from chalice.utils import OSUtils +from chalice.deploy.packager import ( + DependencyBuilder as DependencyBuilderPackage +) @pytest.fixture @@ -306,6 +309,7 @@ def lambda_function(self): security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, ) @@ -1197,6 +1201,7 @@ def test_sam_injects_policy(self, sample_app): security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, ) template = self.template_gen.generate([function]) @@ -1204,6 +1209,9 @@ def test_sam_injects_policy(self, sample_app): assert cfn_resource == { 'Type': 'AWS::Serverless::Function', 'Properties': { + 'Architectures': [ + 'x86_64', + ], 'CodeUri': 'foo.zip', 'Handler': 'app.app', 'MemorySize': 128, @@ -1294,6 +1302,7 @@ def test_role_arn_inserted_when_necessary(self): security_group_ids=[], subnet_ids=[], layers=[], + architecture='x86_64', reserved_concurrency=None, ) template = self.template_gen.generate([function]) @@ -1301,6 +1310,9 @@ def test_role_arn_inserted_when_necessary(self): assert cfn_resource == { 'Type': 'AWS::Serverless::Function', 'Properties': { + 'Architectures': [ + 'x86_64', + ], 'CodeUri': 'foo.zip', 'Handler': 'app.app', 'MemorySize': 128, @@ -2104,6 +2116,47 @@ def test_merge_can_change_type(self): } +class TestArchitectureSupport: + + def test_pip_platforms_enumerate_aarch64_for_arm64(self): + mock_osutils = mock.Mock(spec=OSUtils) + builder = DependencyBuilderPackage(mock_osutils) + platforms = builder._get_pip_platforms('cp312', architecture='arm64') + # Should contain aarch64, not x86_64 + assert 'manylinux_2_17_aarch64' in platforms + assert 'manylinux2014_aarch64' in platforms + assert 'x86_64' not in str(platforms) + + def test_compatible_wheel_for_arm64(self): + mock_osutils = mock.Mock(spec=OSUtils) + builder = DependencyBuilderPackage(mock_osutils) + assert builder._is_compatible_wheel_filename( + 'cp312', 'foo-1.0-cp312-cp312-manylinux_2_34_aarch64.whl', + architecture='arm64' + ) + # x86_64 wheel should NOT be compatible when targeting arm64 + assert not builder._is_compatible_wheel_filename( + 'cp312', 'foo-1.0-cp312-cp312-manylinux_2_34_x86_64.whl', + architecture='arm64' + ) + + def test_x86_64_wheel_incompatible_with_arm64(self): + mock_osutils = mock.Mock(spec=OSUtils) + builder = DependencyBuilderPackage(mock_osutils) + assert not builder._is_compatible_platform_tag( + 'cp312', 'manylinux_2_34_x86_64', architecture='arm64' + ) + + def test_pure_python_wheel_compatible_with_arm64(self): + mock_osutils = mock.Mock(spec=OSUtils) + builder = DependencyBuilderPackage(mock_osutils) + # Pure python wheels (platform with 'any') work on any architecture + assert builder._is_compatible_wheel_filename( + 'cp312', 'requests-2.28.0-py3-none-any.whl', + architecture='arm64' + ) + + @pytest.mark.parametrize('filename,is_yaml', [ ('extras.yaml', True), ('extras.YAML', True),