Merge pull request #337 from aws/develop

chore: Merge develop into master

Author: hawflau

.appveyor.yml

@@ -51,7 +51,6 @@ for:
       - "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
       - "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
       - "%PYTHON%\\python.exe -m pip install -e ."
-      - "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
       - "gem --version"
       - "gem install bundler -v 1.17.3"
       - "bundler --version"
@@ -76,6 +75,9 @@ for:
       # setup make
       - "choco install make"
 
+      # install dotnet6
+      - ps: "&powershell -NoProfile -ExecutionPolicy unrestricted -Command \"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version 6.0.200 -InstallDir 'C:\\Program Files\\dotnet\\'\""
+
       # Echo final Path
       - "echo %PATH%"
 
@@ -91,7 +93,6 @@ for:
       - sh: "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64"
       - sh: "PATH=$JAVA_HOME/bin:$PATH"
       - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate"
-      - sh: "rvm use 2.5"
       - sh: "nvm install ${nodejs_version}"
       - sh: "npm install [email protected] -g"
       - sh: "npm -v"

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.12.0"
+__version__ = "1.13.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/validator.py

@@ -11,21 +11,18 @@
 LOG = logging.getLogger(__name__)
 
 SUPPORTED_RUNTIMES = {
-    "nodejs10.x": [X86_64],
     "nodejs12.x": [ARM64, X86_64],
     "nodejs14.x": [ARM64, X86_64],
-    "python2.7": [X86_64],
     "python3.6": [X86_64],
     "python3.7": [X86_64],
     "python3.8": [ARM64, X86_64],
     "python3.9": [ARM64, X86_64],
-    "ruby2.5": [X86_64],
     "ruby2.7": [ARM64, X86_64],
     "java8": [ARM64, X86_64],
     "java11": [ARM64, X86_64],
     "go1.x": [ARM64, X86_64],
-    "dotnetcore2.1": [X86_64],
     "dotnetcore3.1": [ARM64, X86_64],
+    "dotnet6": [ARM64, X86_64],
     "provided": [ARM64, X86_64],
 }
 

aws_lambda_builders/workflows/python_pip/DESIGN.md

@@ -49,8 +49,8 @@ def build_dependencies(artifacts_dir_path,
 
     :type runtime: str
     :param runtime: Python version to build dependencies for. This can
-        either be python2.7 or python3.6. These are currently the only
-        supported values.
+        either be python3.6, python3.7, python3.8 or python3.9. These are 
+        currently the only supported values.
 
     :type ui: :class:`lambda_builders.actions.python_pip.utils.UI`
     :param ui: A class that traps all progress information such as status

aws_lambda_builders/workflows/python_pip/packager.py

@@ -81,7 +81,6 @@ def __init__(self, version):
 
 def get_lambda_abi(runtime):
     supported = {
-        "python2.7": "cp27mu",
         "python3.6": "cp36m",
         "python3.7": "cp37m",
         "python3.8": "cp38",
@@ -100,7 +99,7 @@ def __init__(self, runtime, osutils=None, dependency_builder=None, architecture=
 
         :type runtime: str
         :param runtime: Python version to build dependencies for. This can
-            either be python2.7, python3.6 or python3.7. These are currently the
+            either be python3.6, python3.7, python3.8 or python3.9. These are currently the
             only supported values.
 
         :type osutils: :class:`lambda_builders.utils.OSUtils`

aws_lambda_builders/workflows/ruby_bundler/DESIGN.md

@@ -64,7 +64,7 @@ bundle install --deployment
 
 ### sam build --use-container
 
-This command would use some sort of container, such as `lambci/lambda:build-ruby2.5`.
+This command would use some sort of container, such as `public.ecr.aws/sam/build-ruby2.7`.
 
 ```shell
 # exit with error if vendor/bundle and/or .bundle directory exists and is non-empty

tests/integration/workflows/dotnet_clipackage/test_dotnet.py

@@ -25,34 +25,38 @@ def setUp(self):
         self.artifacts_dir = tempfile.mkdtemp()
         self.scratch_dir = tempfile.mkdtemp()
         self.builder = LambdaBuilder(language="dotnet", dependency_manager="cli-package", application_framework=None)
-        self.runtime = "dotnetcore2.1"  # default to 2.1
+        self.runtime = "dotnetcore3.1"
 
     def tearDown(self):
         shutil.rmtree(self.artifacts_dir)
         shutil.rmtree(self.scratch_dir)
 
-    def verify_architecture(self, deps_file_name, expected_architecture):
+    def verify_architecture(self, deps_file_name, expected_architecture, version=None):
         deps_file = pathlib.Path(self.artifacts_dir, deps_file_name)
 
         if not deps_file.exists():
             self.fail("Failed verifying architecture, {} file not found".format(deps_file_name))
 
         with open(str(deps_file)) as f:
             deps_json = json.loads(f.read())
-        version = self.runtime[-3:]
+        version = version or self.runtime[-3:]
         target_name = ".NETCoreApp,Version=v{}/{}".format(version, expected_architecture)
         target = deps_json.get("runtimeTarget").get("name")
 
         self.assertEqual(target, target_name)
 
 
-class TestDotnet21(TestDotnetBase):
+class TestDotnet31(TestDotnetBase):
     """
-    Tests for dotnetcore 2.1
+    Tests for dotnetcore 3.1
     """
 
+    def setUp(self):
+        super(TestDotnet31, self).setUp()
+        self.runtime = "dotnetcore3.1"
+
     def test_with_defaults_file(self):
-        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile2.1")
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1")
 
         self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime)
 
@@ -71,45 +75,62 @@ def test_with_defaults_file(self):
         self.assertEqual(expected_files, output_files)
         self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64")
 
-    def test_require_parameters(self):
-        source_dir = os.path.join(self.TEST_DATA_FOLDER, "RequireParameters")
+    def test_with_defaults_file_x86(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1")
 
         self.builder.build(
-            source_dir,
-            self.artifacts_dir,
-            self.scratch_dir,
-            source_dir,
-            runtime=self.runtime,
-            options={"--framework": "netcoreapp2.1", "--configuration": "Debug"},
+            source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=X86_64
         )
 
         expected_files = {
             "Amazon.Lambda.Core.dll",
             "Amazon.Lambda.Serialization.Json.dll",
             "Newtonsoft.Json.dll",
-            "RequireParameters.deps.json",
-            "RequireParameters.dll",
-            "RequireParameters.pdb",
-            "RequireParameters.runtimeconfig.json",
+            "WithDefaultsFile.deps.json",
+            "WithDefaultsFile.dll",
+            "WithDefaultsFile.pdb",
+            "WithDefaultsFile.runtimeconfig.json",
         }
 
         output_files = set(os.listdir(self.artifacts_dir))
 
         self.assertEqual(expected_files, output_files)
-        self.verify_architecture("RequireParameters.deps.json", "linux-x64")
+        self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64")
+
+    def test_with_defaults_file_arm64(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1")
 
+        self.builder.build(
+            source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=ARM64
+        )
 
-class TestDotnet31(TestDotnetBase):
+        expected_files = {
+            "Amazon.Lambda.Core.dll",
+            "Amazon.Lambda.Serialization.Json.dll",
+            "Newtonsoft.Json.dll",
+            "WithDefaultsFile.deps.json",
+            "WithDefaultsFile.dll",
+            "WithDefaultsFile.pdb",
+            "WithDefaultsFile.runtimeconfig.json",
+        }
+
+        output_files = set(os.listdir(self.artifacts_dir))
+
+        self.assertEqual(expected_files, output_files)
+        self.verify_architecture("WithDefaultsFile.deps.json", "linux-arm64")
+
+
+class TestDotnet6(TestDotnetBase):
     """
-    Tests for dotnetcore 3.1
+    Tests for dotnet 6
     """
 
     def setUp(self):
-        super(TestDotnet31, self).setUp()
-        self.runtime = "dotnetcore3.1"
+        super(TestDotnet6, self).setUp()
+        self.runtime = "dotnet6"
 
     def test_with_defaults_file(self):
-        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1")
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile6")
 
         self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime)
 
@@ -126,10 +147,10 @@ def test_with_defaults_file(self):
         output_files = set(os.listdir(self.artifacts_dir))
 
         self.assertEqual(expected_files, output_files)
-        self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64")
+        self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64", version="6.0")
 
     def test_with_defaults_file_x86(self):
-        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1")
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile6")
 
         self.builder.build(
             source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=X86_64
@@ -148,10 +169,10 @@ def test_with_defaults_file_x86(self):
         output_files = set(os.listdir(self.artifacts_dir))
 
         self.assertEqual(expected_files, output_files)
-        self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64")
+        self.verify_architecture("WithDefaultsFile.deps.json", "linux-x64", version="6.0")
 
     def test_with_defaults_file_arm64(self):
-        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile3.1")
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile6")
 
         self.builder.build(
             source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime, architecture=ARM64
@@ -170,4 +191,4 @@ def test_with_defaults_file_arm64(self):
         output_files = set(os.listdir(self.artifacts_dir))
 
         self.assertEqual(expected_files, output_files)
-        self.verify_architecture("WithDefaultsFile.deps.json", "linux-arm64")
+        self.verify_architecture("WithDefaultsFile.deps.json", "linux-arm64", version="6.0")

tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/Function.cs renamed to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile6/Function.cs

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
     <AWSProjectType>Lambda</AWSProjectType>
   </PropertyGroup>

tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/WithDefaultsFile.csproj renamed to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile6/WithDefaultsFile.csproj

@@ -1,19 +1,16 @@
 {
-  "Information" : [
+  "Information": [
     "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.",
     "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.",
-
     "dotnet lambda help",
-
     "All the command line options for the Lambda command can be specified in this file."
   ],
-
-  "profile":"",
-  "region" : "",
-  "configuration" : "Release",
-  "framework" : "netcoreapp2.1",
-  "function-runtime":"dotnetcore2.1",
-  "function-memory-size" : 256,
-  "function-timeout" : 30,
-  "function-handler" : "WithDefaultsFile::WithDefaultsFile.Function::FunctionHandler"
-}
+  "profile": "",
+  "region": "",
+  "configuration": "Release",
+  "framework": "net6.0",
+  "function-runtime": "dotnet6",
+  "function-memory-size": 256,
+  "function-timeout": 30,
+  "function-handler": "WithDefaultsFile::WithDefaultsFile.Function::FunctionHandler"
+}

tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile2.1/aws-lambda-tools-defaults.json renamed to tests/integration/workflows/dotnet_clipackage/testdata/WithDefaultsFile6/aws-lambda-tools-defaults.json

@@ -43,11 +43,10 @@ def setUp(self):
             language=self.builder.capability.language, major=sys.version_info.major, minor=sys.version_info.minor
         )
         self.runtime_mismatch = {
-            "python3.6": "python2.7",
-            "python3.7": "python2.7",
-            "python2.7": "python3.8",
-            "python3.8": "python2.7",
-            "python3.9": "python2.7",
+            "python3.6": "python3.7",
+            "python3.7": "python3.8",
+            "python3.8": "python3.9",
+            "python3.9": "python3.7",
         }
 
     def tearDown(self):
@@ -70,18 +69,12 @@ def check_architecture_in(self, library, architectures):
             )
         )
 
-    # Temporarily skipping this test in Windows
-    # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue?
-    # TODO: Revisit when we deprecate Python2
-    @skipIf(IS_WINDOWS, "Skip in windows tests")
     def test_must_build_python_project(self):
         self.builder.build(
             self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime
         )
 
-        if self.runtime == "python2.7":
-            expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
-        elif self.runtime == "python3.6":
+        if self.runtime == "python3.6":
             self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
             expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"})
         else:
@@ -109,13 +102,9 @@ def test_must_build_python_project_with_arm_architecture(self):
 
         self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"])
 
-    # Temporarily skipping this test in Windows
-    # Fails and we are not sure why: pip version/multiple Python versions in path/os/pypa issue?
-    # TODO: Revisit when we deprecate Python2
-    @skipIf(IS_WINDOWS, "Skip in windows tests")
     def test_mismatch_runtime_python_project(self):
-        # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python2.7
-        # is still accessible within a python 3 virtualenv.
+        # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python3.7
+        # is still accessible within a python 3.8 virtualenv.
         try:
             self.builder.build(
                 self.source_dir,
@@ -162,11 +151,7 @@ def test_must_fail_to_resolve_dependencies(self):
                 self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_invalid, runtime=self.runtime
             )
 
-        # In Python2 a 'u' is now added to the exception string. To account for this, we see if either one is in the
-        # output
-        message_in_exception = "Invalid requirement: 'boto3=1.19.99'" in str(
-            ctx.exception
-        ) or "Invalid requirement: u'boto3=1.19.99'" in str(ctx.exception)
+        message_in_exception = "Invalid requirement: 'boto3=1.19.99'" in str(ctx.exception)
         self.assertTrue(message_in_exception)
 
     def test_must_log_warning_if_requirements_not_found(self):

tests/integration/workflows/python_pip/test_python_pip.py

@@ -27,7 +27,7 @@ def setUp(self):
         self.dependencies_dir = tempfile.mkdtemp()
         self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
         self.builder = LambdaBuilder(language="ruby", dependency_manager="bundler", application_framework=None)
-        self.runtime = "ruby2.5"
+        self.runtime = "ruby2.7"
 
     def tearDown(self):
         shutil.rmtree(self.artifacts_dir)

tests/integration/workflows/ruby_bundler/test_ruby.py

@@ -22,7 +22,7 @@ def test_validate_with_unsupported_runtime(self):
             validator.validate("/usr/bin/unknown_runtime")
 
     def test_validate_with_runtime_and_incompatible_architecture(self):
-        runtime_list = ["dotnetcore2.1", "nodejs10.x", "ruby2.5", "python3.6", "python3.7", "python2.7"]
+        runtime_list = ["python3.6", "python3.7"]
         for runtime in runtime_list:
             validator = RuntimeValidator(runtime=runtime, architecture="arm64")
             with self.assertRaises(UnsupportedArchitectureError):

tests/unit/test_validator.py

@@ -324,7 +324,7 @@ def test_must_raise_for_incompatible_runtime_and_architecture(self):
             "artifacts_dir",
             "scratch_dir",
             "manifest_path",
-            runtime="python2.7",
+            runtime="python3.7",
             executable_search_paths=[str(pathlib.Path(os.getcwd()).parent)],
             optimizations={"a": "b"},
             options={"c": "d"},
@@ -333,7 +333,7 @@ def test_must_raise_for_incompatible_runtime_and_architecture(self):
         validator_mock = Mock()
         validator_mock.validate = Mock()
         validator_mock.validate = MagicMock(
-            side_effect=UnsupportedArchitectureError(runtime="python2.7", architecture="arm64")
+            side_effect=UnsupportedArchitectureError(runtime="python3.7", architecture="arm64")
         )
 
         resolver_mock = Mock()
@@ -349,7 +349,7 @@ def test_must_raise_for_incompatible_runtime_and_architecture(self):
         with self.assertRaises(WorkflowFailedError) as ex:
             self.work.run()
 
-        self.assertIn("Architecture arm64 is not supported for runtime python2.7", str(ex.exception))
+        self.assertIn("Architecture arm64 is not supported for runtime python3.7", str(ex.exception))
 
 
 class TestBaseWorkflow_repr(TestCase):