test updates to adopt https://github.com/devcontainers/cli/pull/124 (#114)

* test updates to adopt https://github.com/devcontainers/cli/pull/124

* remove accidental tgz check in

* typo in option

* typo in python install script on dotnet6/jammy branch

* no scenarios

* code review and test name

* Delete install_os_provided_python.sh

31 files changed, 280 insertions, 177 deletions

diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml
index 6b453f8..621ee83 100644
--- a/.github/workflows/test-all.yaml
+++ b/.github/workflows/test-all.yaml
@@ -37,13 +37,73 @@ jobs:
             "rust",
             "sshd",
             "terraform",
+        ]
+        baseImage:
+          [
+            "ubuntu:focal",
+            "ubuntu:jammy",
+            "debian:11",
+            "mcr.microsoft.com/devcontainers/base:ubuntu",
+            "mcr.microsoft.com/devcontainers/base:debian",
           ]
-        baseImage: [ "ubuntu:focal", "ubuntu:jammy", "debian:11", "mcr.microsoft.com/vscode/devcontainers/base:ubuntu", "mcr.microsoft.com/vscode/devcontainers/base:debian" ]
     steps:
       - uses: actions/checkout@v2
 
       - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
+        run: npm install -g @devcontainers/cli
 
-      - name: " Testing '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
-        run: devcontainer features test -f ${{ matrix.features }} -i ${{ matrix.baseImage }} -c `pwd`
+      - name: "Generating tests for '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
+        run: devcontainer features test  --skip-scenarios -f ${{ matrix.features }} -i ${{ matrix.baseImage }} .
+
+  test-scenarios:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        features: [
+            "anaconda",
+            "aws-cli",
+            "azure-cli",
+            "common-utils",
+            "desktop-lite",
+            "docker-from-docker",
+            "docker-in-docker",
+            "dotnet",
+            "git",
+            "git-lfs",
+            "github-cli",
+            "go",
+            "hugo",
+            "java",
+            "kubectl-helm-minikube",
+            "node",
+            "nvidia-cuda",
+            "oryx",
+            "php",
+            "powershell",
+            "python",
+            "ruby",
+            "rust",
+            "sshd",
+            "terraform",
+        ]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Testing '${{ matrix.features }}' scenarios"
+        run: devcontainer features test -f ${{ matrix.features }} --skip-autogenerated  .
+
+  test-global:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Testing global scenarios"
+        run: devcontainer features test --global-scenarios-only .
\ No newline at end of file
diff --git a/.github/workflows/test-manual.yaml b/.github/workflows/test-manual.yaml
index 3e285a7..333dbb8 100644
--- a/.github/workflows/test-manual.yaml
+++ b/.github/workflows/test-manual.yaml
@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
     inputs:
       features:
-        description: "list of features to execute tests against"
+        description: "List of features to execute tests against"
         required: true
         default: "go dotnet"
       baseImage:
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
+        run: npm install -g @devcontainers/cli
 
       - name: "Testing '${{ github.event.inputs.features }}' against '${{ github.event.inputs.baseImage }}'"
-        run: devcontainer features test --features ${{ github.event.inputs.features }} --base-image ${{ github.event.inputs.baseImage }} --collection-folder `pwd` --log-level ${{ github.event.inputs.logLevel }}
+        run: devcontainer features test --features ${{ github.event.inputs.features }} --base-image ${{ github.event.inputs.baseImage }} --log-level ${{ github.event.inputs.logLevel }} .
diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml
index 7624701..c8dec8f 100644
--- a/.github/workflows/test-pr.yaml
+++ b/.github/workflows/test-pr.yaml
@@ -1,4 +1,4 @@
-name: "PR -Test Updated Features"
+name: "PR - Test Updated Features"
 on:
   pull_request:
 
@@ -50,14 +50,30 @@ jobs:
             "ubuntu:focal",
             "ubuntu:jammy",
             "debian:11",
-            "mcr.microsoft.com/vscode/devcontainers/base:ubuntu",
-            "mcr.microsoft.com/vscode/devcontainers/base:debian",
+            "mcr.microsoft.com/devcontainers/base:ubuntu",
+            "mcr.microsoft.com/devcontainers/base:debian",
           ]
     steps:
       - uses: actions/checkout@v2
 
       - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
+        run: npm install -g @devcontainers/cli
 
-      - name: "Testing '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
-        run: devcontainer features test -f ${{ matrix.features }} -i ${{ matrix.baseImage }} -c `pwd`
+      - name: "Generating tests for '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
+        run: devcontainer features test  --skip-scenarios -f ${{ matrix.features }} -i ${{ matrix.baseImage }} .
+
+  test-scenarios:
+    needs: [detect-changes]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        features: ${{ fromJSON(needs.detect-changes.outputs.features) }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Install latest devcontainer CLI"
+        run: npm install -g @devcontainers/cli
+
+      - name: "Testing '${{ matrix.features }}' scenarios"
+        run: devcontainer features test -f ${{ matrix.features }} --skip-autogenerated  .
\ No newline at end of file
diff --git a/.github/workflows/test-scenarios.yaml b/.github/workflows/test-scenarios.yaml
deleted file mode 100644
index cc71737..0000000
--- a/.github/workflows/test-scenarios.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-name: "Test Scenarios"
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-
-jobs:
-  scenarios:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: "Install latest devcontainer CLI"
-        run: npm install -g @devcontainers/cli@0.12.1
-
-      - name: "Running predefined test scenarios"
-        run: devcontainer features test --scenarios `pwd`/test-scenarios -c `pwd`
diff --git a/test-scenarios/install_python_twice.sh b/test-scenarios/install_python_twice.sh
deleted file mode 100644
index 911ee54..0000000
--- a/test-scenarios/install_python_twice.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Optional: Import test library
-source dev-container-features-test-lib
-
-check "version 3.9 installed"  ls -1 /usr/local/python/ | grep 3.9
-check "version 3.8 installed"  ls -1 /usr/local/python/ | grep 3.8
-
-check "3.9 alias to python on path" python  --version | grep 3.9
-
-# Report result
-reportResults
diff --git a/test-scenarios/scenarios.json b/test-scenarios/scenarios.json
deleted file mode 100644
index ddf15d0..0000000
--- a/test-scenarios/scenarios.json
+++ /dev/null
@@ -1,130 +0,0 @@
-{
-    "install_os_provided_python": {
-        "image": "mcr.microsoft.com/devcontainers/base:0-bullseye",
-        "features": {
-            "node": "latest",
-            "python": "os-provided"
-        }
-    },
-    "install_gradle_and_maven": {
-        "image": "ubuntu:focal",
-        "features": {
-            "java": {
-                "version": "latest",
-                "installGradle": true,
-                "installMaven": true
-            }
-        }
-    },
-    "install_additional_php": {
-        "image": "ubuntu:focal",
-        "features": {
-            "php": {
-                "version": "8.1.4",
-                "additionalVersions": "8.0.17,8.0.3"
-            }
-        }
-    },
-    "install_additional_java": {
-        "image": "ubuntu:focal",
-        "features": {
-            "java": {
-                "version": "17",
-                "additionalVersions": "11,8"
-            }
-        }
-    },
-    "install_jupyterlab": {
-        "image": "mcr.microsoft.com/devcontainers/base:focal",
-        "remoteUser": "vscode",
-        "features": {
-            "common-utils": {
-                "username": "vscode"
-            },
-            "python": {
-                "installJupyterlab": true,
-                "configureJupyterlabAllowOrigin": "*"
-            }
-        }
-    },
-    "install_additional_node": {
-        "image": "debian:11",
-        "features": {
-            "node": {
-                "version": "latest",
-                "additionalVersions": "v17.9.1,v14.19.3"
-            }
-        }
-    },
-    "install_additional_python": {
-        "image": "ubuntu:focal",
-        "features": {
-            "python": {
-                "version": "3.10.5",
-                "additionalVersions": "3.8,3.9.13"
-            }
-        }
-    },
-    "install_additional_ruby": {
-        "image": "ubuntu:focal",
-        "features": {
-            "ruby": {
-                "version": "3.1.2",
-                "additionalVersions": "2.5,3.0.4"
-            }
-        }
-    },
-    "install_additional_dotnet": {
-        "image": "ubuntu:focal",
-        "features": {
-            "dotnet": {
-                "version": "6.0.301",
-                "additionalVersions": "5.0,3.1.420"
-            }
-        }
-    },
-    "install_dotnet_5": {
-        "image": "ubuntu:focal",
-        "features": {
-            "dotnet": {
-                "version": "5"
-            }
-        }
-    },
-    "install_dotnet_3": {
-        "image": "ubuntu:focal",
-        "features": {
-            "dotnet": {
-                "version": "3"
-            }
-        }
-    },
-    "rust_at_pinned_version": {
-        "image": "ubuntu:focal",
-        "features": {
-            "rust": {
-                "version": "1.62.0"
-            }
-        }
-    },
-    "install_cudnn_nvxt": {
-        "image": "debian",
-        "features": {
-            "nvidia-cuda": {
-                "installCudnn": true,
-                "installNvtx": true
-            }
-        }
-    },
-    "install_cudnn_nvxt_version": {
-        "image": "debian",
-        "features": {
-            "nvidia-cuda": {
-                "installCudnn": true,
-                "installNvtx": true,
-                "cudaVersion": "11.5",
-                "cudnnVersion": "8.3.2.44"
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/test/_global/all_the_clis.sh b/test/_global/all_the_clis.sh
new file mode 100644
index 0000000..71171c1
--- /dev/null
+++ b/test/_global/all_the_clis.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# Definition specific tests
+check "check for aws" aws --version
+check "check for gh" gh --version
+check "check for azure" az  --version
+
+# Report result
+reportResults
\ No newline at end of file
diff --git a/test/_global/node_java_rust.sh b/test/_global/node_java_rust.sh
new file mode 100644
index 0000000..5d05729
--- /dev/null
+++ b/test/_global/node_java_rust.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# Definition specific tests
+check "check for node" node --version
+check "check for java" java --version
+check "check for rust" rustc  --version
+
+# Report result
+reportResults
\ No newline at end of file
diff --git a/test/_global/scenarios.json b/test/_global/scenarios.json
new file mode 100644
index 0000000..2228c7d
--- /dev/null
+++ b/test/_global/scenarios.json
@@ -0,0 +1,18 @@
+{
+    "all_the_clis": {
+        "image": "ubuntu:focal",
+        "features": {
+            "aws-cli": {},
+            "azure-cli": {},
+            "github-cli": {}
+        }
+    },
+    "node_java_rust": {
+        "image": "ubuntu:focal",
+        "features": {
+            "node": {},
+            "java": {},
+            "rust": {}
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/install_additional_dotnet.sh b/test/dotnet/install_additional_dotnet.sh
index fdb2da7..fdb2da7 100644
--- a/test-scenarios/install_additional_dotnet.sh
+++ b/test/dotnet/install_additional_dotnet.sh
diff --git a/test-scenarios/install_dotnet_3.sh b/test/dotnet/install_dotnet_3.sh
index 0d05215..0d05215 100644
--- a/test-scenarios/install_dotnet_3.sh
+++ b/test/dotnet/install_dotnet_3.sh
diff --git a/test-scenarios/install_dotnet_5.sh b/test/dotnet/install_dotnet_5.sh
index 5f48eb1..5f48eb1 100644
--- a/test-scenarios/install_dotnet_5.sh
+++ b/test/dotnet/install_dotnet_5.sh
diff --git a/test/dotnet/scenarios.json b/test/dotnet/scenarios.json
new file mode 100644
index 0000000..edf4a75
--- /dev/null
+++ b/test/dotnet/scenarios.json
@@ -0,0 +1,27 @@
+{
+    "install_additional_dotnet": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "6.0.301",
+                "additionalVersions": "5.0,3.1.420"
+            }
+        }
+    },
+    "install_dotnet_3": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "3"
+            }
+        }
+    },
+    "install_dotnet_5": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "5"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/install_additional_java.sh b/test/java/install_additional_java.sh
index a796fee..a796fee 100644
--- a/test-scenarios/install_additional_java.sh
+++ b/test/java/install_additional_java.sh
diff --git a/test-scenarios/install_gradle_and_maven.sh b/test/java/install_gradle_and_maven.sh
index a81dd9e..a81dd9e 100644
--- a/test-scenarios/install_gradle_and_maven.sh
+++ b/test/java/install_gradle_and_maven.sh
diff --git a/test/java/scenarios.json b/test/java/scenarios.json
new file mode 100644
index 0000000..659a857
--- /dev/null
+++ b/test/java/scenarios.json
@@ -0,0 +1,21 @@
+{
+    "install_additional_java": {
+        "image": "ubuntu:focal",
+        "features": {
+            "java": {
+                "version": "17",
+                "additionalVersions": "11,8"
+            }
+        }
+    },
+    "install_gradle_and_maven": {
+        "image": "ubuntu:focal",
+        "features": {
+            "java": {
+                "version": "latest",
+                "installGradle": true,
+                "installMaven": true
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/install_additional_node.sh b/test/node/install_additional_node.sh
index 83dae86..83dae86 100644
--- a/test-scenarios/install_additional_node.sh
+++ b/test/node/install_additional_node.sh
diff --git a/test/node/scenarios.json b/test/node/scenarios.json
new file mode 100644
index 0000000..845c892
--- /dev/null
+++ b/test/node/scenarios.json
@@ -0,0 +1,11 @@
+{
+    "install_additional_node": {
+        "image": "debian:11",
+        "features": {
+            "node": {
+                "version": "latest",
+                "additionalVersions": "v17.9.1,v14.19.3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/install_cudnn_nvxt.sh b/test/nvidia-cuda/install_cudnn_nvxt.sh
index 7b20f43..7b20f43 100644
--- a/test-scenarios/install_cudnn_nvxt.sh
+++ b/test/nvidia-cuda/install_cudnn_nvxt.sh
diff --git a/test-scenarios/install_cudnn_nvxt_version.sh b/test/nvidia-cuda/install_cudnn_nvxt_version.sh
index a7f46bd..a7f46bd 100644
--- a/test-scenarios/install_cudnn_nvxt_version.sh
+++ b/test/nvidia-cuda/install_cudnn_nvxt_version.sh
diff --git a/test/nvidia-cuda/scenarios.json b/test/nvidia-cuda/scenarios.json
new file mode 100644
index 0000000..1993dcd
--- /dev/null
+++ b/test/nvidia-cuda/scenarios.json
@@ -0,0 +1,22 @@
+{
+    "install_cudnn_nvxt": {
+        "image": "debian",
+        "features": {
+            "nvidia-cuda": {
+                "installCudnn": true,
+                "installNvtx": true
+            }
+        }
+    },
+    "install_cudnn_nvxt_version": {
+        "image": "debian",
+        "features": {
+            "nvidia-cuda": {
+                "installCudnn": true,
+                "installNvtx": true,
+                "cudaVersion": "11.5",
+                "cudnnVersion": "8.3.2.44"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/install_additional_php.sh b/test/php/install_additional_php.sh
index 590187a..590187a 100644
--- a/test-scenarios/install_additional_php.sh
+++ b/test/php/install_additional_php.sh
diff --git a/test/php/scenarios.json b/test/php/scenarios.json
new file mode 100644
index 0000000..f24753b
--- /dev/null
+++ b/test/php/scenarios.json
@@ -0,0 +1,11 @@
+{
+    "install_additional_php": {
+        "image": "ubuntu:focal",
+        "features": {
+            "php": {
+                "version": "8.1.4",
+                "additionalVersions": "8.0.17,8.0.3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/install_additional_python.sh b/test/python/install_additional_python.sh
index b0393d3..b0393d3 100644
--- a/test-scenarios/install_additional_python.sh
+++ b/test/python/install_additional_python.sh
diff --git a/test-scenarios/install_jupyterlab.sh b/test/python/install_jupyterlab.sh
index 9c5e453..9c5e453 100644
--- a/test-scenarios/install_jupyterlab.sh
+++ b/test/python/install_jupyterlab.sh
diff --git a/test-scenarios/install_os_provided_python.sh b/test/python/install_os_provided_python.sh
index 4dba59d..50a617e 100644
--- a/test-scenarios/install_os_provided_python.sh
+++ b/test/python/install_os_provided_python.sh
@@ -13,4 +13,4 @@ check "pip is installed" pip3 --version
 check "node is installed" node --version
 
 # Report result
-reportResults
+reportResults
\ No newline at end of file
diff --git a/test/python/scenarios.json b/test/python/scenarios.json
new file mode 100644
index 0000000..2aa246f
--- /dev/null
+++ b/test/python/scenarios.json
@@ -0,0 +1,31 @@
+{
+    "install_additional_python": {
+        "image": "ubuntu:focal",
+        "features": {
+            "python": {
+                "version": "3.10.5",
+                "additionalVersions": "3.8,3.9.13"
+            }
+        }
+    },
+    "install_jupyterlab": {
+        "image": "mcr.microsoft.com/vscode/devcontainers/base:focal",
+        "remoteUser": "vscode",
+        "features": {
+            "common-utils": {
+                "username": "vscode"
+            },
+            "python": {
+                "installJupyterlab": true,
+                "configureJupyterlabAllowOrigin": "*"
+            }
+        }
+    },
+    "install_os_provided_python": {
+        "image": "mcr.microsoft.com/devcontainers/base:0-bullseye",
+        "features": {
+            "node": "latest",
+            "python": "os-provided"
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/install_additional_ruby.sh b/test/ruby/install_additional_ruby.sh
index 3e76159..3e76159 100644
--- a/test-scenarios/install_additional_ruby.sh
+++ b/test/ruby/install_additional_ruby.sh
diff --git a/test/ruby/scenarios.json b/test/ruby/scenarios.json
new file mode 100644
index 0000000..9f88d9e
--- /dev/null
+++ b/test/ruby/scenarios.json
@@ -0,0 +1,11 @@
+{
+    "install_additional_ruby": {
+        "image": "ubuntu:focal",
+        "features": {
+            "ruby": {
+                "version": "3.1.2",
+                "additionalVersions": "2.5,3.0.4"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test-scenarios/rust_at_pinned_version.sh b/test/rust/rust_at_pinned_version.sh
index 2954a7a..2954a7a 100644
--- a/test-scenarios/rust_at_pinned_version.sh
+++ b/test/rust/rust_at_pinned_version.sh
diff --git a/test/rust/scenarios.json b/test/rust/scenarios.json
new file mode 100644
index 0000000..4a2d540
--- /dev/null
+++ b/test/rust/scenarios.json
@@ -0,0 +1,10 @@
+{
+    "rust_at_pinned_version": {
+        "image": "ubuntu:focal",
+        "features": {
+            "rust": {
+                "version": "1.62.0"
+            }
+        }
+    }
+}
\ No newline at end of file