From def3a41f576ceffebdff8266b41b8589a2512ac2 Mon Sep 17 00:00:00 2001 From: Philipp Wagner Date: Wed, 6 Dec 2023 02:02:58 +0100 Subject: [PATCH 001/285] Fix common-utils installation on RHEL (and friends) (#772) * Fix indentation in common-utils/main.sh * common-utils: Fix install error on RHEL On RHEL (and derivatives) the installation of the common-utils feature could fail if the feature ran before (i.e., `PACKAGES_ALREADY_INSTALLED` is set) and if either `INSTALL_ZSH` is false, or zsh was installed earlier and `ZSH_ALREADY_INSTALLED` is true. In these cases the script the `package_list` is empty, and `dnf` terminates with the following error message: ``` usage: dnf install [-c [config file]] [-q] [-v] [--version] [--installroot [path]] [--nodocs] [--noplugins] [--enableplugin [plugin]] [--disableplugin [plugin]] [--releasever RELEASEVER] [--setopt SETOPTS] [--skip-broken] [-h] [--allowerasing] [-b | --nobest] [-C] [-R [minutes]] [-d [debug level]] [--debugsolver] [--showduplicates] [-e ERRORLEVEL] [--obsoletes] [--rpmverbosity [debug level name]] [-y] [--assumeno] [--enablerepo [repo]] [--disablerepo [repo] | --repo [repo]] [--enable | --disable] [-x [package]] [--disableexcludes [repo]] [--repofrompath [repo,path]] [--noautoremove] [--nogpgcheck] [--color COLOR] [--refresh] [-4] [-6] [--destdir DESTDIR] [--downloadonly] [--comment COMMENT] [--bugfix] [--enhancement] [--newpackage] [--security] [--advisory ADVISORY] [--bz BUGZILLA] [--cve CVES] [--sec-severity {Critical,Important,Moderate,Low}] [--forcearch ARCH] PACKAGE [PACKAGE ...] dnf install: error: the following arguments are required: PACKAGE ``` Fix the problem by running `dnf` only with a non-zero `package_list`. --- src/common-utils/main.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index d1a977f9c..a291f98e2 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -189,11 +189,11 @@ install_redhat_packages() { man-db \ strace" - # rockylinux:9 installs 'curl-minimal' which clashes with 'curl' - # Install 'curl' for every OS except this rockylinux:9 - if [[ "${ID}" = "rocky" ]] && [[ "${VERSION}" != *"9."* ]]; then - package_list="${package_list} curl" - fi + # rockylinux:9 installs 'curl-minimal' which clashes with 'curl' + # Install 'curl' for every OS except this rockylinux:9 + if [[ "${ID}" = "rocky" ]] && [[ "${VERSION}" != *"9."* ]]; then + package_list="${package_list} curl" + fi # Install OpenSSL 1.0 compat if needed if ${install_cmd} -q list compat-openssl10 >/dev/null 2>&1; then @@ -222,7 +222,9 @@ install_redhat_packages() { package_list="${package_list} zsh" fi - ${install_cmd} -y install ${package_list} + if [ -n "${package_list}" ]; then + ${install_cmd} -y install ${package_list} + fi # Get to latest versions of all packages if [ "${UPGRADE_PACKAGES}" = "true" ]; then From 3ea4d6bbd7864bcf7b5a91fdeeb66e4f5a6f46c0 Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Thu, 7 Dec 2023 10:49:18 -0800 Subject: [PATCH 002/285] Fix `.zprofile` is owned by root (#775) * revert behavior of sourcing .profile in .zprofile https://github.com/devcontainers/features/pull/736/files#diff-3e71c0a0669a0410f7dd0d8f2b83f3b6bf6b525d3eabd354f19f32822da669fcR460-R463 * add test * increment version * create empty .zprofile is not present, and fix test * temporarily add back .profile to .zprofile (only if file did not exist) --- src/common-utils/devcontainer-feature.json | 2 +- src/common-utils/main.sh | 6 ++++-- test/common-utils/configure_zsh_as_default_shell.sh | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index dc54e6918..329f8dec8 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.3.0", + "version": "2.3.1", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index a291f98e2..26f0a7512 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -459,8 +459,10 @@ fi # Optionally configure zsh and Oh My Zsh! if [ "${INSTALL_ZSH}" = "true" ]; then - if [ ! -f "${user_home}/.zprofile" ] || ! grep -Fxq 'source $HOME/.profile' "${user_home}/.zprofile" ; then - echo 'source $HOME/.profile' >> "${user_home}/.zprofile" + if [ ! -f "${user_home}/.zprofile" ]; then + touch "${user_home}/.zprofile" + echo 'source $HOME/.profile' >> "${user_home}/.zprofile" # TODO: Reconsider adding '.profile' to '.zprofile' + chown ${USERNAME}:${group_name} "${user_home}/.zprofile" fi if [ "${ZSH_ALREADY_INSTALLED}" != "true" ]; then diff --git a/test/common-utils/configure_zsh_as_default_shell.sh b/test/common-utils/configure_zsh_as_default_shell.sh index 01d15227e..21b12101e 100644 --- a/test/common-utils/configure_zsh_as_default_shell.sh +++ b/test/common-utils/configure_zsh_as_default_shell.sh @@ -11,5 +11,7 @@ check "default-shell-is-zsh" bash -c "getent passwd $(whoami) | awk -F: '{ print check "default-zshrc-is-dev-container-template" bash -c "cat ~/.zshrc | grep ZSH_THEME | grep devcontainers" check "zsh-path-contains-local-bin" zsh -l -c "echo $PATH | grep '/home/devcontainer/.local/bin'" +check "Ensure .zprofile is owned by remoteUser" bash -c "stat -c '%U' /home/devcontainer/.zprofile | grep devcontainer" + # Report result reportResults From 5258a80c67e92b548c7a53d27be1a30669308521 Mon Sep 17 00:00:00 2001 From: Eljo George Date: Mon, 8 Jan 2024 13:16:23 -0800 Subject: [PATCH 003/285] Alpine - Install libssl1.1 only if available (#793) --- src/common-utils/main.sh | 7 ++++++- test/common-utils/alpine-3-18.sh | 16 ++++++++++++++++ test/common-utils/scenarios.json | 7 +++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100755 test/common-utils/alpine-3-18.sh diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index 26f0a7512..6214d2580 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -265,7 +265,6 @@ install_alpine_packages() { libstdc++ \ krb5-libs \ libintl \ - libssl1.1 \ lttng-ust \ tzdata \ userspace-rcu \ @@ -279,6 +278,12 @@ install_alpine_packages() { shadow \ strace + # # Include libssl1.1 if available (not available for 3.19 and newer) + LIBSSL1_PKG=libssl1.1 + if [[ $(apk search --no-cache -a $LIBSSL1_PKG | grep $LIBSSL1_PKG) ]]; then + apk add --no-cache $LIBSSL1_PKG + fi + # Install man pages - package name varies between 3.12 and earlier versions if apk info man > /dev/null 2>&1; then apk add --no-cache man man-pages diff --git a/test/common-utils/alpine-3-18.sh b/test/common-utils/alpine-3-18.sh new file mode 100755 index 000000000..eaf384b9e --- /dev/null +++ b/test/common-utils/alpine-3-18.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${ID}" = "alpine" +check "bashrc" ls /etc/bash/bashrc +check "libssl1.1 is installed" grep "libssl1.1" <(apk list --no-cache libssl1.1) + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/scenarios.json b/test/common-utils/scenarios.json index c1ee18dde..eef2bffcd 100644 --- a/test/common-utils/scenarios.json +++ b/test/common-utils/scenarios.json @@ -162,6 +162,13 @@ "common-utils": {} } }, + "alpine-3-18": { + "image": "alpine:3.18", + "remoteUser": "devcontainer", + "features": { + "common-utils": {} + } + }, "devcontainer-info": { "build": { "dockerfile": "Dockerfile" From 562305d37b97d47331d96306ffc2a0a3cce55e64 Mon Sep 17 00:00:00 2001 From: Eljo George Date: Mon, 8 Jan 2024 14:53:08 -0800 Subject: [PATCH 004/285] Bump version for common-utils (#794) Version bump for https://github.com/devcontainers/features/pull/793 --- src/common-utils/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index 329f8dec8..dbe466beb 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.3.1", + "version": "2.3.2", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", From 6c87ba7564e9db5f7cc380c238b196e5fc676cd9 Mon Sep 17 00:00:00 2001 From: gmeans Date: Tue, 16 Jan 2024 19:01:41 -0500 Subject: [PATCH 005/285] Fix kubectl zsh completions typo (#792) * fix typo * version bump --------- Co-authored-by: Eljo George --- src/kubectl-helm-minikube/devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index 4206772b2..fa8ea6138 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.5", + "version": "1.1.6", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index 35444c5dc..5deb377c3 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -148,7 +148,7 @@ if [ ${KUBECTL_VERSION} != "none" ]; then kubectl completion bash > /etc/bash_completion.d/kubectl # kubectl zsh completion - if [ -e "${USERHOME}}/.oh-my-zsh" ]; then + if [ -e "${USERHOME}/.oh-my-zsh" ]; then mkdir -p "${USERHOME}/.oh-my-zsh/completions" kubectl completion zsh > "${USERHOME}/.oh-my-zsh/completions/_kubectl" chown -R "${USERNAME}" "${USERHOME}/.oh-my-zsh" From f3cbd989e15761534f3cb8efa6b8fce150e5fe3a Mon Sep 17 00:00:00 2001 From: Michael Persico Date: Wed, 17 Jan 2024 14:19:23 -0800 Subject: [PATCH 006/285] [common-utils] Add xz (#798) Bump common-utils to v2.4.0 to reflect new package addition Co-authored-by: Samruddhi Khandale --- src/common-utils/devcontainer-feature.json | 2 +- src/common-utils/main.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index dbe466beb..de0d02d29 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.3.2", + "version": "2.4.0", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index 6214d2580..4f3dcf8e8 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -48,6 +48,7 @@ install_debian_packages() { ca-certificates \ unzip \ bzip2 \ + xz-utils \ zip \ nano \ vim-tiny \ @@ -173,6 +174,7 @@ install_redhat_packages() { ca-certificates \ rsync \ unzip \ + xz \ zip \ nano \ vim-minimal \ @@ -256,6 +258,7 @@ install_alpine_packages() { rsync \ ca-certificates \ unzip \ + xz \ zip \ nano \ vim \ From 2b2a5f652bd3df0cb514e21af064c5390b53d0f0 Mon Sep 17 00:00:00 2001 From: "nohz.afk" <149959021+nohzafk@users.noreply.github.com> Date: Fri, 19 Jan 2024 01:20:49 +0800 Subject: [PATCH 007/285] feature nix: add option useAttrPath to use packages attribute path (#787) * add option `useAttrPath` to support `nix-env -iA` * add test and fix problem * change to useAttributePath * bump the version * update test --- src/nix/devcontainer-feature.json | 11 +++++++--- src/nix/install.sh | 9 ++++---- src/nix/post-install-steps.sh | 19 ++++++++++++++++ test/nix/packages-use-attr-path.sh | 35 ++++++++++++++++++++++++++++++ test/nix/scenarios.json | 13 ++++++++++- 5 files changed, 79 insertions(+), 8 deletions(-) create mode 100755 test/nix/packages-use-attr-path.sh diff --git a/src/nix/devcontainer-feature.json b/src/nix/devcontainer-feature.json index 96a75016b..0e37efed2 100644 --- a/src/nix/devcontainer-feature.json +++ b/src/nix/devcontainer-feature.json @@ -1,9 +1,9 @@ { "id": "nix", - "version": "1.1.3", + "version": "1.2.0", "name": "Nix Package Manager", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/nix", - "description": "Installs the Nix package manager and optionally a set of packages.", + "description": "Installs the Nix package manager and optionally a set of packages.", "options": { "version": { "type": "string", @@ -21,6 +21,11 @@ "default": "", "description": "Optional comma separated list of Nix packages to install in profile." }, + "useAttributePath": { + "type": "boolean", + "default": false, + "description": "Enable this option to use exact attribute path of the package in the Nixpkgs repository, aligning with the nix-env -iA command." + }, "flakeUri": { "type": "string", "default": "", @@ -39,4 +44,4 @@ "PATH": "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:${PATH}" }, "entrypoint": "/usr/local/share/nix-entrypoint.sh" -} \ No newline at end of file +} diff --git a/src/nix/install.sh b/src/nix/install.sh index ed048fe8d..5b8fe8f28 100755 --- a/src/nix/install.sh +++ b/src/nix/install.sh @@ -8,6 +8,7 @@ cd "${FEATURE_DIR}" VERSION="${VERSION:-"latest"}" MULTIUSER="${MULTIUSER:-"true"}" PACKAGES="${PACKAGES//,/ }" +USEATTRIBUTEPATH="${USEATTRIBUTEPATH:-"false"}" FLAKEURI="${FLAKEURI:-""}" EXTRANIXCONFIG="${EXTRANIXCONFIG:-""}" USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" @@ -68,7 +69,7 @@ else exit 1 fi echo "(*) Performing single-user install..." - echo -e "\n**NOTE: Nix will only work for user ${USERNAME} on Linux if the host machine user's UID is $(id -u ${USERNAME}). You will need to chown /nix otherwise.**\n" + echo -e "\n**NOTE: Nix will only work for user ${USERNAME} on Linux if the host machine user's UID is $(id -u ${USERNAME}). You will need to chown /nix otherwise.**\n" # Install per https://nixos.org/manual/nix/stable/installation/installing-binary.html#single-user-installation mkdir -p /nix chown ${USERNAME} /nix ${tmpdir} @@ -79,14 +80,14 @@ else ' update_rc_file "$home_dir/.bashrc" "${snippet}" update_rc_file "$home_dir/.zshenv" "${snippet}" - update_rc_file "$home_dir/.profile" "${snippet}" + update_rc_file "$home_dir/.profile" "${snippet}" fi rm -rf "${tmpdir}" "/tmp/tmp-gnupg" fi # Set nix config mkdir -p /etc/nix -create_or_update_file /etc/nix/nix.conf 'sandbox = false' +create_or_update_file /etc/nix/nix.conf 'sandbox = false' if [ ! -z "${FLAKEURI}" ] && [ "${FLAKEURI}" != "none" ]; then create_or_update_file /etc/nix/nix.conf 'experimental-features = nix-command flakes' fi @@ -127,4 +128,4 @@ else " fi -echo "Done!" \ No newline at end of file +echo "Done!" diff --git a/src/nix/post-install-steps.sh b/src/nix/post-install-steps.sh index aa466798b..68f93a391 100755 --- a/src/nix/post-install-steps.sh +++ b/src/nix/post-install-steps.sh @@ -2,10 +2,29 @@ set -e echo "(*) Executing post-installation steps..." +# if not starts with "nixpkgs." add it as prefix to package name +add_nixpkgs_prefix() { + local packages=$1 + local -a addr + IFS=' ' read -ra addr <<<"$packages" + for i in "${!addr[@]}"; do + if [[ ${addr[i]} != nixpkgs.* ]]; then + addr[i]="nixpkgs.${addr[i]}" + fi + done + IFS=' ' echo "${addr[*]}" +} + # Install list of packages in profile if specified. if [ ! -z "${PACKAGES}" ] && [ "${PACKAGES}" != "none" ]; then + if [ "${USEATTRIBUTEPATH}" = "true" ]; then + PACKAGES=$(add_nixpkgs_prefix "$PACKAGES") + echo "Installing packages \"${PACKAGES}\" in profile..." + nix-env -iA ${PACKAGES} + else echo "Installing packages \"${PACKAGES}\" in profile..." nix-env --install ${PACKAGES} + fi fi # Install Nix flake in profile if specified diff --git a/test/nix/packages-use-attr-path.sh b/test/nix/packages-use-attr-path.sh new file mode 100755 index 000000000..7f34a96ea --- /dev/null +++ b/test/nix/packages-use-attr-path.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e + +# Optional: Import test library bundled with the devcontainer CLI +source dev-container-features-test-lib + +uid="$(id -u)" +echo "Current user UID is ${uid}." +if [ "${uid}" != "1000" ]; then + echo "Current user UID was adjusted." +fi +set +e +vscode_uid="$(id -u vscode)" +set -e +if [ "${vscode_uid}" != "" ]; then + echo "User vscode UID is ${vscode_uid}." + if [ "${vscode_uid}" != "1000" ]; then + echo "User vscode UID was adjusted." + fi +fi +nix_uid="$(stat /nix -c "%u")" +echo "/nix UID is ${nix_uid}." + +cat /etc/os-release + +# Feature-specific tests +# The 'check' command comes from the dev-container-features-test-lib. +check "nix-env" type nix-env +check "vim_installed" type vim +check "node_installed" type node +check "yarn_installed" type yarn + +# Report result +# If any of the checks above exited with a non-zero exit code, the test will fail. +reportResults &2>1 diff --git a/test/nix/scenarios.json b/test/nix/scenarios.json index c287f58c7..80ead7ff6 100644 --- a/test/nix/scenarios.json +++ b/test/nix/scenarios.json @@ -81,6 +81,17 @@ } } }, + "packages-use-attr-path": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "remoteUser": "vscode", + "features": { + "nix": { + "packages": "nodePackages.nodejs,nixpkgs.vim,nixpkgs.yarn", + "useAttributePath": true + } + } + }, + "flake": { "image": "mcr.microsoft.com/devcontainers/base:ubuntu", "remoteUser": "vscode", @@ -99,4 +110,4 @@ } } } -} \ No newline at end of file +} From 8a63fabbf91c64309fbf0873c311a468f7910755 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Thu, 18 Jan 2024 09:38:18 -0800 Subject: [PATCH 008/285] Automated documentation update (#799) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/nix/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/README.md b/src/nix/README.md index 3ab9b0017..4fd9700f3 100644 --- a/src/nix/README.md +++ b/src/nix/README.md @@ -18,6 +18,7 @@ Installs the Nix package manager and optionally a set of packages. | version | Version of Nix to install. | string | latest | | multiUser | Perform a multi-user install (instead of single user) | boolean | true | | packages | Optional comma separated list of Nix packages to install in profile. | string | - | +| useAttributePath | Enable this option to use exact attribute path of the package in the Nixpkgs repository, aligning with the nix-env -iA command. | boolean | false | | flakeUri | Optional URI to a Nix Flake to install in profile. | string | - | | extraNixConfig | Optional comma separated list of extra lines to add to /etc/nix/nix.conf. | string | - | From 65fb90b7fc8634986ab5644d00d66b983e9a50df Mon Sep 17 00:00:00 2001 From: Kai Groner Date: Fri, 19 Jan 2024 00:04:19 -0500 Subject: [PATCH 009/285] [Docker-outside-of-docker] Increase backlog depth for socat proxy (#789) (#790) Co-authored-by: Eljo George --- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index b82fc6d8b..4e5eae721 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.3.1", + "version": "1.3.2", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 3c0520698..901dd7870 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -329,7 +329,7 @@ if [ "${ENABLE_NONROOT_DOCKER}" = "true" ] && [ "${SOURCE_SOCKET}" != "${TARGET_ log "Enabling socket proxy." log "Proxying ${SOURCE_SOCKET} to ${TARGET_SOCKET} for vscode" sudoIf rm -rf ${TARGET_SOCKET} - (sudoIf socat UNIX-LISTEN:${TARGET_SOCKET},fork,mode=660,user=${USERNAME} UNIX-CONNECT:${SOURCE_SOCKET} 2>&1 | sudoIf tee -a \${SOCAT_LOG} > /dev/null & echo "\$!" | sudoIf tee \${SOCAT_PID} > /dev/null) + (sudoIf socat UNIX-LISTEN:${TARGET_SOCKET},fork,mode=660,user=${USERNAME},backlog=128 UNIX-CONNECT:${SOURCE_SOCKET} 2>&1 | sudoIf tee -a \${SOCAT_LOG} > /dev/null & echo "\$!" | sudoIf tee \${SOCAT_PID} > /dev/null) else log "Socket proxy already running." fi From 76ffd41c97d7e6c7bd5219b9e29ee9e22b4a3c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8C=C3=A1bera?= Date: Mon, 22 Jan 2024 22:45:23 +0100 Subject: [PATCH 010/285] [java] Support groovy installations (#779) * [java] Support groovy installations Fix java test failing with non existent 19 version of `open` distro * fix: Typo * feat: Implement code review comments --------- Co-authored-by: Samruddhi Khandale --- src/java/devcontainer-feature.json | 23 ++++++++++++++++--- src/java/install.sh | 7 ++++++ ...ll_ant_and_gradle_and_maven_and_groovy.sh} | 6 +++++ ...d_gradle_and_maven_and_groovy_for_user.sh} | 6 +++++ ...maven_and_groovy_with_specific_version.sh} | 2 ++ test/java/install_from_non_default_distro.sh | 2 +- test/java/scenarios.json | 20 +++++++++------- 7 files changed, 54 insertions(+), 12 deletions(-) rename test/java/{install_ant_and_gradle_and_maven.sh => install_ant_and_gradle_and_maven_and_groovy.sh} (84%) rename test/java/{install_ant_and_gradle_and_maven_for_user.sh => install_ant_and_gradle_and_maven_and_groovy_for_user.sh} (84%) rename test/java/{install_ant_and_gradle_and_maven_with_specific_version.sh => install_ant_and_gradle_and_maven_and_groovy_with_specific_version.sh} (92%) diff --git a/src/java/devcontainer-feature.json b/src/java/devcontainer-feature.json index d4aede873..64a4ca932 100644 --- a/src/java/devcontainer-feature.json +++ b/src/java/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "java", - "version": "1.2.2", + "version": "1.3.0", "name": "Java (via SDKMAN!)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/java", "description": "Installs Java, SDKMAN! (if not installed), and needed dependencies.", @@ -23,7 +23,8 @@ "ms", "open", "oracle", - "tem" + "tem", + "amzn" ], "default": "ms", "description": "Select or enter a JDK distribution" @@ -74,6 +75,22 @@ ], "default": "latest", "description": "Select or enter an Ant version" + }, + "installGroovy": { + "type": "boolean", + "default": false, + "description": "Install Groovy, powerful, optionally typed and dynamic language with static-typing and static compilation capabilities" + }, + "groovyVersion": { + "type": "string", + "proposals": [ + "latest", + "2.5.22", + "3.0.19", + "4.0.16" + ], + "default": "latest", + "description": "Select or enter a Groovy version" } }, "customizations": { @@ -92,6 +109,6 @@ "PATH": "/usr/local/sdkman/bin:/usr/local/sdkman/candidates/java/current/bin:/usr/local/sdkman/candidates/gradle/current/bin:/usr/local/sdkman/candidates/maven/current/bin:/usr/local/sdkman/candidates/ant/current/bin:${PATH}" }, "installsAfter": [ - "ghcr.io/devcontainers/features/common-utils" + "ghcr.io/devcontainers/features/common-utils" ] } diff --git a/src/java/install.sh b/src/java/install.sh index 2decc31b5..16bbb1c66 100644 --- a/src/java/install.sh +++ b/src/java/install.sh @@ -16,6 +16,8 @@ INSTALL_MAVEN="${INSTALLMAVEN:-"false"}" MAVEN_VERSION="${MAVENVERSION:-"latest"}" INSTALL_ANT="${INSTALLANT:-"false"}" ANT_VERSION="${ANTVERSION:-"latest"}" +INSTALL_GROOVY="${INSTALLGROOVY:-"false"}" +GROOVY_VERSION="${GROOVYVERSION:-"latest"}" JDK_DISTRO="${JDKDISTRO}" export SDKMAN_DIR="${SDKMAN_DIR:-"/usr/local/sdkman"}" @@ -190,6 +192,11 @@ if [[ "${INSTALL_MAVEN}" = "true" ]] && ! mvn --version > /dev/null; then sdk_install maven ${MAVEN_VERSION} fi +# Install Groovy +if [[ "${INSTALL_GROOVY}" = "true" ]] && ! groovy --version > /dev/null; then + sdk_install groovy "${GROOVY_VERSION}" +fi + # Clean up rm -rf /var/lib/apt/lists/* diff --git a/test/java/install_ant_and_gradle_and_maven.sh b/test/java/install_ant_and_gradle_and_maven_and_groovy.sh similarity index 84% rename from test/java/install_ant_and_gradle_and_maven.sh rename to test/java/install_ant_and_gradle_and_maven_and_groovy.sh index 599f8d9d5..e171f5be5 100644 --- a/test/java/install_ant_and_gradle_and_maven.sh +++ b/test/java/install_ant_and_gradle_and_maven_and_groovy.sh @@ -24,5 +24,11 @@ check "maven" mvn --version cd /tmp && mvn archetype:generate -DgroupId=com.mycompany.app -DartifactId=my-app -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false check "m2 exists" grep ".m2" <(ls -la /root) +check "groovy" groovy --version +cat << EOF > /tmp/test.groovy +println("verify") +EOF +check "groovy works" test "$(groovy /tmp/test.groovy)" = "verify" + # Report result reportResults diff --git a/test/java/install_ant_and_gradle_and_maven_for_user.sh b/test/java/install_ant_and_gradle_and_maven_and_groovy_for_user.sh similarity index 84% rename from test/java/install_ant_and_gradle_and_maven_for_user.sh rename to test/java/install_ant_and_gradle_and_maven_and_groovy_for_user.sh index 4e8b43134..8d9f293e9 100644 --- a/test/java/install_ant_and_gradle_and_maven_for_user.sh +++ b/test/java/install_ant_and_gradle_and_maven_and_groovy_for_user.sh @@ -24,5 +24,11 @@ check "maven" mvn --version cd /tmp && mvn archetype:generate -DgroupId=com.mycompany.app -DartifactId=my-app -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false check "m2 exists" grep ".m2" <(ls -la /home/vscode) +check "groovy" groovy --version +cat << EOF > /tmp/test.groovy +println("verify") +EOF +check "groovy works" test "$(groovy /tmp/test.groovy)" = "verify" + # Report result reportResults diff --git a/test/java/install_ant_and_gradle_and_maven_with_specific_version.sh b/test/java/install_ant_and_gradle_and_maven_and_groovy_with_specific_version.sh similarity index 92% rename from test/java/install_ant_and_gradle_and_maven_with_specific_version.sh rename to test/java/install_ant_and_gradle_and_maven_and_groovy_with_specific_version.sh index 7727f51b6..54e13e0cf 100644 --- a/test/java/install_ant_and_gradle_and_maven_with_specific_version.sh +++ b/test/java/install_ant_and_gradle_and_maven_and_groovy_with_specific_version.sh @@ -24,5 +24,7 @@ check "maven version" grep "Apache Maven 3.6.3" <(mvn --version) cd /tmp && mvn archetype:generate -DgroupId=com.mycompany.app -DartifactId=my-app -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false check "m2 exists" grep ".m2" <(ls -la /root) +check "groovy version" grep "Groovy Version: 2.5.22" <(groovy --version) + # Report result reportResults diff --git a/test/java/install_from_non_default_distro.sh b/test/java/install_from_non_default_distro.sh index 021a72a21..3ab648cab 100644 --- a/test/java/install_from_non_default_distro.sh +++ b/test/java/install_from_non_default_distro.sh @@ -5,7 +5,7 @@ set -e # Optional: Import test library source dev-container-features-test-lib -check "java version openjdk 19 installed" grep "openjdk 19." <(java --version) +check "java version openjdk 21 installed" grep "openjdk 21." <(java --version) # Report result reportResults diff --git a/test/java/scenarios.json b/test/java/scenarios.json index 5d7e684ba..dac9850f2 100644 --- a/test/java/scenarios.json +++ b/test/java/scenarios.json @@ -3,7 +3,7 @@ "image": "ubuntu:focal", "features": { "java": { - "version": "19", + "version": "21", "jdkDistro": "open" } } @@ -17,7 +17,7 @@ } } }, - "install_ant_and_gradle_and_maven_for_user": { + "install_ant_and_gradle_and_maven_and_groovy_for_user": { "image": "ubuntu:focal", "remoteUser": "vscode", "features": { @@ -28,22 +28,24 @@ "version": "latest", "installAnt": true, "installGradle": true, - "installMaven": true + "installMaven": true, + "installGroovy": true } } }, - "install_ant_and_gradle_and_maven": { + "install_ant_and_gradle_and_maven_and_groovy": { "image": "ubuntu:focal", "features": { "java": { "version": "latest", "installAnt": true, "installGradle": true, - "installMaven": true + "installMaven": true, + "installGroovy": true } } }, - "install_ant_and_gradle_and_maven_with_specific_version": { + "install_ant_and_gradle_and_maven_and_groovy_with_specific_version": { "image": "ubuntu:focal", "features": { "java": { @@ -53,13 +55,15 @@ "installGradle": "true", "gradleVersion": "6.8.3", "installMaven": "true", - "mavenVersion": "3.6.3" + "mavenVersion": "3.6.3", + "installGroovy": "true", + "groovyVersion": "2.5.22" } } }, "install_non_conventional_version": { "image": "ubuntu:focal", - "features":{ + "features": { "java": { "version": "21", "jdkDistro": "graalce" From 43bb98bbf2e8adbf19947a94639badbdd68369e1 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:53:23 -0800 Subject: [PATCH 011/285] Automated documentation update (#803) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/java/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/java/README.md b/src/java/README.md index 1ffd6188b..4699eb517 100644 --- a/src/java/README.md +++ b/src/java/README.md @@ -23,6 +23,8 @@ Installs Java, SDKMAN! (if not installed), and needed dependencies. | mavenVersion | Select or enter a Maven version | string | latest | | installAnt | Install Ant, a software tool for automating software build processes | boolean | false | | antVersion | Select or enter an Ant version | string | latest | +| installGroovy | Install Groovy, powerful, optionally typed and dynamic language with static-typing and static compilation capabilities | boolean | false | +| groovyVersion | Select or enter a Groovy version | string | latest | ## Customizations From 96f5303928d9aef8f10965682f4480ac6bd963d7 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 22 Jan 2024 14:48:43 -0800 Subject: [PATCH 012/285] [Dotnet] - Fix tests ; removes stale check (#802) * [Dotnet] - Fix tests ; removes stale check * Update projects --- test/dotnet/install_aspnetcore_runtime_only.sh | 4 ---- test/dotnet/install_dotnet_latest_when_version_is_empty.sh | 6 +----- test/dotnet/install_dotnet_lts.sh | 2 +- test/dotnet/install_dotnet_runtime_only.sh | 4 ---- test/dotnet/test.sh | 6 +----- 5 files changed, 3 insertions(+), 19 deletions(-) diff --git a/test/dotnet/install_aspnetcore_runtime_only.sh b/test/dotnet/install_aspnetcore_runtime_only.sh index 35bd540b7..78e4f099c 100644 --- a/test/dotnet/install_aspnetcore_runtime_only.sh +++ b/test/dotnet/install_aspnetcore_runtime_only.sh @@ -18,10 +18,6 @@ expected=$(fetch_latest_version "aspnetcore") check "Latest ASP.NET Core Runtime version installed" \ is_aspnetcore_runtime_version_installed "$expected" -# Expect this check to fail in November 2023 when .NET 8.0 becomes GA -check "It is a flavor of .NET 7.0" \ -is_aspnetcore_runtime_version_installed "7.0" - # Report results # If any of the checks above exited with a non-zero exit code, the test will fail. reportResults \ No newline at end of file diff --git a/test/dotnet/install_dotnet_latest_when_version_is_empty.sh b/test/dotnet/install_dotnet_latest_when_version_is_empty.sh index 7418892e1..cd23edcf1 100644 --- a/test/dotnet/install_dotnet_latest_when_version_is_empty.sh +++ b/test/dotnet/install_dotnet_latest_when_version_is_empty.sh @@ -18,12 +18,8 @@ expected=$(fetch_latest_version) check "Latest .NET SDK version installed" \ is_dotnet_sdk_version_installed "$expected" -# Expect this check to fail in November 2023 when .NET 8.0 becomes GA -check "It is a flavor of .NET 7.0" \ -is_dotnet_sdk_version_installed "7.0" - check "Build and run example project" \ -dotnet run --project projects/net7.0 +dotnet run --project projects/net8.0 # Report results # If any of the checks above exited with a non-zero exit code, the test will fail. diff --git a/test/dotnet/install_dotnet_lts.sh b/test/dotnet/install_dotnet_lts.sh index fe37b89f7..bc2d40782 100644 --- a/test/dotnet/install_dotnet_lts.sh +++ b/test/dotnet/install_dotnet_lts.sh @@ -19,7 +19,7 @@ check "Latest LTS version installed" \ is_dotnet_sdk_version_installed "$expected" check "Build and run example project" \ -dotnet run --project projects/net6.0 +dotnet run --project projects/net8.0 # Report results # If any of the checks above exited with a non-zero exit code, the test will fail. diff --git a/test/dotnet/install_dotnet_runtime_only.sh b/test/dotnet/install_dotnet_runtime_only.sh index 65188c5ec..3c58340d1 100644 --- a/test/dotnet/install_dotnet_runtime_only.sh +++ b/test/dotnet/install_dotnet_runtime_only.sh @@ -18,10 +18,6 @@ expected=$(fetch_latest_version "dotnet") check "Latest .NET Runtime version installed" \ is_dotnet_runtime_version_installed "$expected" -# Expect this check to fail in November 2023 when .NET 8.0 becomes GA -check "It is a flavor of .NET 7.0" \ -is_dotnet_runtime_version_installed "7.0" - # Report results # If any of the checks above exited with a non-zero exit code, the test will fail. reportResults \ No newline at end of file diff --git a/test/dotnet/test.sh b/test/dotnet/test.sh index 7418892e1..cd23edcf1 100644 --- a/test/dotnet/test.sh +++ b/test/dotnet/test.sh @@ -18,12 +18,8 @@ expected=$(fetch_latest_version) check "Latest .NET SDK version installed" \ is_dotnet_sdk_version_installed "$expected" -# Expect this check to fail in November 2023 when .NET 8.0 becomes GA -check "It is a flavor of .NET 7.0" \ -is_dotnet_sdk_version_installed "7.0" - check "Build and run example project" \ -dotnet run --project projects/net7.0 +dotnet run --project projects/net8.0 # Report results # If any of the checks above exited with a non-zero exit code, the test will fail. From 4697d25cc57cb79fd37845147d6f9adb6044b8c4 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:55:21 -0800 Subject: [PATCH 013/285] [Updates] Automated vendor dotnet-install script (#801) * Automated dotnet-install script update * Bump version --------- Co-authored-by: github-actions Co-authored-by: Samruddhi Khandale --- src/dotnet/devcontainer-feature.json | 2 +- src/dotnet/scripts/vendor/dotnet-install.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index b9c7260ea..7abfc2d30 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.2", + "version": "2.0.3", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", diff --git a/src/dotnet/scripts/vendor/dotnet-install.sh b/src/dotnet/scripts/vendor/dotnet-install.sh index b350ad4e1..86a2d0bad 100755 --- a/src/dotnet/scripts/vendor/dotnet-install.sh +++ b/src/dotnet/scripts/vendor/dotnet-install.sh @@ -1708,7 +1708,11 @@ do -?|--?|-h|--help|-[Hh]elp) script_name="$(basename "$0")" echo ".NET Tools Installer" - echo "Usage: $script_name [-c|--channel ] [-v|--version ] [-p|--prefix ]" + echo "Usage:" + echo " # Install a .NET SDK of a given Quality from a given Channel" + echo " $script_name [-c|--channel ] [-q|--quality ]" + echo " # Install a .NET SDK of a specific public version" + echo " $script_name [-v|--version ]" echo " $script_name -h|-?|--help" echo "" echo "$script_name is a simple command line interface for obtaining dotnet cli." From 3e0c0ae17831b21f93a929e7264710456e37c684 Mon Sep 17 00:00:00 2001 From: Jeff Putsch <150963136+jdputschadi@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:38:39 -0800 Subject: [PATCH 014/285] allow common-utils to be installed on "minimal" RHEL-based distributions (#811) * allow common-utils to be installed on "minimal" RHEL-based distributions. * bump version to reflect bug fix * add missing test scripts for alma scenarios --------- Co-authored-by: Jeff Putsch --- src/common-utils/devcontainer-feature.json | 4 +- src/common-utils/main.sh | 9 +++-- test/common-utils/alma-8-minimal.sh | 16 ++++++++ test/common-utils/alma-8.sh | 16 ++++++++ test/common-utils/alma-9-minimal.sh | 16 ++++++++ test/common-utils/alma-9.sh | 16 ++++++++ test/common-utils/rocky-8-minimal.sh | 16 ++++++++ test/common-utils/rocky-9-minimal.sh | 16 ++++++++ test/common-utils/scenarios.json | 44 +++++++++++++++++++++- 9 files changed, 147 insertions(+), 6 deletions(-) create mode 100755 test/common-utils/alma-8-minimal.sh create mode 100755 test/common-utils/alma-8.sh create mode 100755 test/common-utils/alma-9-minimal.sh create mode 100755 test/common-utils/alma-9.sh create mode 100755 test/common-utils/rocky-8-minimal.sh create mode 100755 test/common-utils/rocky-9-minimal.sh diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index de0d02d29..44f9b32b4 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.4.0", + "version": "2.4.1", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", @@ -66,4 +66,4 @@ "description": "Add packages from non-free Debian repository? (Debian only)" } } -} +} \ No newline at end of file diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index 4f3dcf8e8..0eec1ecb9 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -155,9 +155,12 @@ install_debian_packages() { install_redhat_packages() { local package_list="" local remove_epel="false" - local install_cmd=dnf - if ! type dnf > /dev/null 2>&1; then - install_cmd=yum + local install_cmd=microdnf + if ! type microdnf > /dev/null 2>&1; then + install_cmd=dnf + if ! type dnf > /dev/null 2>&1; then + install_cmd=yum + fi fi if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then diff --git a/test/common-utils/alma-8-minimal.sh b/test/common-utils/alma-8-minimal.sh new file mode 100755 index 000000000..d08731824 --- /dev/null +++ b/test/common-utils/alma-8-minimal.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el8" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/alma-8.sh b/test/common-utils/alma-8.sh new file mode 100755 index 000000000..d08731824 --- /dev/null +++ b/test/common-utils/alma-8.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el8" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/alma-9-minimal.sh b/test/common-utils/alma-9-minimal.sh new file mode 100755 index 000000000..cb2b339e1 --- /dev/null +++ b/test/common-utils/alma-9-minimal.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el9" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/alma-9.sh b/test/common-utils/alma-9.sh new file mode 100755 index 000000000..cb2b339e1 --- /dev/null +++ b/test/common-utils/alma-9.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el9" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/rocky-8-minimal.sh b/test/common-utils/rocky-8-minimal.sh new file mode 100755 index 000000000..d08731824 --- /dev/null +++ b/test/common-utils/rocky-8-minimal.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el8" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/rocky-9-minimal.sh b/test/common-utils/rocky-9-minimal.sh new file mode 100755 index 000000000..cb2b339e1 --- /dev/null +++ b/test/common-utils/rocky-9-minimal.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el9" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/scenarios.json b/test/common-utils/scenarios.json index eef2bffcd..2e357bc2b 100644 --- a/test/common-utils/scenarios.json +++ b/test/common-utils/scenarios.json @@ -41,6 +41,34 @@ "common-utils": {} } }, + "alma-8": { + "image": "almalinux:8", + "remoteUser": "devcontainer", + "features": { + "common-utils": {} + } + }, + "alma-9": { + "image": "almalinux:9", + "remoteUser": "devcontainer", + "features": { + "common-utils": {} + } + }, + "alma-8-minimal": { + "image": "almalinux:8-minimal", + "remoteUser": "devcontainer", + "features": { + "common-utils": {} + } + }, + "alma-9-minimal": { + "image": "almalinux:9-minimal", + "remoteUser": "devcontainer", + "features": { + "common-utils": {} + } + }, "rocky-8": { "image": "rockylinux:8", "remoteUser": "devcontainer", @@ -55,6 +83,20 @@ "common-utils": {} } }, + "rocky-8-minimal": { + "image": "rockylinux:8-minimal", + "remoteUser": "devcontainer", + "features": { + "common-utils": {} + } + }, + "rocky-9-minimal": { + "image": "rockylinux:9-minimal", + "remoteUser": "devcontainer", + "features": { + "common-utils": {} + } + }, "fedora": { "image": "fedora", "remoteUser": "devcontainer", @@ -209,4 +251,4 @@ "common-utils": {} } } -} +} \ No newline at end of file From 97565a7e60cbdf7fa1e2d079e260d875f2cd11ed Mon Sep 17 00:00:00 2001 From: Jeff Putsch <150963136+jdputschadi@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:49:46 -0800 Subject: [PATCH 015/285] Add RHEL support to git feature (#782) * Git feature passes all tests: existing + rhel-based base images: centos7, alma-{8,9}, alma-{8,9}-minimal * update per PR comments * Fix apt typo... * udpate per PR code review * revert README.md * update per samruddhikhandale's feedback. * Update src/git/install.sh Co-authored-by: Samruddhi Khandale * stop non-fatal error messages in RHEL tests. * remove tests for alma-[89]-minimal base images for now. * add install system git tests for RHEL base images * *_system_* tests install from system repositories, not latest version * fix broken install_git_from_system_mariner test * fix broken install_git_from_system_mariner test --------- Co-authored-by: Jeff Putsch Co-authored-by: Samruddhi Khandale --- src/git/NOTES.md | 2 +- src/git/devcontainer-feature.json | 6 +- src/git/install.sh | 154 +++++++++++++++---- test/common-utils/alma-minimal-8.sh | 16 ++ test/common-utils/alma-minimal-9.sh | 16 ++ test/git/install_git_from_src.sh | 1 + test/git/install_git_from_src_alma-8.sh | 16 ++ test/git/install_git_from_src_alma-9.sh | 16 ++ test/git/install_git_from_src_bionic.sh | 1 + test/git/install_git_from_src_bullseye.sh | 1 + test/git/install_git_from_src_buster.sh | 1 + test/git/install_git_from_src_centos-7.sh | 16 ++ test/git/install_git_from_src_fedora.sh | 16 ++ test/git/install_git_from_src_jammy.sh | 1 + test/git/install_git_from_src_mariner.sh | 16 ++ test/git/install_git_from_src_rocky-8.sh | 16 ++ test/git/install_git_from_src_rocky-9.sh | 16 ++ test/git/install_git_from_system_alma-8.sh | 16 ++ test/git/install_git_from_system_alma-9.sh | 16 ++ test/git/install_git_from_system_centos-7.sh | 16 ++ test/git/install_git_from_system_fedora.sh | 16 ++ test/git/install_git_from_system_mariner.sh | 15 ++ test/git/install_git_from_system_rocky-8.sh | 16 ++ test/git/install_git_from_system_rocky-9.sh | 16 ++ test/git/scenarios.json | 128 ++++++++++++++- 25 files changed, 513 insertions(+), 37 deletions(-) mode change 100755 => 100644 src/git/install.sh create mode 100755 test/common-utils/alma-minimal-8.sh create mode 100755 test/common-utils/alma-minimal-9.sh create mode 100644 test/git/install_git_from_src_alma-8.sh create mode 100644 test/git/install_git_from_src_alma-9.sh create mode 100644 test/git/install_git_from_src_centos-7.sh create mode 100644 test/git/install_git_from_src_fedora.sh create mode 100644 test/git/install_git_from_src_mariner.sh create mode 100644 test/git/install_git_from_src_rocky-8.sh create mode 100644 test/git/install_git_from_src_rocky-9.sh create mode 100644 test/git/install_git_from_system_alma-8.sh create mode 100644 test/git/install_git_from_system_alma-9.sh create mode 100644 test/git/install_git_from_system_centos-7.sh create mode 100644 test/git/install_git_from_system_fedora.sh create mode 100644 test/git/install_git_from_system_mariner.sh create mode 100644 test/git/install_git_from_system_rocky-8.sh create mode 100644 test/git/install_git_from_system_rocky-9.sh diff --git a/src/git/NOTES.md b/src/git/NOTES.md index 19fe92f31..e809f2099 100644 --- a/src/git/NOTES.md +++ b/src/git/NOTES.md @@ -2,6 +2,6 @@ ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the `apt`, `yum`, `dnf`, or `microdnf` package manager installed. `bash` is required to execute the `install.sh` script. diff --git a/src/git/devcontainer-feature.json b/src/git/devcontainer-feature.json index 8cbca6cf8..8d88c4f3b 100644 --- a/src/git/devcontainer-feature.json +++ b/src/git/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "git", - "version": "1.1.6", + "version": "1.2.0", "name": "Git (from source)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/git", "description": "Install an up-to-date version of Git, built from source as needed. Useful for when you want the latest and greatest features. Auto-detects latest stable version and installs needed dependencies.", @@ -17,10 +17,10 @@ "ppa": { "type": "boolean", "default": true, - "description": "Install from PPA if available" + "description": "Install from PPA if available (only supported for Ubuntu distributions)" } }, "installsAfter": [ "ghcr.io/devcontainers/features/common-utils" ] -} +} \ No newline at end of file diff --git a/src/git/install.sh b/src/git/install.sh old mode 100755 new mode 100644 index 67338b8da..8a0731933 --- a/src/git/install.sh +++ b/src/git/install.sh @@ -16,16 +16,51 @@ keyserver hkp://keyserver.ubuntu.com:80 keyserver hkps://keys.openpgp.org keyserver hkp://keyserver.pgp.com" -set -e - -# Clean up -rm -rf /var/lib/apt/lists/* - if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 fi +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + VERSION_CODENAME="${ID}{$VERSION_ID}" +else + echo "Linux distro ${ID} not supported." + exit 1 +fi + +if type apt-get > /dev/null 2>&1; then + INSTALL_CMD=apt-get +elif type microdnf > /dev/null 2>&1; then + INSTALL_CMD=microdnf +elif type dnf > /dev/null 2>&1; then + INSTALL_CMD=dnf +elif type yum > /dev/null 2>&1; then + INSTALL_CMD=yum +else + echo "(Error) Unable to find a supported package manager." + exit 1 +fi + +# Clean up +clean_up() { + case $ADJUSTED_ID in + debian) + rm -rf /var/lib/apt/lists/* + ;; + rhel) + rm -rf /var/cache/dnf/* + rm -rf /var/cache/yum/* + ;; + esac +} +clean_up + # Import the specified key in a variable name passed in as receive_gpg_keys() { local keys=${!1} @@ -61,40 +96,73 @@ receive_gpg_keys() { fi } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - apt-get update -y +pkg_mgr_update() { + if [ ${INSTALL_CMD} = "apt-get" ]; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${INSTALL_CMD} update -y + fi + elif [ ${INSTALL_CMD} = "dnf" ] || [ ${INSTALL_CMD} = "yum" ]; then + if [ "$(find /var/cache/${INSTALL_CMD}/* | wc -l)" = "0" ]; then + echo "Running ${INSTALL_CMD} check-update ..." + ${INSTALL_CMD} check-update + fi fi } + # Checks if packages are installed and installs them if not check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - apt_get_update - apt-get -y install --no-install-recommends "$@" + if [ ${INSTALL_CMD} = "apt-get" ]; then + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} -y install --no-install-recommends "$@" + fi + elif [ ${INSTALL_CMD} = "dnf" ] || [ ${INSTALL_CMD} = "yum" ]; then + _num_pkgs=$(echo "$@" | tr ' ' \\012 | wc -l) + _num_installed=$(${INSTALL_CMD} -C list installed "$@" | sed '1,/^Installed/d' | wc -l) + if [ ${_num_pkgs} != ${_num_installed} ]; then + pkg_mgr_update + ${INSTALL_CMD} -y install "$@" + fi + elif [ ${INSTALL_CMD} = "microdnf" ]; then + ${INSTALL_CMD} -y install \ + --refresh \ + --best \ + --nodocs \ + --noplugins \ + --setopt=install_weak_deps=0 \ + "$@" + else + echo "Linux distro ${ID} not supported." + exit 1 fi } export DEBIAN_FRONTEND=noninteractive -# Source /etc/os-release to get OS info -. /etc/os-release +# Debian / Ubuntu packages # If the os provided version is "good enough", just install that. if [ ${GIT_VERSION} = "os-provided" ] || [ ${GIT_VERSION} = "system" ]; then if type git > /dev/null 2>&1; then echo "Detected existing system install: $(git version)" # Clean up - rm -rf /var/lib/apt/lists/* + clean_up exit 0 fi - echo "Installing git from OS apt repository" + if [ "$INSTALL_CMD" = "apt-get" ]; then + echo "Installing git from OS apt repository" + else + echo "Installing git from OS yum/dnf repository" + fi + if [ $ID = "mariner" ]; then + check_packages ca-certificates + fi check_packages git # Clean up - rm -rf /var/lib/apt/lists/* + clean_up exit 0 fi @@ -104,15 +172,47 @@ if ([ "${GIT_VERSION}" = "latest" ] || [ "${GIT_VERSION}" = "lts" ] || [ "${GIT_ check_packages apt-transport-https curl ca-certificates gnupg2 dirmngr receive_gpg_keys GIT_CORE_PPA_ARCHIVE_GPG_KEY /usr/share/keyrings/gitcoreppa-archive-keyring.gpg echo -e "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gitcoreppa-archive-keyring.gpg] http://ppa.launchpad.net/git-core/ppa/ubuntu ${VERSION_CODENAME} main\ndeb-src [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gitcoreppa-archive-keyring.gpg] http://ppa.launchpad.net/git-core/ppa/ubuntu ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/git-core-ppa.list - apt-get update - apt-get -y install --no-install-recommends git + ${INSTALL_CMD} update + ${INSTALL_CMD} -y install --no-install-recommends git rm -rf "/tmp/tmp-gnupg" rm -rf /var/lib/apt/lists/* exit 0 fi # Install required packages to build if missing -check_packages build-essential curl ca-certificates tar gettext libssl-dev zlib1g-dev libcurl?-openssl-dev libexpat1-dev +if [ "${ADJUSTED_ID}" = "debian" ]; then + + check_packages build-essential curl ca-certificates tar gettext libssl-dev zlib1g-dev libcurl?-openssl-dev libexpat1-dev + + check_packages libpcre2-dev + + if [ "${VERSION_CODENAME}" = "focal" ] || [ "${VERSION_CODENAME}" = "bullseye" ]; then + check_packages libpcre2-posix2 + elif [ "${VERSION_CODENAME}" = "bionic" ] || [ "${VERSION_CODENAME}" = "buster" ]; then + check_packages libpcre2-posix0 + else + check_packages libpcre2-posix3 + fi + +elif [ "${ADJUSTED_ID}" = "rhel" ]; then + + if [ $VERSION_CODENAME = "centos7" ]; then + check_packages centos-release-scl + check_packages devtoolset-11 + source /opt/rh/devtoolset-11/enable + else + check_packages gcc + fi + + + check_packages libcurl-devel expat-devel gettext-devel openssl-devel perl-devel zlib-devel cmake pcre2-devel tar gzip ca-certificates + if ! type curl > /dev/null 2>&1; then + check_packages curl + fi + if [ $ID = "mariner" ]; then + check_packages glibc-devel kernel-headers binutils + fi +fi # Partial version matching if [ "$(echo "${GIT_VERSION}" | grep -o '\.' | wc -l)" != "2" ]; then @@ -131,21 +231,11 @@ if [ "$(echo "${GIT_VERSION}" | grep -o '\.' | wc -l)" != "2" ]; then fi fi -check_packages libpcre2-dev - -if [ "${VERSION_CODENAME}" = "focal" ] || [ "${VERSION_CODENAME}" = "bullseye" ]; then - check_packages libpcre2-posix2 -elif [ "${VERSION_CODENAME}" = "bionic" ] || [ "${VERSION_CODENAME}" = "buster" ]; then - check_packages libpcre2-posix0 -else - check_packages libpcre2-posix3 -fi - echo "Downloading source for ${GIT_VERSION}..." curl -sL https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz | tar -xzC /tmp 2>&1 echo "Building..." cd /tmp/git-${GIT_VERSION} make -s USE_LIBPCRE=YesPlease prefix=/usr/local sysconfdir=/etc all && make -s USE_LIBPCRE=YesPlease prefix=/usr/local sysconfdir=/etc install 2>&1 rm -rf /tmp/git-${GIT_VERSION} -rm -rf /var/lib/apt/lists/* +clean_up echo "Done!" diff --git a/test/common-utils/alma-minimal-8.sh b/test/common-utils/alma-minimal-8.sh new file mode 100755 index 000000000..cb2b339e1 --- /dev/null +++ b/test/common-utils/alma-minimal-8.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el9" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/alma-minimal-9.sh b/test/common-utils/alma-minimal-9.sh new file mode 100755 index 000000000..cb2b339e1 --- /dev/null +++ b/test/common-utils/alma-minimal-9.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${PLATFORM_ID}" = "platform:el9" +check "curl" curl --version +check "jq" jq --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/git/install_git_from_src.sh b/test/git/install_git_from_src.sh index c3552240d..d0ebaa282 100644 --- a/test/git/install_git_from_src.sh +++ b/test/git/install_git_from_src.sh @@ -5,6 +5,7 @@ set -e # Optional: Import test library source dev-container-features-test-lib +# Definition specific tests check "version" git --version check "gettext" dpkg-query -l gettext diff --git a/test/git/install_git_from_src_alma-8.sh b/test/git/install_git_from_src_alma-8.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_src_alma-8.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_src_alma-9.sh b/test/git/install_git_from_src_alma-9.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_src_alma-9.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_src_bionic.sh b/test/git/install_git_from_src_bionic.sh index 4bdcf3066..6a1f639cf 100644 --- a/test/git/install_git_from_src_bionic.sh +++ b/test/git/install_git_from_src_bionic.sh @@ -5,6 +5,7 @@ set -e # Optional: Import test library source dev-container-features-test-lib +# Definition specific tests check "version" git --version check "gettext" dpkg-query -l gettext diff --git a/test/git/install_git_from_src_bullseye.sh b/test/git/install_git_from_src_bullseye.sh index 4bdcf3066..6a1f639cf 100644 --- a/test/git/install_git_from_src_bullseye.sh +++ b/test/git/install_git_from_src_bullseye.sh @@ -5,6 +5,7 @@ set -e # Optional: Import test library source dev-container-features-test-lib +# Definition specific tests check "version" git --version check "gettext" dpkg-query -l gettext diff --git a/test/git/install_git_from_src_buster.sh b/test/git/install_git_from_src_buster.sh index 4bdcf3066..6a1f639cf 100644 --- a/test/git/install_git_from_src_buster.sh +++ b/test/git/install_git_from_src_buster.sh @@ -5,6 +5,7 @@ set -e # Optional: Import test library source dev-container-features-test-lib +# Definition specific tests check "version" git --version check "gettext" dpkg-query -l gettext diff --git a/test/git/install_git_from_src_centos-7.sh b/test/git/install_git_from_src_centos-7.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_src_centos-7.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_src_fedora.sh b/test/git/install_git_from_src_fedora.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_src_fedora.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_src_jammy.sh b/test/git/install_git_from_src_jammy.sh index 4bdcf3066..6a1f639cf 100644 --- a/test/git/install_git_from_src_jammy.sh +++ b/test/git/install_git_from_src_jammy.sh @@ -5,6 +5,7 @@ set -e # Optional: Import test library source dev-container-features-test-lib +# Definition specific tests check "version" git --version check "gettext" dpkg-query -l gettext diff --git a/test/git/install_git_from_src_mariner.sh b/test/git/install_git_from_src_mariner.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_src_mariner.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_src_rocky-8.sh b/test/git/install_git_from_src_rocky-8.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_src_rocky-8.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_src_rocky-9.sh b/test/git/install_git_from_src_rocky-9.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_src_rocky-9.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_system_alma-8.sh b/test/git/install_git_from_system_alma-8.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_system_alma-8.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_system_alma-9.sh b/test/git/install_git_from_system_alma-9.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_system_alma-9.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_system_centos-7.sh b/test/git/install_git_from_system_centos-7.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_system_centos-7.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_system_fedora.sh b/test/git/install_git_from_system_fedora.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_system_fedora.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_system_mariner.sh b/test/git/install_git_from_system_mariner.sh new file mode 100644 index 000000000..2114dfdf6 --- /dev/null +++ b/test/git/install_git_from_system_mariner.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +# Ensure git clone works, i.e. ca-certificates are installed. +check "git clone" bash -c "cd /tmp && git clone https://github.com/devcontainers/feature-starter.git" + +# Report result +reportResults diff --git a/test/git/install_git_from_system_rocky-8.sh b/test/git/install_git_from_system_rocky-8.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_system_rocky-8.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/install_git_from_system_rocky-9.sh b/test/git/install_git_from_system_rocky-9.sh new file mode 100644 index 000000000..84800b543 --- /dev/null +++ b/test/git/install_git_from_system_rocky-9.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" git --version + +cd /tmp && git clone https://github.com/devcontainers/feature-starter.git +cd feature-starter +check "perl" bash -c "git -c grep.patternType=perl grep -q 'a.+b'" + +# Report result +reportResults diff --git a/test/git/scenarios.json b/test/git/scenarios.json index 22af7a860..583c1adbe 100644 --- a/test/git/scenarios.json +++ b/test/git/scenarios.json @@ -43,5 +43,131 @@ "ppa": "false" } } + }, + "install_git_from_src_centos-7": { + "image": "centos:centos7", + "features": { + "git": { + "version": "latest", + "ppa": "false" + } + } + }, + "install_git_from_src_alma-8": { + "image": "almalinux:8", + "features": { + "git": { + "version": "latest", + "ppa": "false" + } + } + }, + "install_git_from_src_alma-9": { + "image": "almalinux:9", + "features": { + "git": { + "version": "latest", + "ppa": "false" + } + } + }, + "install_git_from_src_rocky-8": { + "image": "rockylinux:8", + "features": { + "git": { + "version": "latest", + "ppa": "false" + } + } + }, + "install_git_from_src_rocky-9": { + "image": "rockylinux:9", + "features": { + "git": { + "version": "latest", + "ppa": "false" + } + } + }, + "install_git_from_src_fedora": { + "image": "fedora", + "features": { + "git": { + "version": "latest", + "ppa": "false" + } + } + }, + "install_git_from_src_mariner": { + "image": "mcr.microsoft.com/cbl-mariner/base/core:2.0", + "features": { + "git": { + "version": "latest", + "ppa": "false" + } + } + }, + "install_git_from_system_centos-7": { + "image": "centos:centos7", + "features": { + "git": { + "version": "system", + "ppa": "true" + } + } + }, + "install_git_from_system_alma-8": { + "image": "almalinux:8", + "features": { + "git": { + "version": "system", + "ppa": "true" + } + } + }, + "install_git_from_system_alma-9": { + "image": "almalinux:9", + "features": { + "git": { + "version": "system", + "ppa": "true" + } + } + }, + "install_git_from_system_rocky-8": { + "image": "rockylinux:8", + "features": { + "git": { + "version": "system", + "ppa": "true" + } + } + }, + "install_git_from_system_rocky-9": { + "image": "rockylinux:9", + "features": { + "git": { + "version": "system", + "ppa": "true" + } + } + }, + "install_git_from_system_fedora": { + "image": "fedora", + "features": { + "git": { + "version": "system", + "ppa": "true" + } + } + }, + "install_git_from_system_mariner": { + "image": "mcr.microsoft.com/cbl-mariner/base/core:2.0", + "features": { + "git": { + "version": "system", + "ppa": "true" + } + } } -} +} \ No newline at end of file From 239e6dd5f2a8cf4f9b2143b93796094887efa523 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:53:39 -0800 Subject: [PATCH 016/285] Automated documentation update (#818) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/git/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/git/README.md b/src/git/README.md index a9e54fa18..adcb04e36 100644 --- a/src/git/README.md +++ b/src/git/README.md @@ -16,13 +16,13 @@ Install an up-to-date version of Git, built from source as needed. Useful for wh | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select or enter a Git version. | string | os-provided | -| ppa | Install from PPA if available | boolean | true | +| ppa | Install from PPA if available (only supported for Ubuntu distributions) | boolean | true | ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the `apt`, `yum`, `dnf`, or `microdnf` package manager installed. `bash` is required to execute the `install.sh` script. From 84883548465e18f88b1831d11ac363b253ecfc43 Mon Sep 17 00:00:00 2001 From: "David J. Malan" Date: Mon, 29 Jan 2024 16:06:27 -0500 Subject: [PATCH 017/285] Adds a number for formatting consistency (#813) * Adds number for consistency * Added numbering to NOTES.md instead * Update README.md --- src/docker-outside-of-docker/NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docker-outside-of-docker/NOTES.md b/src/docker-outside-of-docker/NOTES.md index fede053ae..6e85010ea 100644 --- a/src/docker-outside-of-docker/NOTES.md +++ b/src/docker-outside-of-docker/NOTES.md @@ -36,7 +36,7 @@ services: - The defaults value `./` is added so that the `docker-compose.yaml` file can work when it is run outside of the container -### Change the workspace to `${localWorkspaceFolder}` +### 2. Change the workspace to `${localWorkspaceFolder}` - This is useful if we don't want to edit the `docker-compose.yaml` file From b08484e74f82ceaa35a48bef4b126b2c25343790 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:08:33 -0800 Subject: [PATCH 018/285] Automated documentation update (#819) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/docker-outside-of-docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docker-outside-of-docker/README.md b/src/docker-outside-of-docker/README.md index 94c891564..74dca7ae4 100644 --- a/src/docker-outside-of-docker/README.md +++ b/src/docker-outside-of-docker/README.md @@ -66,7 +66,7 @@ services: - The defaults value `./` is added so that the `docker-compose.yaml` file can work when it is run outside of the container -### Change the workspace to `${localWorkspaceFolder}` +### 2. Change the workspace to `${localWorkspaceFolder}` - This is useful if we don't want to edit the `docker-compose.yaml` file From a88a95e1e8a10965eafe2e95438da61fbf4e09c0 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 31 Jan 2024 09:01:17 -0800 Subject: [PATCH 019/285] Oryx: Update to use .NET 8 (#825) * Oryx: Update to use .NET 8 * Fix install_prev_dotnet_and_oryx * fix install_prev_dotnet_and_oryx --- src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 12 +++--- test/oryx/install_dotnet_and_oryx.sh | 6 +-- test/oryx/install_older_dotnet_and_oryx.sh | 47 ++++++++++++++++++++++ test/oryx/install_prev_dotnet_and_oryx.sh | 6 +-- test/oryx/scenarios.json | 20 ++++++--- 6 files changed, 75 insertions(+), 18 deletions(-) create mode 100644 test/oryx/install_older_dotnet_and_oryx.sh diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index 66a528ec5..aae6d43db 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.1.0", + "version": "1.2.0", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index cb2e1b6b7..a16890bef 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -74,7 +74,7 @@ install_dotnet_using_apt() { echo "Attempting to auto-install dotnet..." install_from_microsoft_feed=false apt_get_update - DOTNET_INSTALLATION_PACKAGE="dotnet7" + DOTNET_INSTALLATION_PACKAGE="dotnet8" apt-get -yq install $DOTNET_INSTALLATION_PACKAGE || install_from_microsoft_feed="true" if [ "${install_from_microsoft_feed}" = "true" ]; then @@ -82,7 +82,7 @@ install_dotnet_using_apt() { curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list apt-get update -y - DOTNET_INSTALLATION_PACKAGE="dotnet-sdk-7.0" + DOTNET_INSTALLATION_PACKAGE="dotnet-sdk-8.0" DOTNET_SKIP_FIRST_TIME_EXPERIENCE="true" apt-get install -yq $DOTNET_INSTALLATION_PACKAGE fi @@ -132,9 +132,11 @@ if dotnet --version > /dev/null ; then DOTNET_BINARY=$(which dotnet) fi -# Oryx needs to be built with .NET 7 -if [[ "${DOTNET_BINARY}" = "" ]] || [[ "$(dotnet --version)" != *"7"* ]] ; then - echo "'dotnet 7' was not detected. Attempting to install .NET 7 to build oryx." +MAJOR_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 1) + +# Oryx needs to be built with .NET 8 +if [[ "${DOTNET_BINARY}" = "" ]] || [[ $MAJOR_VERSION_ID != "8" ]] ; then + echo "'dotnet 8' was not detected. Attempting to install .NET 8 to build oryx." install_dotnet_using_apt if ! dotnet --version > /dev/null ; then diff --git a/test/oryx/install_dotnet_and_oryx.sh b/test/oryx/install_dotnet_and_oryx.sh index 175482847..da0a9162d 100644 --- a/test/oryx/install_dotnet_and_oryx.sh +++ b/test/oryx/install_dotnet_and_oryx.sh @@ -26,9 +26,9 @@ mkdir -p /opt/oryx echo "vso-focal" >> /opt/oryx/.imagetype mkdir -p /opt/dotnet/lts -cp -R /usr/local/dotnet/current/dotnet /opt/dotnet/lts -cp -R /usr/local/dotnet/current/LICENSE.txt /opt/dotnet/lts -cp -R /usr/local/dotnet/current/ThirdPartyNotices.txt /opt/dotnet/lts +cp -R /usr/share/dotnet/dotnet /opt/dotnet/lts +cp -R /usr/share/dotnet/LICENSE.txt /opt/dotnet/lts +cp -R /usr/share/dotnet/ThirdPartyNotices.txt /opt/dotnet/lts # Install platforms with oryx build tool check "oryx-install-dotnet-2.1-universal" oryx prep --skip-detection --platforms-and-versions dotnet=2.1.30 diff --git a/test/oryx/install_older_dotnet_and_oryx.sh b/test/oryx/install_older_dotnet_and_oryx.sh new file mode 100644 index 000000000..da0a9162d --- /dev/null +++ b/test/oryx/install_older_dotnet_and_oryx.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "Oryx version" oryx --version +check "Dotnet is not removed if it is not installed by the Oryx Feature" dotnet --version + +# Install platforms with oryx build tool +check "oryx-install-dotnet-2.1" oryx prep --skip-detection --platforms-and-versions dotnet=2.1.30 +check "dotnet-2-installed-by-oryx" ls /opt/dotnet/ | grep 2.1 + +check "oryx-install-nodejs-12.22.11" oryx prep --skip-detection --platforms-and-versions nodejs=12.22.11 +check "nodejs-12.22.11-installed-by-oryx" ls /opt/nodejs/ | grep 12.22.11 + +check "oryx-install-php-7.3.25" oryx prep --skip-detection --platforms-and-versions php=7.3.25 +check "php-7.3.25-installed-by-oryx" ls /opt/php/ | grep 7.3.25 + +check "oryx-install-java-12.0.2" oryx prep --skip-detection --platforms-and-versions java=12.0.2 +check "java-12.0.2-installed-by-oryx" ls /opt/java/ | grep 12.0.2 + +# Replicates Oryx's behavior for universal image +mkdir -p /opt/oryx +echo "vso-focal" >> /opt/oryx/.imagetype + +mkdir -p /opt/dotnet/lts +cp -R /usr/share/dotnet/dotnet /opt/dotnet/lts +cp -R /usr/share/dotnet/LICENSE.txt /opt/dotnet/lts +cp -R /usr/share/dotnet/ThirdPartyNotices.txt /opt/dotnet/lts + +# Install platforms with oryx build tool +check "oryx-install-dotnet-2.1-universal" oryx prep --skip-detection --platforms-and-versions dotnet=2.1.30 +check "dotnet-2-installed-by-oryx-universal" ls /opt/dotnet/ | grep 2.1 + +check "oryx-install-nodejs-12.22.11-universal" oryx prep --skip-detection --platforms-and-versions nodejs=12.22.11 +check "nodejs-12.22.11-installed-by-oryx-universal" ls /opt/nodejs/ | grep 12.22.11 + +check "oryx-install-php-7.3.25-universal" oryx prep --skip-detection --platforms-and-versions php=7.3.25 +check "php-7.3.25-installed-by-oryx-universal" ls /opt/php/ | grep 7.3.25 + +check "oryx-install-java-12.0.2-universal" oryx prep --skip-detection --platforms-and-versions java=12.0.2 +check "java-12.0.2-installed-by-oryx-universal" ls /opt/java/ | grep 12.0.2 + +# Report result +reportResults diff --git a/test/oryx/install_prev_dotnet_and_oryx.sh b/test/oryx/install_prev_dotnet_and_oryx.sh index 175482847..da0a9162d 100644 --- a/test/oryx/install_prev_dotnet_and_oryx.sh +++ b/test/oryx/install_prev_dotnet_and_oryx.sh @@ -26,9 +26,9 @@ mkdir -p /opt/oryx echo "vso-focal" >> /opt/oryx/.imagetype mkdir -p /opt/dotnet/lts -cp -R /usr/local/dotnet/current/dotnet /opt/dotnet/lts -cp -R /usr/local/dotnet/current/LICENSE.txt /opt/dotnet/lts -cp -R /usr/local/dotnet/current/ThirdPartyNotices.txt /opt/dotnet/lts +cp -R /usr/share/dotnet/dotnet /opt/dotnet/lts +cp -R /usr/share/dotnet/LICENSE.txt /opt/dotnet/lts +cp -R /usr/share/dotnet/ThirdPartyNotices.txt /opt/dotnet/lts # Install platforms with oryx build tool check "oryx-install-dotnet-2.1-universal" oryx prep --skip-detection --platforms-and-versions dotnet=2.1.30 diff --git a/test/oryx/scenarios.json b/test/oryx/scenarios.json index c61483eb4..d7a70f319 100644 --- a/test/oryx/scenarios.json +++ b/test/oryx/scenarios.json @@ -3,8 +3,18 @@ "image": "ubuntu:focal", "features": { "dotnet": { - "version": "7", - "installUsingApt": "false" + "version": "8.0", + "dotnetRuntimeVersions": "7.0", + "aspNetCoreRuntimeVersions": "7.0" + }, + "oryx": {} + } + }, + "install_older_dotnet_and_oryx": { + "image": "ubuntu:focal", + "features": { + "dotnet": { + "version": "7.0" }, "oryx": {} } @@ -13,8 +23,7 @@ "image": "ubuntu:focal", "features": { "dotnet": { - "version": "6", - "installUsingApt": "false" + "version": "6.0" }, "oryx": {} } @@ -28,8 +37,7 @@ "useOryxIfAvailable": "false" }, "dotnet": { - "version": "7", - "installUsingApt": "false" + "version": "8.0" }, "oryx": {} } From 67de3c279a50a12175580e199071ecfc083c610b Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 31 Jan 2024 09:05:26 -0800 Subject: [PATCH 020/285] Anaconda: Fix bug - Pass proxy variables to new shell for successful anaconda installation behind proxies (#808) Anaconda: Fix bug - Pass proxy variables to new shell --- src/anaconda/devcontainer-feature.json | 2 +- src/anaconda/install.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/anaconda/devcontainer-feature.json b/src/anaconda/devcontainer-feature.json index e0d013d86..6b92b9147 100644 --- a/src/anaconda/devcontainer-feature.json +++ b/src/anaconda/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "anaconda", - "version": "1.0.11", + "version": "1.0.12", "name": "Anaconda", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/anaconda", "options": { diff --git a/src/anaconda/install.sh b/src/anaconda/install.sh index 53c606a1e..7c7af5b00 100755 --- a/src/anaconda/install.sh +++ b/src/anaconda/install.sh @@ -97,7 +97,8 @@ if ! conda --version &> /dev/null ; then CONDA_VERSION="2021.11" fi - su --login -c "wget -q https://repo.anaconda.com/archive/Anaconda3-${CONDA_VERSION}-Linux-x86_64.sh -O /tmp/anaconda-install.sh \ + su --login -c "export http_proxy=${http_proxy:-} && export https_proxy=${https_proxy:-} \ + && wget -q https://repo.anaconda.com/archive/Anaconda3-${CONDA_VERSION}-Linux-x86_64.sh -O /tmp/anaconda-install.sh \ && /bin/bash /tmp/anaconda-install.sh -u -b -p ${CONDA_DIR}" ${USERNAME} 2>&1 if [ "${VERSION}" = "latest" ] || [ "${VERSION}" = "lts" ]; then From 3063f5f105224ee039cddee4df8b6aaee934430e Mon Sep 17 00:00:00 2001 From: Jeff Putsch <150963136+jdputschadi@users.noreply.github.com> Date: Wed, 31 Jan 2024 09:21:33 -0800 Subject: [PATCH 021/285] Add RHEL support to Java feature (#821) * Initial rhel update * RHEL support added. * Add tests * quiet down test for ant/gradle/maven/groovy when they are not yet installed. * revert README -- it is autobuilt * Update src/java/NOTES.md add "`" around package manager names. Co-authored-by: Samruddhi Khandale * address PR feedback --------- Co-authored-by: Jeff Putsch Co-authored-by: Samruddhi Khandale --- src/java/NOTES.md | 2 +- src/java/devcontainer-feature.json | 4 +- src/java/install.sh | 160 +++++++++++++++--- test/java/alma-8-minimal.sh | 15 ++ test/java/alma-8.sh | 15 ++ test/java/alma-9-minimal.sh | 15 ++ test/java/alma-9.sh | 15 ++ test/java/centos-7.sh | 15 ++ test/java/fedora.sh | 15 ++ .../install_additional_java_rhel_family.sh | 13 ++ ...d_maven_and_groovy_for_user_rhel_family.sh | 34 ++++ ...gradle_and_maven_and_groovy_rhel_family.sh | 34 ++++ ...roovy_with_specific_version_rhel_family.sh | 30 ++++ ...all_from_non_default_distro_rhel_family.sh | 11 ++ ...ll_non_conventional_version_rhel_family.sh | 11 ++ test/java/mariner.sh | 15 ++ test/java/rocky-8-minimal.sh | 15 ++ test/java/rocky-8.sh | 15 ++ test/java/rocky-9-minimal.sh | 15 ++ test/java/rocky-9.sh | 15 ++ test/java/scenarios.json | 139 ++++++++++++++- 21 files changed, 574 insertions(+), 29 deletions(-) create mode 100644 test/java/alma-8-minimal.sh create mode 100644 test/java/alma-8.sh create mode 100644 test/java/alma-9-minimal.sh create mode 100644 test/java/alma-9.sh create mode 100644 test/java/centos-7.sh create mode 100644 test/java/fedora.sh create mode 100644 test/java/install_additional_java_rhel_family.sh create mode 100644 test/java/install_ant_and_gradle_and_maven_and_groovy_for_user_rhel_family.sh create mode 100644 test/java/install_ant_and_gradle_and_maven_and_groovy_rhel_family.sh create mode 100644 test/java/install_ant_and_gradle_and_maven_and_groovy_with_specific_version_rhel_family.sh create mode 100644 test/java/install_from_non_default_distro_rhel_family.sh create mode 100644 test/java/install_non_conventional_version_rhel_family.sh create mode 100644 test/java/mariner.sh create mode 100644 test/java/rocky-8-minimal.sh create mode 100644 test/java/rocky-8.sh create mode 100644 test/java/rocky-9-minimal.sh create mode 100644 test/java/rocky-9.sh diff --git a/src/java/NOTES.md b/src/java/NOTES.md index 63622afaf..edc310e23 100644 --- a/src/java/NOTES.md +++ b/src/java/NOTES.md @@ -5,6 +5,6 @@ For the Java Feature from this repository, see [NOTICE.txt](https://github.com/d ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the `apt`, `yum`, `dnf`, or `microdnf` package manager installed. `bash` is required to execute the `install.sh` script. diff --git a/src/java/devcontainer-feature.json b/src/java/devcontainer-feature.json index 64a4ca932..f65a8f360 100644 --- a/src/java/devcontainer-feature.json +++ b/src/java/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "java", - "version": "1.3.0", + "version": "1.4.0", "name": "Java (via SDKMAN!)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/java", "description": "Installs Java, SDKMAN! (if not installed), and needed dependencies.", @@ -111,4 +111,4 @@ "installsAfter": [ "ghcr.io/devcontainers/features/common-utils" ] -} +} \ No newline at end of file diff --git a/src/java/install.sh b/src/java/install.sh index 16bbb1c66..37cb9e52b 100644 --- a/src/java/install.sh +++ b/src/java/install.sh @@ -30,14 +30,65 @@ ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" set -e -# Clean up -rm -rf /var/lib/apt/lists/* - if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 fi +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +MAJOR_VERSION_ID=$(echo ${VERSION_ID} | cut -d . -f 1) +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + if [[ "${ID}" = "rhel" ]] || [[ "${ID}" = *"alma"* ]] || [[ "${ID}" = *"rocky"* ]]; then + VERSION_CODENAME="rhel${MAJOR_VERSION_ID}" + else + VERSION_CODENAME="${ID}${MAJOR_VERSION_ID}" + fi +else + echo "Linux distro ${ID} not supported." + exit 1 +fi + +# Setup INSTALL_CMD & PKG_MGR_CMD +if type apt-get > /dev/null 2>&1; then + PKG_MGR_CMD=apt-get + INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" +elif type microdnf > /dev/null 2>&1; then + PKG_MGR_CMD=microdnf + INSTALL_CMD="${PKG_MGR_CMD} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} -y install" +elif type yum > /dev/null 2>&1; then + PKG_MGR_CMD=yum + INSTALL_CMD="${PKG_MGR_CMD} -y install" +else + echo "(Error) Unable to find a supported package manager." + exit 1 +fi + +# Clean up +clean_up() { + local pkg + case ${ADJUSTED_ID} in + debian) + rm -rf /var/lib/apt/lists/* + ;; + rhel) + for pkg in epel-release epel-release-latest packages-microsoft-prod; do + ${PKG_MGR_CMD} -y remove $pkg 2>/dev/null || /bin/true + done + rm -rf /var/cache/dnf/* /var/cache/yum/* + rm -f /etc/yum.repos.d/docker-ce.repo + ;; + esac +} +clean_up + # Ensure that login shells get the correct path if the user updated the PATH using ENV. rm -f /etc/profile.d/00-restore-env.sh echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh @@ -61,31 +112,79 @@ elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then fi updaterc() { + local _bashrc + local _zshrc if [ "${UPDATE_RC}" = "true" ]; then - echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." - if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/bash.bashrc + case $ADJUSTED_ID in + debian) + _bashrc=/etc/bash.bashrc + _zshrc=/etc/zsh/zshrc + ;; + rhel) + _bashrc=/etc/bashrc + _zshrc=/etc/zshrc + ;; + esac + echo "Updating ${_bashrc} and ${_zshrc}..." + if [[ "$(cat ${_bashrc})" != *"$1"* ]]; then + echo -e "$1" >> "${_bashrc}" fi - if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/zsh/zshrc + if [ -f "${_zshrc}" ] && [[ "$(cat ${_zshrc})" != *"$1"* ]]; then + echo -e "$1" >> "${_zshrc}" fi fi } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - apt-get update -y - fi + +pkg_manager_update() { + case $ADJUSTED_ID in + debian) + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${PKG_MGR_CMD} update -y + fi + ;; + rhel) + if [ ${PKG_MGR_CMD} = "microdnf" ]; then + if [ "$(ls /var/cache/yum/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} makecache ..." + ${PKG_MGR_CMD} makecache + fi + else + if [ "$(ls /var/cache/${PKG_MGR_CMD}/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} check-update ..." + set +e + stderr_messages=$(${PKG_MGR_CMD} -q check-update 2>&1) + rc=$? + # centos 7 sometimes returns a status of 100 when it apears to work. + if [ $rc != 0 ] && [ $rc != 100 ]; then + echo "(Error) ${PKG_MGR_CMD} check-update produced the following error message(s):" + echo "${stderr_messages}" + exit 1 + fi + set -e + fi + fi + ;; + esac } # Checks if packages are installed and installs them if not check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - apt_get_update - apt-get -y install --no-install-recommends "$@" - fi + case ${ADJUSTED_ID} in + debian) + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_manager_update + ${INSTALL_CMD} "$@" + fi + ;; + rhel) + if ! rpm -q "$@" > /dev/null 2>&1; then + pkg_manager_update + ${INSTALL_CMD} "$@" + fi + ;; + esac } # Use Microsoft JDK for everything but JDK 8 and 18 (unless specified differently with jdkDistro option) @@ -142,8 +241,19 @@ if [ "${architecture}" != "amd64" ] && [ "${architecture}" != "x86_64" ] && [ "$ exit 1 fi -# Install dependencies -check_packages curl ca-certificates zip unzip sed +# Install dependencies, +check_packages ca-certificates zip unzip sed findutils util-linux tar +# Make sure passwd (Debian) and shadow-utils RHEL family is installed +if [ ${ADJUSTED_ID} = "debian" ]; then + check_packages passwd +elif [ ${ADJUSTED_ID} = "rhel" ]; then + check_packages shadow-utils +fi +# minimal RHEL installs may not include curl, or includes curl-minimal instead. +# Install curl if the "curl" command is not present. +if ! type curl > /dev/null 2>&1; then + check_packages curl +fi # Install sdkman if not installed if [ ! -d "${SDKMAN_DIR}" ]; then @@ -178,26 +288,26 @@ if [ ! -z "${ADDITIONAL_VERSIONS}" ]; then fi # Install Ant -if [[ "${INSTALL_ANT}" = "true" ]] && ! ant -version > /dev/null; then +if [[ "${INSTALL_ANT}" = "true" ]] && ! ant -version > /dev/null 2>&1; then sdk_install ant ${ANT_VERSION} fi # Install Gradle -if [[ "${INSTALL_GRADLE}" = "true" ]] && ! gradle --version > /dev/null; then +if [[ "${INSTALL_GRADLE}" = "true" ]] && ! gradle --version > /dev/null 2>&1; then sdk_install gradle ${GRADLE_VERSION} fi # Install Maven -if [[ "${INSTALL_MAVEN}" = "true" ]] && ! mvn --version > /dev/null; then +if [[ "${INSTALL_MAVEN}" = "true" ]] && ! mvn --version > /dev/null 2>&1; then sdk_install maven ${MAVEN_VERSION} fi # Install Groovy -if [[ "${INSTALL_GROOVY}" = "true" ]] && ! groovy --version > /dev/null; then +if [[ "${INSTALL_GROOVY}" = "true" ]] && ! groovy --version > /dev/null 2>&1; then sdk_install groovy "${GROOVY_VERSION}" fi # Clean up -rm -rf /var/lib/apt/lists/* +clean_up echo "Done!" diff --git a/test/java/alma-8-minimal.sh b/test/java/alma-8-minimal.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/alma-8-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/alma-8.sh b/test/java/alma-8.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/alma-8.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/alma-9-minimal.sh b/test/java/alma-9-minimal.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/alma-9-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/alma-9.sh b/test/java/alma-9.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/alma-9.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/centos-7.sh b/test/java/centos-7.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/centos-7.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/fedora.sh b/test/java/fedora.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/fedora.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/install_additional_java_rhel_family.sh b/test/java/install_additional_java_rhel_family.sh new file mode 100644 index 000000000..882c49d1c --- /dev/null +++ b/test/java/install_additional_java_rhel_family.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "java version 11 installed as default" grep "11\." <(java --version) +check "java version 17 installed" grep "^17\." <(ls /usr/local/sdkman/candidates/java) +check "java version 8 installed" grep "^8\." <(ls /usr/local/sdkman/candidates/java) + +# Report result +reportResults diff --git a/test/java/install_ant_and_gradle_and_maven_and_groovy_for_user_rhel_family.sh b/test/java/install_ant_and_gradle_and_maven_and_groovy_for_user_rhel_family.sh new file mode 100644 index 000000000..8d9f293e9 --- /dev/null +++ b/test/java/install_ant_and_gradle_and_maven_and_groovy_for_user_rhel_family.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "user is vscode" grep vscode <(whoami) + +check "java" java --version + +check "ant" ant -version +cat << EOF > /tmp/build.xml + +EOF +cd /tmp && ant init +check "ant-src exists" grep "ant-src" <(ls -la /tmp) + +check "gradle" gradle --version +cd /tmp && gradle init --type basic --dsl groovy --incubating --project-name test +check "GRADLE_USER_HOME exists" grep ".gradle" <(ls -la /home/vscode) + +check "maven" mvn --version +cd /tmp && mvn archetype:generate -DgroupId=com.mycompany.app -DartifactId=my-app -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false +check "m2 exists" grep ".m2" <(ls -la /home/vscode) + +check "groovy" groovy --version +cat << EOF > /tmp/test.groovy +println("verify") +EOF +check "groovy works" test "$(groovy /tmp/test.groovy)" = "verify" + +# Report result +reportResults diff --git a/test/java/install_ant_and_gradle_and_maven_and_groovy_rhel_family.sh b/test/java/install_ant_and_gradle_and_maven_and_groovy_rhel_family.sh new file mode 100644 index 000000000..e171f5be5 --- /dev/null +++ b/test/java/install_ant_and_gradle_and_maven_and_groovy_rhel_family.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "user is root" grep root <(whoami) + +check "java" java --version + +check "ant" ant -version +cat << EOF > /tmp/build.xml + +EOF +cd /tmp && ant init +check "ant-src exists" grep "ant-src" <(ls -la /tmp) + +check "gradle" gradle --version +cd /tmp && gradle init --type basic --dsl groovy --incubating --project-name test +check "GRADLE_USER_HOME exists" grep ".gradle" <(ls -la /root) + +check "maven" mvn --version +cd /tmp && mvn archetype:generate -DgroupId=com.mycompany.app -DartifactId=my-app -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false +check "m2 exists" grep ".m2" <(ls -la /root) + +check "groovy" groovy --version +cat << EOF > /tmp/test.groovy +println("verify") +EOF +check "groovy works" test "$(groovy /tmp/test.groovy)" = "verify" + +# Report result +reportResults diff --git a/test/java/install_ant_and_gradle_and_maven_and_groovy_with_specific_version_rhel_family.sh b/test/java/install_ant_and_gradle_and_maven_and_groovy_with_specific_version_rhel_family.sh new file mode 100644 index 000000000..54e13e0cf --- /dev/null +++ b/test/java/install_ant_and_gradle_and_maven_and_groovy_with_specific_version_rhel_family.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "user is root" grep root <(whoami) + +check "java" java --version + +check "ant version" grep "Ant(TM) version 1.10.12" <(ant -version) +cat << EOF > /tmp/build.xml + +EOF +cd /tmp && ant init +check "ant-src exists" grep "ant-src" <(ls -la /tmp) + +check "gradle version" grep "Gradle 6.8.3" <(gradle --version) +cd /tmp && gradle init --type basic --dsl groovy --project-name test +check "GRADLE_USER_HOME exists" grep ".gradle" <(ls -la /root) + +check "maven version" grep "Apache Maven 3.6.3" <(mvn --version) +cd /tmp && mvn archetype:generate -DgroupId=com.mycompany.app -DartifactId=my-app -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false +check "m2 exists" grep ".m2" <(ls -la /root) + +check "groovy version" grep "Groovy Version: 2.5.22" <(groovy --version) + +# Report result +reportResults diff --git a/test/java/install_from_non_default_distro_rhel_family.sh b/test/java/install_from_non_default_distro_rhel_family.sh new file mode 100644 index 000000000..3ab648cab --- /dev/null +++ b/test/java/install_from_non_default_distro_rhel_family.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "java version openjdk 21 installed" grep "openjdk 21." <(java --version) + +# Report result +reportResults diff --git a/test/java/install_non_conventional_version_rhel_family.sh b/test/java/install_non_conventional_version_rhel_family.sh new file mode 100644 index 000000000..3ab648cab --- /dev/null +++ b/test/java/install_non_conventional_version_rhel_family.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "java version openjdk 21 installed" grep "openjdk 21." <(java --version) + +# Report result +reportResults diff --git a/test/java/mariner.sh b/test/java/mariner.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/mariner.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/rocky-8-minimal.sh b/test/java/rocky-8-minimal.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/rocky-8-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/rocky-8.sh b/test/java/rocky-8.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/rocky-8.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/rocky-9-minimal.sh b/test/java/rocky-9-minimal.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/rocky-9-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/rocky-9.sh b/test/java/rocky-9.sh new file mode 100644 index 000000000..40f48562c --- /dev/null +++ b/test/java/rocky-9.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" java --version + +# Check env +check "JAVA_HOME is set correctly" echo $JAVA_HOME | grep "/usr/local/sdkman/candidates/java/current" + +# Report result +reportResults diff --git a/test/java/scenarios.json b/test/java/scenarios.json index dac9850f2..6e6426969 100644 --- a/test/java/scenarios.json +++ b/test/java/scenarios.json @@ -69,5 +69,142 @@ "jdkDistro": "graalce" } } + }, + "install_from_non_default_distro_rhel_family": { + "image": "almalinux:8", + "features": { + "java": { + "version": "21", + "jdkDistro": "open" + } + } + }, + "install_additional_java_rhel_family": { + "image": "almalinux:8", + "features": { + "java": { + "version": "11", + "additionalVersions": "17,8" + } + } + }, + "install_ant_and_gradle_and_maven_and_groovy_for_user_rhel_family": { + "image": "almalinux:8", + "remoteUser": "vscode", + "features": { + "common-utils": { + "username": "vscode" + }, + "java": { + "version": "latest", + "installAnt": true, + "installGradle": true, + "installMaven": true, + "installGroovy": true + } + } + }, + "install_ant_and_gradle_and_maven_and_groovy_rhel_family": { + "image": "almalinux:8", + "features": { + "java": { + "version": "latest", + "installAnt": true, + "installGradle": true, + "installMaven": true, + "installGroovy": true + } + } + }, + "install_ant_and_gradle_and_maven_and_groovy_with_specific_version_rhel_family": { + "image": "almalinux:8", + "features": { + "java": { + "version": "latest", + "installAnt": "true", + "antVersion": "1.10.12", + "installGradle": "true", + "gradleVersion": "6.8.3", + "installMaven": "true", + "mavenVersion": "3.6.3", + "installGroovy": "true", + "groovyVersion": "2.5.22" + } + } + }, + "install_non_conventional_version_rhel_family": { + "image": "almalinux:8", + "features": { + "java": { + "version": "21", + "jdkDistro": "graalce" + } + } + }, + "centos-7": { + "image": "centos:centos7", + "features": { + "java": {} + } + }, + "alma-8": { + "image": "almalinux:8", + "features": { + "java": {} + } + }, + "alma-8-minimal": { + "image": "almalinux:8-minimal", + "features": { + "java": {} + } + }, + "alma-9": { + "image": "almalinux:9", + "features": { + "java": {} + } + }, + "alma-9-minimal": { + "image": "almalinux:9-minimal", + "features": { + "java": {} + } + }, + "rocky-8": { + "image": "rockylinux:8", + "features": { + "java": {} + } + }, + "rocky-8-minimal": { + "image": "rockylinux:8-minimal", + "features": { + "java": {} + } + }, + "rocky-9": { + "image": "rockylinux:9", + "features": { + "java": {} + } + }, + "rocky-9-minimal": { + "image": "rockylinux:9-minimal", + "features": { + "java": {} + } + }, + "mariner": { + "image": "mcr.microsoft.com/cbl-mariner/base/core:2.0", + "features": { + "java": {} + } + }, + "fedora": { + "image": "fedora", + "features": { + "java": {} + } } -} +} \ No newline at end of file From 19d1edcbdf758d94f5ff7580da31c3f23dc2529c Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Wed, 31 Jan 2024 09:23:10 -0800 Subject: [PATCH 022/285] Automated documentation update (#826) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/java/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/README.md b/src/java/README.md index 4699eb517..938cd0f89 100644 --- a/src/java/README.md +++ b/src/java/README.md @@ -39,7 +39,7 @@ For the Java Feature from this repository, see [NOTICE.txt](https://github.com/d ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the `apt`, `yum`, `dnf`, or `microdnf` package manager installed. `bash` is required to execute the `install.sh` script. From 4f0d020909096dac60d3fb379b034eb2d02ad323 Mon Sep 17 00:00:00 2001 From: Jeff Putsch <150963136+jdputschadi@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:26:41 -0800 Subject: [PATCH 023/285] Add RHEL support to node feature. (#823) * node feature working for RHEL and family, passing all tests * RHEL support added * fix linting error by removing debugging code. * incorporate PR feedback * address PR feedback --------- Co-authored-by: Jeff Putsch --- src/node/NOTES.md | 4 +- src/node/devcontainer-feature.json | 4 +- src/node/install.sh | 274 ++++++++++++++---- test/node/alma-8-minimal.sh | 15 + test/node/alma-8.sh | 15 + test/node/alma-9-minimal.sh | 15 + test/node/alma-9.sh | 15 + test/node/centos-7.sh | 15 + test/node/fedora.sh | 15 + test/node/install_additional_node.sh | 6 +- .../install_additional_node_on_rhel_family.sh | 18 ++ test/node/install_nvm_0.39_on_rhel_family.sh | 14 + test/node/mariner.sh | 17 ++ test/node/rocky-8-minimal.sh | 15 + test/node/rocky-8.sh | 15 + test/node/rocky-9-minimal.sh | 15 + test/node/rocky-9.sh | 15 + test/node/scenarios.json | 127 +++++++- test/node/version_none_on_rhel_family.sh | 12 + test/node/zsh_default_on_rhel_family.sh | 12 + 20 files changed, 570 insertions(+), 68 deletions(-) create mode 100755 test/node/alma-8-minimal.sh create mode 100755 test/node/alma-8.sh create mode 100755 test/node/alma-9-minimal.sh create mode 100755 test/node/alma-9.sh create mode 100755 test/node/centos-7.sh create mode 100755 test/node/fedora.sh create mode 100644 test/node/install_additional_node_on_rhel_family.sh create mode 100755 test/node/install_nvm_0.39_on_rhel_family.sh create mode 100755 test/node/mariner.sh create mode 100755 test/node/rocky-8-minimal.sh create mode 100755 test/node/rocky-8.sh create mode 100755 test/node/rocky-9-minimal.sh create mode 100755 test/node/rocky-9.sh create mode 100755 test/node/version_none_on_rhel_family.sh create mode 100755 test/node/zsh_default_on_rhel_family.sh diff --git a/src/node/NOTES.md b/src/node/NOTES.md index 65eb93bf3..506fa1b4e 100644 --- a/src/node/NOTES.md +++ b/src/node/NOTES.md @@ -20,6 +20,8 @@ Alternatively, you can start up an interactive shell which will in turn source ` ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and Rocky Linux distributions with the `apt`, `yum`, `dnf`, or `microdnf` package manager installed. + +**Note**: RedHat 7 Family (RedHat, CentOS, etc.) must use Node versions less than 18 due to its system libraries and long-term support (LTS) policies. `bash` is required to execute the `install.sh` script. diff --git a/src/node/devcontainer-feature.json b/src/node/devcontainer-feature.json index 0123fdb73..a93f75ace 100644 --- a/src/node/devcontainer-feature.json +++ b/src/node/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "node", - "version": "1.3.1", + "version": "1.4.0", "name": "Node.js (via nvm), yarn and pnpm", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/node", "description": "Installs Node.js, nvm, yarn, pnpm, and needed dependencies.", @@ -53,4 +53,4 @@ "installsAfter": [ "ghcr.io/devcontainers/features/common-utils" ] -} +} \ No newline at end of file diff --git a/src/node/install.sh b/src/node/install.sh index 9a696ca54..493166d84 100755 --- a/src/node/install.sh +++ b/src/node/install.sh @@ -21,61 +21,136 @@ UPDATE_RC="${UPDATE_RC:-"true"}" set -e -# Clean up -rm -rf /var/lib/apt/lists/* - if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 fi +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +MAJOR_VERSION_ID=$(echo ${VERSION_ID} | cut -d . -f 1) +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + if [[ "${ID}" = "rhel" ]] || [[ "${ID}" = *"alma"* ]] || [[ "${ID}" = *"rocky"* ]]; then + VERSION_CODENAME="rhel${MAJOR_VERSION_ID}" + else + VERSION_CODENAME="${ID}${MAJOR_VERSION_ID}" + fi +else + echo "Linux distro ${ID} not supported." + exit 1 +fi + +# Setup INSTALL_CMD & PKG_MGR_CMD +if type apt-get > /dev/null 2>&1; then + PKG_MGR_CMD=apt-get + INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" +elif type microdnf > /dev/null 2>&1; then + PKG_MGR_CMD=microdnf + INSTALL_CMD="${PKG_MGR_CMD} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} -y install" +else + PKG_MGR_CMD=yum + INSTALL_CMD="${PKG_MGR_CMD} -y install" +fi + +# Clean up +clean_up() { + case ${ADJUSTED_ID} in + debian) + rm -rf /var/lib/apt/lists/* + ;; + rhel) + rm -rf /var/cache/dnf/* /var/cache/yum/* + rm -f /etc/yum.repos.d/yarn.repo + ;; + esac +} +clean_up + # Ensure that login shells get the correct path if the user updated the PATH using ENV. rm -f /etc/profile.d/00-restore-env.sh echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh chmod +x /etc/profile.d/00-restore-env.sh -# Determine the appropriate non-root user -if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then - USERNAME="" - POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") - for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do - if id -u ${CURRENT_USER} > /dev/null 2>&1; then - USERNAME=${CURRENT_USER} - break - fi - done - if [ "${USERNAME}" = "" ]; then - USERNAME=root - fi -elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then - USERNAME=root -fi - updaterc() { + local _bashrc + local _zshrc if [ "${UPDATE_RC}" = "true" ]; then - echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." - if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/bash.bashrc + case $ADJUSTED_ID in + debian) + _bashrc=/etc/bash.bashrc + _zshrc=/etc/zsh/zshrc + ;; + rhel) + _bashrc=/etc/bashrc + _zshrc=/etc/zshrc + ;; + esac + echo "Updating ${_bashrc} and ${_zshrc}..." + if [[ "$(cat ${_bashrc})" != *"$1"* ]]; then + echo -e "$1" >> "${_bashrc}" fi - if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/zsh/zshrc + if [ -f "${_zshrc}" ] && [[ "$(cat ${_zshrc})" != *"$1"* ]]; then + echo -e "$1" >> "${_zshrc}" fi fi } -apt_get_update() { - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - apt-get update -y - fi +pkg_mgr_update() { + case $ADJUSTED_ID in + debian) + if [ "$(find /var/lib/apt/lists/* 2>/dev/null | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${PKG_MGR_CMD} update -y + fi + ;; + rhel) + if [ ${PKG_MGR_CMD} = "microdnf" ]; then + if [ "$(ls /var/cache/yum/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} makecache ..." + ${PKG_MGR_CMD} makecache + fi + else + if [ "$(ls /var/cache/${PKG_MGR_CMD}/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} check-update ..." + set +e + stderr_messages=$(${PKG_MGR_CMD} -q check-update 2>&1) + rc=$? + # centos 7 sometimes returns a status of 100 when it apears to work. + if [ $rc != 0 ] && [ $rc != 100 ]; then + echo "(Error) ${PKG_MGR_CMD} check-update produced the following error message(s):" + echo "${stderr_messages}" + exit 1 + fi + set -e + fi + fi + ;; + esac } # Checks if packages are installed and installs them if not check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - apt_get_update - apt-get -y install --no-install-recommends "$@" - fi + case ${ADJUSTED_ID} in + debian) + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + rhel) + if ! rpm -q "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + esac } # Figure out correct version of a three part version number is not passed @@ -112,35 +187,95 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +install_yarn() { + if [ "${ADJUSTED_ID}" = "debian" ]; then + # for backward compatiblity with existing devcontainer features, install yarn + # via apt-get on Debian systems + if ! type yarn >/dev/null 2>&1; then + # Import key safely (new method rather than deprecated apt-key approach) and install + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor > /usr/share/keyrings/yarn-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/yarn-archive-keyring.gpg] https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list + apt-get update + apt-get -y install --no-install-recommends yarn + else + echo "Yarn is already installed." + fi + else + local _ver=${1:-node} + # on non-debian systems, prefer corepack, fallback to npm based installation of yarn... + # Try to leverage corepack if possible + # From https://yarnpkg.com: + # The preferred way to manage Yarn is by-project and through Corepack, a tool + # shipped by default with Node.js. Modern releases of Yarn aren't meant to be + # installed globally, or from npm. + if ! bash -c ". '${NVM_DIR}/nvm.sh' && nvm use ${_ver} && type yarn >/dev/null 2>&1"; then + if bash -c ". '${NVM_DIR}/nvm.sh' && nvm use ${_ver} && type corepack >/dev/null 2>&1"; then + su ${USERNAME} -c "umask 0002 && . '${NVM_DIR}/nvm.sh' && nvm use ${_ver} && corepack enable" + fi + if ! bash -c ". '${NVM_DIR}/nvm.sh' && nvm use ${_ver} && type yarn >/dev/null 2>&1"; then + # Yum/DNF want to install nodejs dependencies, we'll use NPM to install yarn + su ${USERNAME} -c "umask 0002 && . '${NVM_DIR}/nvm.sh' && nvm use ${_ver} && npm install --global yarn" + fi + else + echo "Yarn already installed." + fi + fi +} + +# Mariner does not have awk installed by default, this can cause +# problems is username is auto* and later when we try to install +# node via npm. +if ! type awk >/dev/null 2>&1; then + check_packages awk +fi + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + # Ensure apt is in non-interactive to avoid prompts export DEBIAN_FRONTEND=noninteractive -. /etc/os-release -if [[ "bionic" = *"${VERSION_CODENAME}"* ]]; then - if [[ "${NODE_VERSION}" =~ "18" ]] || [[ "${NODE_VERSION}" = "lts" ]]; then - echo "(!) Unsupported distribution version '${VERSION_CODENAME}' for Node 18. Details: https://github.com/nodejs/node/issues/42351#issuecomment-1068424442" +if ( [ -n "${VERSION_CODENAME}" ] && [[ "bionic" = *"${VERSION_CODENAME}"* ]] ) || [[ "rhel7" = *"${ADJUSTED_ID}${MAJOR_VERSION_ID}"* ]]; then + node_major_version=$(echo "${NODE_VERSION}" | cut -d . -f 1) + if [[ "${node_major_version}" -ge 18 ]] || [[ "${NODE_VERSION}" = "lts" ]] || [[ "${NODE_VERSION}" = "latest" ]]; then + echo "(!) Unsupported distribution version '${VERSION_CODENAME}' for Node >= 18. Details: https://github.com/nodejs/node/issues/42351#issuecomment-1068424442" exit 1 fi fi # Install dependencies -check_packages apt-transport-https curl ca-certificates tar gnupg2 dirmngr +case ${ADJUSTED_ID} in + debian) + check_packages apt-transport-https curl ca-certificates tar gnupg2 dirmngr + ;; + rhel) + check_packages ca-certificates tar gnupg2 which findutils util-linux tar + # minimal RHEL installs may not include curl, or includes curl-minimal instead. + # Install curl if the "curl" command is not present. + if ! type curl > /dev/null 2>&1; then + check_packages curl + fi + ;; +esac if ! type git > /dev/null 2>&1; then check_packages git fi -# Install yarn -if type yarn > /dev/null 2>&1; then - echo "Yarn already installed." -else - # Import key safely (new method rather than deprecated apt-key approach) and install - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor > /usr/share/keyrings/yarn-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/yarn-archive-keyring.gpg] https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list - apt-get update - apt-get -y install --no-install-recommends yarn -fi - # Adjust node version if required if [ "${NODE_VERSION}" = "none" ]; then export NODE_VERSION= @@ -159,6 +294,7 @@ umask 0002 # Do not update profile - we'll do this manually export PROFILE=/dev/null curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh" | bash + source "${NVM_DIR}/nvm.sh" if [ "${NODE_VERSION}" != "" ]; then nvm alias default "${NODE_VERSION}" @@ -204,6 +340,9 @@ else fi fi +# Possibly install yarn (puts yarn in per-Node install on RHEL, uses system yarn on Debian) +install_yarn + # Additional node versions to be installed but not be set as # default we can assume the nvm is the group owner of the nvm # directory and the sticky bit on directories so any installed @@ -214,6 +353,8 @@ if [ ! -z "${ADDITIONAL_VERSIONS}" ]; then read -a additional_versions <<< "$ADDITIONAL_VERSIONS" for ver in "${additional_versions[@]}"; do su ${USERNAME} -c "umask 0002 && . '$NVM_DIR/nvm.sh' && nvm install '${ver}'" + # possibly install yarn (puts yarn in per-Node install on RHEL, uses system yarn on Debian) + install_yarn "${ver}" done # Ensure $NODE_VERSION is on the $PATH @@ -224,14 +365,17 @@ if [ ! -z "${ADDITIONAL_VERSIONS}" ]; then fi # Install pnpm -if type pnpm > /dev/null 2>&1; then +if bash -c ". '${NVM_DIR}/nvm.sh' && type pnpm >/dev/null 2>&1"; then echo "pnpm already installed." else - if type npm > /dev/null 2>&1; then - [ ! -z "$http_proxy" ] && npm set proxy="$http_proxy" - [ ! -z "$https_proxy" ] && npm set https-proxy="$https_proxy" - [ ! -z "$no_proxy" ] && npm set noproxy="$no_proxy" - npm install -g pnpm + if bash -c ". '${NVM_DIR}/nvm.sh' && type npm >/dev/null 2>&1"; then + ( + . "${NVM_DIR}/nvm.sh" + [ ! -z "$http_proxy" ] && npm set proxy="$http_proxy" + [ ! -z "$https_proxy" ] && npm set https-proxy="$https_proxy" + [ ! -z "$no_proxy" ] && npm set noproxy="$no_proxy" + npm install -g pnpm + ) else echo "Skip installing pnpm because npm is missing" fi @@ -248,21 +392,29 @@ if [ "${INSTALL_TOOLS_FOR_NODE_GYP}" = "true" ]; then to_install="${to_install} gcc" fi if ! type g++ > /dev/null 2>&1; then - to_install="${to_install} g++" + if [ ${ADJUSTED_ID} = "debian" ]; then + to_install="${to_install} g++" + elif [ ${ADJUSTED_ID} = "rhel" ]; then + to_install="${to_install} gcc-c++" + fi fi if ! type python3 > /dev/null 2>&1; then - to_install="${to_install} python3-minimal" + if [ ${ADJUSTED_ID} = "debian" ]; then + to_install="${to_install} python3-minimal" + elif [ ${ADJUSTED_ID} = "rhel" ]; then + to_install="${to_install} python3" + fi fi if [ ! -z "${to_install}" ]; then - apt_get_update - apt-get -y install ${to_install} + pkg_mgr_update + check_packages ${to_install} fi fi # Clean up su ${USERNAME} -c "umask 0002 && . '$NVM_DIR/nvm.sh' && nvm clear-cache" -rm -rf /var/lib/apt/lists/* +clean_up # Ensure privs are correct for installed node versions. Unfortunately the # way nvm installs node versions pulls privs from the tar which does not diff --git a/test/node/alma-8-minimal.sh b/test/node/alma-8-minimal.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/alma-8-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/alma-8.sh b/test/node/alma-8.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/alma-8.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/alma-9-minimal.sh b/test/node/alma-9-minimal.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/alma-9-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/alma-9.sh b/test/node/alma-9.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/alma-9.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/centos-7.sh b/test/node/centos-7.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/centos-7.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/fedora.sh b/test/node/fedora.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/fedora.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/install_additional_node.sh b/test/node/install_additional_node.sh index f47f928c0..d8f27317f 100644 --- a/test/node/install_additional_node.sh +++ b/test/node/install_additional_node.sh @@ -5,11 +5,11 @@ set -e # Optional: Import test library source dev-container-features-test-lib -# 'lts' is some version of node 18 for a while. -check "version_on_path" node -v | grep 18 +# 'lts' is now some version of node 20... +check "version_on_path" node -v | grep 20 check "pnpm" pnpm -v -check "v18_installed" ls -1 /usr/local/share/nvm/versions/node | grep 18 +check "v20_installed" ls -1 /usr/local/share/nvm/versions/node | grep 20 check "v14_installed" ls -1 /usr/local/share/nvm/versions/node | grep 14.19.3 check "v17_installed" ls -1 /usr/local/share/nvm/versions/node | grep 17.9.1 diff --git a/test/node/install_additional_node_on_rhel_family.sh b/test/node/install_additional_node_on_rhel_family.sh new file mode 100644 index 000000000..d8f27317f --- /dev/null +++ b/test/node/install_additional_node_on_rhel_family.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# 'lts' is now some version of node 20... +check "version_on_path" node -v | grep 20 +check "pnpm" pnpm -v + +check "v20_installed" ls -1 /usr/local/share/nvm/versions/node | grep 20 +check "v14_installed" ls -1 /usr/local/share/nvm/versions/node | grep 14.19.3 +check "v17_installed" ls -1 /usr/local/share/nvm/versions/node | grep 17.9.1 + + +# Report result +reportResults diff --git a/test/node/install_nvm_0.39_on_rhel_family.sh b/test/node/install_nvm_0.39_on_rhel_family.sh new file mode 100755 index 000000000..6b96381b5 --- /dev/null +++ b/test/node/install_nvm_0.39_on_rhel_family.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" bash -c "node --version" +check "pnpm" pnpm -v +check "nvm version" bash -c ". /usr/local/share/nvm/nvm.sh && nvm --version | grep 0.39" + +# Report result +reportResults diff --git a/test/node/mariner.sh b/test/node/mariner.sh new file mode 100755 index 000000000..d4ebb7d69 --- /dev/null +++ b/test/node/mariner.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +# for some reason the "nvm" test switches the default node version on Mariner +# test yarn before that: it is only enabled in the default node version +check "yarn" yarn --version_list +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/rocky-8-minimal.sh b/test/node/rocky-8-minimal.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/rocky-8-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/rocky-8.sh b/test/node/rocky-8.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/rocky-8.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/rocky-9-minimal.sh b/test/node/rocky-9-minimal.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/rocky-9-minimal.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/rocky-9.sh b/test/node/rocky-9.sh new file mode 100755 index 000000000..c5980129b --- /dev/null +++ b/test/node/rocky-9.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" node --version +check "pnpm" pnpm -v +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" +check "yarn" yarn --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/node/scenarios.json b/test/node/scenarios.json index a45620fbe..262fba8ea 100644 --- a/test/node/scenarios.json +++ b/test/node/scenarios.json @@ -8,6 +8,15 @@ } } }, + "install_additional_node_on_rhel_family": { + "image": "almalinux:8", + "features": { + "node": { + "version": "lts", + "additionalVersions": "v17.9.1,v14.19.3" + } + } + }, "non_root_user": { "image": "mcr.microsoft.com/devcontainers/base", "remoteUser": "vscode", @@ -28,6 +37,17 @@ } } }, + "zsh_default_on_rhel_family": { + "image": "almalinux:8", + "features": { + "node": { + "version": "lts" + }, + "common-utils": { + "configureZshAsDefaultShell": true + } + } + }, "version_none": { "image": "mcr.microsoft.com/devcontainers/base", "remoteUser": "vscode", @@ -37,6 +57,15 @@ } } }, + "version_none_on_rhel_family": { + "image": "mcr.microsoft.com/devcontainers/base", + "remoteUser": "vscode", + "features": { + "node": { + "version": "none" + } + } + }, "install_node_16_on_bionic": { "image": "mcr.microsoft.com/devcontainers/base:ubuntu-18.04", "features": { @@ -52,5 +81,101 @@ "nvmVersion": "0.39" } } + }, + "install_nvm_0.39_on_rhel_family": { + "image": "almalinux:8", + "features": { + "node": { + "nvmVersion": "0.39" + } + } + }, + "centos-7": { + "image": "centos:centos7", + "features": { + "node": { + "version": "16" + } + } + }, + "alma-8": { + "image": "almalinux:8", + "features": { + "node": { + "version": "lts" + } + } + }, + "alma-8-minimal": { + "image": "almalinux:8-minimal", + "features": { + "node": { + "version": "lts" + } + } + }, + "alma-9": { + "image": "almalinux:9", + "features": { + "node": { + "version": "lts" + } + } + }, + "alma-9-minimal": { + "image": "almalinux:9-minimal", + "features": { + "node": { + "version": "lts" + } + } + }, + "rocky-8": { + "image": "rockylinux:8", + "features": { + "node": { + "version": "lts" + } + } + }, + "rocky-8-minimal": { + "image": "rockylinux:8-minimal", + "features": { + "node": { + "version": "lts" + } + } + }, + "rocky-9": { + "image": "rockylinux:9", + "features": { + "node": { + "version": "lts" + } + } + }, + "rocky-9-minimal": { + "image": "rockylinux:9-minimal", + "features": { + "node": { + "version": "lts" + } + } + }, + "fedora": { + "image": "fedora", + "features": { + "node": { + "version": "lts" + } + } + }, + "mariner": { + "image": "mcr.microsoft.com/cbl-mariner/base/core:2.0", + "features": { + "node": { + "version": "lts" + } + } } -} +} \ No newline at end of file diff --git a/test/node/version_none_on_rhel_family.sh b/test/node/version_none_on_rhel_family.sh new file mode 100755 index 000000000..c7c9e57f4 --- /dev/null +++ b/test/node/version_none_on_rhel_family.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" + +# Report result +reportResults diff --git a/test/node/zsh_default_on_rhel_family.sh b/test/node/zsh_default_on_rhel_family.sh new file mode 100755 index 000000000..c7c9e57f4 --- /dev/null +++ b/test/node/zsh_default_on_rhel_family.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "nvm" bash -c ". /usr/local/share/nvm/nvm.sh && nvm install 10" + +# Report result +reportResults From ccc86033c09f1161e6e55971455fafcd0ed80a4c Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:29:08 -0800 Subject: [PATCH 024/285] Automated documentation update (#828) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/node/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/node/README.md b/src/node/README.md index b98391922..6ac1b0ff0 100644 --- a/src/node/README.md +++ b/src/node/README.md @@ -48,7 +48,9 @@ Alternatively, you can start up an interactive shell which will in turn source ` ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and Rocky Linux distributions with the `apt`, `yum`, `dnf`, or `microdnf` package manager installed. + +**Note**: RedHat 7 Family (RedHat, CentOS, etc.) must use Node versions less than 18 due to its system libraries and long-term support (LTS) policies. `bash` is required to execute the `install.sh` script. From 6d6fb2bab02edd2431ce3b2151ef29a981d8a82a Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 31 Jan 2024 12:04:20 -0800 Subject: [PATCH 025/285] [docker-in-docker] : Bug fix - Buildx installation when moby:false (#827) --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 15 +++++++++------ test/docker-in-docker/docker_build.sh | 2 ++ test/docker-in-docker/docker_build_2.sh | 2 ++ test/docker-in-docker/docker_build_older.sh | 2 ++ test/docker-in-docker/docker_buildx.sh | 19 +++++++++++++++++++ test/docker-in-docker/scenarios.json | 9 +++++++++ 7 files changed, 44 insertions(+), 7 deletions(-) create mode 100755 test/docker-in-docker/docker_buildx.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 366905c2a..3e112ddcb 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.7.1", + "version": "2.8.0", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 3d47d4674..4e042e3e8 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -317,13 +317,16 @@ if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" cd /tmp && wget "https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name}" - mkdir -p ${_REMOTE_USER_HOME}/.docker/cli-plugins - mv ${buildx_file_name} ${_REMOTE_USER_HOME}/.docker/cli-plugins/docker-buildx - chmod +x ${_REMOTE_USER_HOME}/.docker/cli-plugins/docker-buildx + docker_home="/usr/libexec/docker" + cli_plugins_dir="${docker_home}/cli-plugins" - chown -R "${USERNAME}:docker" "${_REMOTE_USER_HOME}/.docker" - chmod -R g+r+w "${_REMOTE_USER_HOME}/.docker" - find "${_REMOTE_USER_HOME}/.docker" -type d -print0 | xargs -n 1 -0 chmod g+s + mkdir -p ${cli_plugins_dir} + mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx + chmod +x ${cli_plugins_dir}/docker-buildx + + chown -R "${USERNAME}:docker" "${docker_home}" + chmod -R g+r+w "${docker_home}" + find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s fi tee /usr/local/share/docker-init.sh > /dev/null \ diff --git a/test/docker-in-docker/docker_build.sh b/test/docker-in-docker/docker_build.sh index d51e7d644..7e71be8a1 100755 --- a/test/docker-in-docker/docker_build.sh +++ b/test/docker-in-docker/docker_build.sh @@ -11,6 +11,8 @@ check "docker-build" docker build ./ check "installs docker-compose v1 install" bash -c "type docker-compose" check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" # Report result reportResults diff --git a/test/docker-in-docker/docker_build_2.sh b/test/docker-in-docker/docker_build_2.sh index 742b222fb..d60cd937a 100644 --- a/test/docker-in-docker/docker_build_2.sh +++ b/test/docker-in-docker/docker_build_2.sh @@ -8,6 +8,8 @@ source dev-container-features-test-lib # Definition specific tests check "docker-buildx" docker buildx version check "docker-build" docker build ./ +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" # Report result reportResults diff --git a/test/docker-in-docker/docker_build_older.sh b/test/docker-in-docker/docker_build_older.sh index 742b222fb..d60cd937a 100644 --- a/test/docker-in-docker/docker_build_older.sh +++ b/test/docker-in-docker/docker_build_older.sh @@ -8,6 +8,8 @@ source dev-container-features-test-lib # Definition specific tests check "docker-buildx" docker buildx version check "docker-build" docker build ./ +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" # Report result reportResults diff --git a/test/docker-in-docker/docker_buildx.sh b/test/docker-in-docker/docker_buildx.sh new file mode 100755 index 000000000..c571596fc --- /dev/null +++ b/test/docker-in-docker/docker_buildx.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ + +check "installs docker-compose v1 install" bash -c "type docker-compose" +check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index 1274877b9..975ea7cd2 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -68,6 +68,15 @@ }, "remoteUser": "node" }, + "docker_buildx": { + "image": "mcr.microsoft.com/devcontainers/base:bookworm", + "features": { + "docker-in-docker": { + "moby": false, + "installDockerBuildx": true + } + } + }, // DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow "docker_with_on_create_command": { "image": "mcr.microsoft.com/devcontainers/base:debian", From 08fb370a59cc311d9f81a9e90be33cf2e6648baa Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 2 Feb 2024 22:56:55 +0530 Subject: [PATCH 026/285] [Python] - Fix for security vulnerability GHSA-r9hx-vwmv-q579 for setuptools-65.5.0 lib on python using python image (#815) * [Python] - Fix for security vulnerability GHSA-r9hx-vwmv-q579 for setuptools-65.5.0 lib on python using python image * [Python] - Added one more test * [Python] - Fix for security vulnerability GHSA-r9hx-vwmv-q579 for setuptools-65.5.0 lib on python using python image * [Python] - Added one more test * Changes as suggested by comments * removed second line (L465) that removed a non existent dirrectory * bump patch version & change code to pass failing tests --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 10 +++ ...tall_python310_setuptools_vulnerability.sh | 89 ++++++++++++++++++ ...tall_python311_setuptools_vulnerability.sh | 90 +++++++++++++++++++ test/python/scenarios.json | 18 ++++ 5 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 test/python/install_python310_setuptools_vulnerability.sh create mode 100644 test/python/install_python311_setuptools_vulnerability.sh diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 7ebdad38a..46f19b6ab 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.3.1", + "version": "1.3.2", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index e6eefd055..799f1ed8a 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -454,6 +454,16 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; echo "${util} already installed. Skipping." fi done + + # Temporary: Removes โ€œsetup toolsโ€ metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 + + VULNERABLE_VERSIONS=("3.10" "3.11") + RUN_TIME_PY_VER_DETECT=$(python --version 2>&1) + PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; + if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then + rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info + fi + rm -rf /tmp/pip-tmp updaterc "export PIPX_HOME=\"${PIPX_HOME}\"" diff --git a/test/python/install_python310_setuptools_vulnerability.sh b/test/python/install_python310_setuptools_vulnerability.sh new file mode 100644 index 000000000..d6f68f1fb --- /dev/null +++ b/test/python/install_python310_setuptools_vulnerability.sh @@ -0,0 +1,89 @@ +#!/bin/bash +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +FAILED=() + +echoStderr() +{ + echo "$@" 1>&2 +} + +check-version-ge() { + LABEL=$1 + CURRENT_VERSION=$2 + REQUIRED_VERSION=$3 + shift + echo -e "\n๐Ÿงช Testing $LABEL: '$CURRENT_VERSION' is >= '$REQUIRED_VERSION'" + local GREATER_VERSION=$((echo ${CURRENT_VERSION}; echo ${REQUIRED_VERSION}) | sort -V | tail -1) + if [ "${CURRENT_VERSION}" == "${GREATER_VERSION}" ]; then + echo "โœ… Passed!" + return 0 + else + echoStderr "โŒ $LABEL check failed." + FAILED+=("$LABEL") + return 1 + fi +} + +checkPythonPackageVersion() +{ + PACKAGE=$1 + REQUIRED_VERSION=$2 + + current_version=$(python -c "import importlib.metadata; print(importlib.metadata.version('${PACKAGE}'))") + check-version-ge "${PACKAGE}-requirement" "${current_version}" "${REQUIRED_VERSION}" +} + +checkPythonPackageVersion "setuptools" "65.5.1" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version +check "setuptools" pip list | grep setuptools + +# Check paths in settings +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +checkVulnerableDir() +{ + DIRECTORY=$1 + VERSION=$2 + + if [[ -d $DIRECTORY ]] ; then + echoStderr "โŒ check for vulnerable setuptools version failed for python ${VERSION}." + return 1 + else + echo "โœ… Passed! Either the container does not have vulnerable version or vulnerable version specific directory got removed." + return 0 + fi +} + +bash -c "echo -e -n '\n'"; +bash -c "echo -e 'Files/Folders related to setuptools :-'"; +bash -c "find / -name \"*setuptools*\";" + +# only for 3.10 +checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.10/site-packages/setuptools-65.5.0.dist-info" "3.10" + +# Report result +reportResults diff --git a/test/python/install_python311_setuptools_vulnerability.sh b/test/python/install_python311_setuptools_vulnerability.sh new file mode 100644 index 000000000..b1e3983d2 --- /dev/null +++ b/test/python/install_python311_setuptools_vulnerability.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +FAILED=() + +echoStderr() +{ + echo "$@" 1>&2 +} + +check-version-ge() { + LABEL=$1 + CURRENT_VERSION=$2 + REQUIRED_VERSION=$3 + shift + echo -e "\n๐Ÿงช Testing $LABEL: '$CURRENT_VERSION' is >= '$REQUIRED_VERSION'" + local GREATER_VERSION=$((echo ${CURRENT_VERSION}; echo ${REQUIRED_VERSION}) | sort -V | tail -1) + if [ "${CURRENT_VERSION}" == "${GREATER_VERSION}" ]; then + echo "โœ… Passed!" + return 0 + else + echoStderr "โŒ $LABEL check failed." + FAILED+=("$LABEL") + return 1 + fi +} + +checkPythonPackageVersion() +{ + PACKAGE=$1 + REQUIRED_VERSION=$2 + + current_version=$(python -c "import importlib.metadata; print(importlib.metadata.version('${PACKAGE}'))") + check-version-ge "${PACKAGE}-requirement" "${current_version}" "${REQUIRED_VERSION}" +} + +checkPythonPackageVersion "setuptools" "65.5.1" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version +check "setuptools" pip list | grep setuptools + +# Check paths in settings +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +checkVulnerableDir() +{ + DIRECTORY=$1 + VERSION=$2 + + if [[ -d $DIRECTORY ]] ; then + echoStderr "โŒ check for vulnerable setuptools version failed for python ${VERSION}." + return 1 + else + echo "โœ… Passed! Either the container does not have vulnerable version or vulnerable version specific directory got removed." + return 0 + fi +} + +bash -c "echo -e -n '\n'"; +bash -c "echo -e 'Files/Folders related to setuptools :-'"; +bash -c "find / -name \"*setuptools*\";" + +# only for 3.11 +checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.11/site-packages/setuptools-65.5.0.dist-info" "3.11" + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 23b6cfc92..493f99f1f 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -1,4 +1,22 @@ { + "install_python310_setuptools_vulnerability": { + "image": "python:3.10", + "features": { + "python": { + "version": "none", + "installTools": true + } + } + }, + "install_python311_setuptools_vulnerability": { + "image": "python:3.11", + "features": { + "python": { + "version": "none", + "installTools": true + } + } + }, "install_additional_python": { "image": "ubuntu:focal", "features": { From d5ef6a74a2f0f393f046a0af5444bd2851e5db37 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 8 Feb 2024 09:16:38 -0800 Subject: [PATCH 027/285] Docker-in-docker - [Bug fix] : Fix externally-managed-environment python error for "bookworm" (#833) * Docker-in-docker - [Bug fix] : Fix externally-managed-environment python error for "bookworm" * add comment --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 3 ++- .../docker_python_bookworm.sh | 22 +++++++++++++++++++ test/docker-in-docker/scenarios.json | 10 +++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100755 test/docker-in-docker/docker_python_bookworm.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 3e112ddcb..6b5c9840c 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.8.0", + "version": "2.8.1", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 4e042e3e8..8543b65dd 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -233,7 +233,8 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then if [ "${target_compose_arch}" = "amd64" ]; then target_compose_arch="x86_64" fi - if [ "${target_compose_arch}" != "x86_64" ]; then + # https://github.com/devcontainers/features/issues/832 + if [ "${target_compose_arch}" != "x86_64" ] && [ "${VERSION_CODENAME}" != "bookworm" ]; then # Use pip to get a version that runs on this architecture check_packages python3-minimal python3-pip libffi-dev python3-venv export PIPX_HOME=/usr/local/pipx diff --git a/test/docker-in-docker/docker_python_bookworm.sh b/test/docker-in-docker/docker_python_bookworm.sh new file mode 100755 index 000000000..b7080dee1 --- /dev/null +++ b/test/docker-in-docker/docker_python_bookworm.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ + +check "installs docker-compose v2 install" bash -c "type docker-compose" +check "docker compose" bash -c "docker compose version | grep -E '2.[0-9]+.[0-9]+'" +check "docker-compose" bash -c "docker-compose --version | grep -E '2.[0-9]+.[0-9]+'" + +check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index 975ea7cd2..6de695555 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -77,6 +77,16 @@ } } }, + "docker_python_bookworm": { + "image": "mcr.microsoft.com/devcontainers/base:bookworm", + "features": { + "docker-in-docker": { + "moby": true, + "installDockerBuildx": true, + "dockerDashComposeVersion": "v2" + } + } + }, // DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow "docker_with_on_create_command": { "image": "mcr.microsoft.com/devcontainers/base:debian", From 1592dd59781fc861a6ea3c6e39faa7cf9eecec7e Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 9 Feb 2024 05:32:03 +0530 Subject: [PATCH 028/285] [common-utils] - Bookworm - enabled nonFreePackages option for installing manpages-posix, manpages-posix-dev packages (#834) * [common-utils] - Bookworm - enabled nonFreePackages option for installing manpages-posix, manpages-posix-dev packages (non-free) * change to correct the regular expression replacement with sed cmd * changes according to comments by Samruddhi * Changes acc. to comments as on 08/02 * Updated patch version based on comments --- src/common-utils/devcontainer-feature.json | 2 +- src/common-utils/main.sh | 28 ++++++++------ test/common-utils/bookworm.sh | 45 ++++++++++++++++++++++ test/common-utils/scenarios.json | 9 +++++ 4 files changed, 71 insertions(+), 13 deletions(-) create mode 100755 test/common-utils/bookworm.sh diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index 44f9b32b4..a058864af 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.4.1", + "version": "2.4.2", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index 0eec1ecb9..8b74830cc 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -104,18 +104,22 @@ install_debian_packages() { # Needed for adding manpages-posix and manpages-posix-dev which are non-free packages in Debian if [ "${ADD_NON_FREE_PACKAGES}" = "true" ]; then - # Bring in variables from /etc/os-release like VERSION_CODENAME - sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list - sed -i -E "s/deb-src http:\/\/(deb|httredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list - sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list - sed -i -E "s/deb-src http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list - sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list - sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list - sed -i "s/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list - sed -i "s/deb-src http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list - # Handle bullseye location for security https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html - sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list - sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list + if [[ ! -e "/etc/apt/sources.list" ]] && [[ -e "/etc/apt/sources.list.d/debian.sources" ]]; then + sed -i '/^URIs: http:\/\/deb.debian.org\/debian$/ { N; N; s/Components: main/Components: main non-free non-free-firmware/ }' /etc/apt/sources.list.d/debian.sources + else + # Bring in variables from /etc/os-release like VERSION_CODENAME + sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb-src http:\/\/(deb|httredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb-src http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list + # Handle bullseye location for security https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html + sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list + fi; echo "Running apt-get update..." package_list="${package_list} manpages-posix manpages-posix-dev" fi diff --git a/test/common-utils/bookworm.sh b/test/common-utils/bookworm.sh new file mode 100755 index 000000000..db8628713 --- /dev/null +++ b/test/common-utils/bookworm.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +FAILED=() +echoStderr() +{ + echo "$@" 1>&2 +} + +checkOSPackages() { + LABEL=$1 + shift + echo -e "\n๐Ÿงช Testing $LABEL" + if dpkg-query --show -f='${Package}: ${Version}\n' "$@"; then + echo "โœ… Passed!" + return 0 + else + echoStderr "โŒ $LABEL check failed." + FAILED+=("$LABEL") + return 1 + fi +} + +checkCommon() +{ + PACKAGE_LIST="manpages-posix \ + manpages-posix-dev" + + checkOSPackages "Installation of manpages-posix and manpages-posix-dev (non-free)" ${PACKAGE_LIST} +} + +# Check for manpages-posix, manpages-posix-dev non-free packages +checkCommon + +# Definition specific tests +. /etc/os-release +check "non-root user" test "$(whoami)" = "devcontainer" +check "distro" test "${VERSION_CODENAME}" = "bookworm" + +# Report result +reportResults \ No newline at end of file diff --git a/test/common-utils/scenarios.json b/test/common-utils/scenarios.json index 2e357bc2b..744f16dc1 100644 --- a/test/common-utils/scenarios.json +++ b/test/common-utils/scenarios.json @@ -34,6 +34,15 @@ "common-utils": {} } }, + "bookworm": { + "image": "debian:bookworm", + "remoteUser": "devcontainer", + "features": { + "common-utils": { + "nonFreePackages": "true" + } + } + }, "centos-7": { "image": "centos:7", "remoteUser": "devcontainer", From 30e03b62e28bbf3abc6fbd80fb75f2b592c81ff3 Mon Sep 17 00:00:00 2001 From: Jeff Putsch <150963136+jdputschadi@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:59:19 -0800 Subject: [PATCH 029/285] Add RHEL support to the python feature (#830) * install gpg2.22 on centos 7 when installing python * RHEL support, exisiting tests pass, new RHEL tests pass. * add tests, cleanup install organization * update testing to include RHEL tests * update testing to include RHEL tests * undo addition of installing additional pip modules * fix errors installing os-provided Python on recent Debian systems and on Mariner systems * adjust to properly use newly installed python (PYTHON_SRC) instead of assuming "python" will work * When installing pipx, check if python is marked as externally managed. If so, add "--break-system-packages" to the pip install flags. This does not really breack system packages due to the setting of PYTHONUSERBASE during the install of pipx, but does get us past checks for installing python packages into the system python install. * merge from main * update check for managed python install. pass all tests. * add "packages" option from PR #768 * remove "packages" option * Address PR feedback, passes all tests locally. * fix install error on centos --------- Co-authored-by: Jeff Putsch --- .github/workflows/test-pr.yaml | 2 +- src/python/NOTES.md | 2 +- src/python/devcontainer-feature.json | 20 +- src/python/install.sh | 412 +++++++++++++++--- test/python/alma-8-minimal.sh | 40 ++ test/python/alma-8.sh | 40 ++ test/python/alma-9-minimal.sh | 40 ++ test/python/alma-9.sh | 40 ++ test/python/centos-7.sh | 40 ++ test/python/fedora.sh | 40 ++ ...stall_additional_jupyterlab_rhel_family.sh | 26 ++ .../install_additional_python_rhel_family.sh | 41 ++ test/python/install_alternate_tools.sh | 32 ++ .../install_alternate_tools_rhel_family.sh | 32 ++ test/python/install_jupyterlab_rhel_family.sh | 26 ++ ...n.sh => install_os_provided_python_deb.sh} | 0 .../install_os_provided_python_mariner.sh | 40 ++ .../install_os_provided_python_rhel_family.sh | 40 ++ ...tall_python310_setuptools_vulnerability.sh | 4 - ...tall_python311_setuptools_vulnerability.sh | 4 - test/python/install_python_shared_lib_deb.sh | 45 ++ .../install_python_shared_lib_rhel_family.sh | 45 ++ test/python/scenarios.json | 138 +++++- 23 files changed, 1073 insertions(+), 76 deletions(-) create mode 100755 test/python/alma-8-minimal.sh create mode 100755 test/python/alma-8.sh create mode 100755 test/python/alma-9-minimal.sh create mode 100755 test/python/alma-9.sh create mode 100755 test/python/centos-7.sh create mode 100755 test/python/fedora.sh create mode 100644 test/python/install_additional_jupyterlab_rhel_family.sh create mode 100644 test/python/install_additional_python_rhel_family.sh create mode 100755 test/python/install_alternate_tools.sh create mode 100644 test/python/install_alternate_tools_rhel_family.sh create mode 100644 test/python/install_jupyterlab_rhel_family.sh rename test/python/{install_os_provided_python.sh => install_os_provided_python_deb.sh} (100%) create mode 100755 test/python/install_os_provided_python_mariner.sh create mode 100644 test/python/install_os_provided_python_rhel_family.sh create mode 100755 test/python/install_python_shared_lib_deb.sh create mode 100755 test/python/install_python_shared_lib_rhel_family.sh diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index d60733574..3aae841e2 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -67,7 +67,7 @@ jobs: test-scenarios: needs: [detect-changes] - runs-on: ubuntu-latest + runs-on: devcontainer-image-builder-ubuntu continue-on-error: true strategy: matrix: diff --git a/src/python/NOTES.md b/src/python/NOTES.md index 19fe92f31..79a308cf5 100644 --- a/src/python/NOTES.md +++ b/src/python/NOTES.md @@ -2,6 +2,6 @@ ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the apt, yum, dnf, or microdnf package manager installed. `bash` is required to execute the `install.sh` script. diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 46f19b6ab..26f2e5a52 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.3.2", + "version": "1.4.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", @@ -25,13 +25,23 @@ "installTools": { "type": "boolean", "default": true, - "description": "Install common Python tools like pylint" + "description": "Flag indicating whether or not to install the tools specified via the 'toolsToInstall' option. Default is 'true'." + }, + "toolsToInstall": { + "type": "string", + "default": "flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest,pylint", + "description": "Comma-separated list of tools to install when 'installTools' is true. Defaults to a set of common Python tools like pylint." }, "optimize": { "type": "boolean", "default": false, "description": "Optimize Python for performance when compiled (slow)" }, + "enableShared": { + "type": "boolean", + "default": false, + "description": "Enable building a shared Python library" + }, "installPath": { "type": "string", "default": "/usr/local/python", @@ -71,7 +81,7 @@ } }, "installsAfter": [ - "ghcr.io/devcontainers/features/common-utils", - "ghcr.io/devcontainers/features/oryx" + "ghcr.io/devcontainers/features/common-utils", + "ghcr.io/devcontainers/features/oryx" ] -} +} \ No newline at end of file diff --git a/src/python/install.sh b/src/python/install.sh index 799f1ed8a..9094191f3 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -10,6 +10,7 @@ PYTHON_VERSION="${VERSION:-"latest"}" # 'system' or 'os-provided' checks the base image first, else installs 'latest' INSTALL_PYTHON_TOOLS="${INSTALLTOOLS:-"true"}" OPTIMIZE_BUILD_FROM_SOURCE="${OPTIMIZE:-"false"}" +ENABLE_SHARED_FROM_SOURCE="${ENABLESHARED:-"false"}" PYTHON_INSTALL_PATH="${INSTALLPATH:-"/usr/local/python"}" OVERRIDE_DEFAULT_VERSION="${OVERRIDEDEFAULTVERSION:-"true"}" @@ -26,7 +27,10 @@ CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN="${CONFIGUREJUPYTERLABALLOWORIGIN:-""}" # alongside PYTHON_VERSION, but not set as default. ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" -DEFAULT_UTILS=("pylint" "flake8" "autopep8" "black" "yapf" "mypy" "pydocstyle" "pycodestyle" "bandit" "pipenv" "virtualenv" "pytest") +# Comma-separated list of additional tools to be installed via pipx. +IFS="," read -r -a DEFAULT_UTILS <<< "${TOOLSTOINSTALL:-flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest}" + + PYTHON_SOURCE_GPG_KEYS="64E628F8D684696D B26995E310250568 2D347EA6AA65421D FB9921286F5E1540 3A5CA953F73C700D 04C367C218ADD4FF 0EDDC5F26A45C816 6AF053F07D9DC8D2 C9BE28DEE6DF025C 126EB563A74B06BF D9866941EA5BBD71 ED9D77D5 A821E680E5FA6305" GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com keyserver hkp://keyserver.ubuntu.com:80 @@ -37,44 +41,90 @@ KEYSERVER_PROXY="${HTTPPROXY:-"${HTTP_PROXY:-""}"}" set -e -# Clean up -rm -rf /var/lib/apt/lists/* - if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 fi -# Ensure that login shells get the correct path if the user updated the PATH using ENV. -rm -f /etc/profile.d/00-restore-env.sh -echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh -chmod +x /etc/profile.d/00-restore-env.sh +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +MAJOR_VERSION_ID=$(echo ${VERSION_ID} | cut -d . -f 1) +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + if [[ "${ID}" = "rhel" ]] || [[ "${ID}" = *"alma"* ]] || [[ "${ID}" = *"rocky"* ]]; then + VERSION_CODENAME="rhel${MAJOR_VERSION_ID}" + else + VERSION_CODENAME="${ID}${MAJOR_VERSION_ID}" + fi +else + echo "Linux distro ${ID} not supported." + exit 1 +fi -# Determine the appropriate non-root user -if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then - USERNAME="" - POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") - for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do - if id -u ${CURRENT_USER} > /dev/null 2>&1; then - USERNAME=${CURRENT_USER} - break - fi - done - if [ "${USERNAME}" = "" ]; then - USERNAME=root +# To find some devel packages, some rhel need to enable specific extra repos, but not on RedHat ubi images... +INSTALL_CMD_ADDL_REPO="" +if [ ${ADJUSTED_ID} = "rhel" ] && [ ${ID} != "rhel" ]; then + if [ ${MAJOR_VERSION_ID} = "8" ]; then + INSTALL_CMD_ADDL_REPOS="--enablerepo powertools" + elif [ ${MAJOR_VERSION_ID} = "9" ]; then + INSTALL_CMD_ADDL_REPOS="--enablerepo crb" fi -elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then - USERNAME=root fi +# Setup INSTALL_CMD & PKG_MGR_CMD +if type apt-get > /dev/null 2>&1; then + PKG_MGR_CMD=apt-get + INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" +elif type microdnf > /dev/null 2>&1; then + PKG_MGR_CMD=microdnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +else + PKG_MGR_CMD=yum + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --noplugins --setopt=install_weak_deps=0" +fi + +# Clean up +clean_up() { + case ${ADJUSTED_ID} in + debian) + rm -rf /var/lib/apt/lists/* + ;; + rhel) + rm -rf /var/cache/dnf/* /var/cache/yum/* + rm -rf /tmp/yum.log + rm -rf ${GPG_INSTALL_PATH} + ;; + esac +} +clean_up + + + updaterc() { + local _bashrc + local _zshrc if [ "${UPDATE_RC}" = "true" ]; then - echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." - if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/bash.bashrc + case $ADJUSTED_ID in + debian) echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." + _bashrc=/etc/bash.bashrc + _zshrc=/etc/zsh/zshrc + ;; + rhel) echo "Updating /etc/bashrc and /etc/zshrc..." + _bashrc=/etc/bashrc + _zshrc=/etc/zshrc + ;; + esac + if [[ "$(cat ${_bashrc})" != *"$1"* ]]; then + echo -e "$1" >> ${_bashrc} fi - if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/zsh/zshrc + if [ -f "${_zshrc}" ] && [[ "$(cat ${_zshrc})" != *"$1"* ]]; then + echo -e "$1" >> ${_zshrc} fi fi } @@ -83,6 +133,7 @@ updaterc() { receive_gpg_keys() { local keys=${!1} local keyring_args="" + local gpg_cmd="gpg" if [ ! -z "$2" ]; then mkdir -p "$(dirname \"$2\")" keyring_args="--no-default-keyring --keyring $2" @@ -116,6 +167,51 @@ receive_gpg_keys() { exit 1 fi } +# RHEL7/CentOS7 has an older gpg that does not have dirmngr +# Iterate through keyservers until we have all the keys downloaded +receive_gpg_keys_centos7() { + local keys=${!1} + local keyring_args="" + local gpg_cmd="gpg" + if [ ! -z "$2" ]; then + mkdir -p "$(dirname \"$2\")" + keyring_args="--no-default-keyring --keyring $2" + fi + if [ ! -z "${KEYSERVER_PROXY}" ]; then + keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + num_keys=$(echo ${keys} | wc -w) + set +e + echo "(*) Downloading GPG keys..." + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do + for keyserver in $(echo "${GPG_KEY_SERVERS}" | sed 's/keyserver //'); do + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver=${keyserver} ) 2>&1 + downloaded_keys=$(gpg --list-keys | grep ^pub | wc -l) + if [[ ${num_keys} = ${downloaded_keys} ]]; then + gpg_ok="true" + break + fi + done + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} # Figure out correct version of a three part version number is not passed find_version_from_git_tags() { @@ -189,20 +285,52 @@ oryx_install() { fi } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - apt-get update -y - fi +pkg_mgr_update() { + case $ADJUSTED_ID in + debian) + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${PKG_MGR_CMD} update -y + fi + ;; + rhel) + if [ ${PKG_MGR_CMD} = "microdnf" ]; then + if [ "$(ls /var/cache/yum/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} makecache ..." + ${PKG_MGR_CMD} makecache + fi + else + if [ "$(ls /var/cache/${PKG_MGR_CMD}/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} check-update ..." + set +e + ${PKG_MGR_CMD} check-update + rc=$? + if [ $rc != 0 ] && [ $rc != 100 ]; then + exit 1 + fi + set -e + fi + fi + ;; + esac } # Checks if packages are installed and installs them if not check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - apt_get_update - apt-get -y install --no-install-recommends "$@" - fi + case ${ADJUSTED_ID} in + debian) + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + rhel) + if ! rpm -q "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + esac } add_symlink() { @@ -218,13 +346,30 @@ add_symlink() { fi } +install_openssl3() { + local _prefix=$1 + mkdir /tmp/openssl3 + ( + cd /tmp/openssl3 + openssl3_version="3.0" + # Find version using soft match + find_version_from_git_tags openssl3_version "https://github.com/openssl/openssl" "openssl-" + local tgz_filename="openssl-${openssl3_version}.tar.gz" + local tgz_url="https://github.com/openssl/openssl/releases/download/openssl-${openssl3_version}/${tgz_filename}" + echo "Downloading ${tgz_filename}..." + curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" + tar xzf ${tgz_filename} + cd openssl-${openssl3_version} + ./config --prefix=${_prefix} --openssldir=${_prefix} --libdir=lib + make -j $(nproc) + make install_dev + ) + rm -rf /tmp/openssl3 +} + install_from_source() { - VERSION=$1 + VERSION=$1 echo "(*) Building Python ${VERSION} from source..." - # Install prereqs if missing - check_packages curl ca-certificates gnupg2 tar make gcc libssl-dev zlib1g-dev libncurses5-dev \ - libbz2-dev libreadline-dev libxml2-dev xz-utils libgdbm-dev tk-dev dirmngr \ - libxmlsec1-dev libsqlite3-dev libffi-dev liblzma-dev uuid-dev if ! type git > /dev/null 2>&1; then check_packages git fi @@ -239,6 +384,17 @@ install_from_source() { exit 1 fi + # Some platforms/os versions need modern versions of openssl installed + # via common package repositories, for now rhel-7 family, use case statement to + # make it easy to expand + case ${VERSION_CODENAME} in + centos7|rhel7) + check_packages perl-IPC-Cmd + install_openssl3 ${INSTALL_PATH} + ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH} --with-openssl-rpath=${INSTALL_PATH}/lib" + ;; + esac + # Download tgz of source mkdir -p /tmp/python-src ${INSTALL_PATH} cd /tmp/python-src @@ -248,13 +404,21 @@ install_from_source() { curl -sSL -o "/tmp/python-src/${tgz_filename}" "${tgz_url}" # Verify signature - receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + if [[ ${VERSION_CODENAME} = "centos7" ]] || [[ ${VERSION_CODENAME} = "rhel7" ]]; then + receive_gpg_keys_centos7 PYTHON_SOURCE_GPG_KEYS + else + receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + fi echo "Downloading ${tgz_filename}.asc..." curl -sSL -o "/tmp/python-src/${tgz_filename}.asc" "${tgz_url}.asc" gpg --verify "${tgz_filename}.asc" # Update min protocol for testing only - https://bugs.python.org/issue41561 - cp /etc/ssl/openssl.cnf /tmp/python-src/ + if [ -f /etc/pki/tls/openssl.cnf ]; then + cp /etc/pki/tls/openssl.cnf /tmp/python-src/ + else + cp /etc/ssl/openssl.cnf /tmp/python-src/ + fi sed -i -E 's/MinProtocol[=\ ]+.*/MinProtocol = TLSv1.0/g' /tmp/python-src/openssl.cnf export OPENSSL_CONF=/tmp/python-src/openssl.cnf @@ -262,11 +426,21 @@ install_from_source() { tar -xzf "/tmp/python-src/${tgz_filename}" -C "/tmp/python-src" --strip-components=1 local config_args="" if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then - config_args="--enable-optimizations" + config_args="${config_args} --enable-optimizations" + fi + if [ "${ENABLESHARED}" = "true" ]; then + config_args=" ${config_args} --enable-shared" + # need double-$: LDFLAGS ends up in Makefile $$ becomes $ when evaluated. + # backslash needed for shell that Make calls escape the $. + export LDFLAGS="${LDFLAGS} -Wl,-rpath="'\$$ORIGIN'"/../lib" + fi + if [ -n "${ADDL_CONFIG_ARGS}" ]; then + config_args="${config_args} ${ADDL_CONFIG_ARGS}" fi ./configure --prefix="${INSTALL_PATH}" --with-ensurepip=install ${config_args} make -j 8 make install + cd /tmp rm -rf /tmp/python-src ${GNUPGHOME} /tmp/vscdc-settings.env @@ -336,7 +510,15 @@ install_python() { version=$1 # If the os-provided versions are "good enough", detect that and bail out. if [ ${version} = "os-provided" ] || [ ${version} = "system" ]; then - check_packages python3 python3-doc python3-pip python3-venv python3-dev python3-tk + if [ ${ADJUSTED_ID} = "debian" ]; then + check_packages python3 python3-doc python3-pip python3-venv python3-dev python3-tk + else + if [ ${ID} != "mariner" ]; then + check_packages python3 python3-pip python3-devel python3-tkinter + else + check_packages python3 python3-pip python3-devel + fi + fi INSTALL_PATH="/usr" local current_bin_path="${CURRENT_PATH}/bin" @@ -355,7 +537,7 @@ install_python() { fi should_install_from_source=false - elif [ "$(dpkg --print-architecture)" = "amd64" ] && [ "${USE_ORYX_IF_AVAILABLE}" = "true" ] && type oryx > /dev/null 2>&1; then + elif [ ${ADJUSTED_ID} = "debian" ] && [ "$(dpkg --print-architecture)" = "amd64" ] && [ "${USE_ORYX_IF_AVAILABLE}" = "true" ] && type oryx > /dev/null 2>&1; then install_using_oryx $version || should_install_from_source=true else should_install_from_source=true @@ -365,14 +547,120 @@ install_python() { fi } +python_is_externally_managed() { + local _python_cmd=$1 + local python_stdlib_dir=$( + ${_python_cmd} -c ' +import sys +import sysconfig +sys.prefix == sys.base_prefix and print(sysconfig.get_path("stdlib", sysconfig.get_default_scheme()))' + ) + if [ -f ${python_stdlib_dir}/EXTERNALLY-MANAGED ]; then + return 0 + else + return 1 + fi +} + +# Ensure that login shells get the correct path if the user updated the PATH using ENV. +rm -f /etc/profile.d/00-restore-env.sh +echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh +chmod +x /etc/profile.d/00-restore-env.sh + +# Some distributions do not install awk by default (e.g. Mariner) +if ! type awk >/dev/null 2>&1; then + check_packages awk +fi + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + # Ensure apt is in non-interactive to avoid prompts export DEBIAN_FRONTEND=noninteractive # General requirements -check_packages curl ca-certificates gnupg2 tar make gcc libssl-dev zlib1g-dev libncurses5-dev \ - libbz2-dev libreadline-dev libxml2-dev xz-utils libgdbm-dev tk-dev dirmngr \ - libxmlsec1-dev libsqlite3-dev libffi-dev liblzma-dev uuid-dev +REQUIRED_PKGS="" +case ${ADJUSTED_ID} in + debian) + REQUIRED_PKGS="${REQUIRED_PKGS} \ + ca-certificates \ + curl \ + dirmngr \ + gcc \ + gnupg2 \ + libbz2-dev \ + libffi-dev \ + libgdbm-dev \ + liblzma-dev \ + libncurses5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libxml2-dev \ + libxmlsec1-dev \ + make \ + tar \ + tk-dev \ + uuid-dev \ + xz-utils \ + zlib1g-dev" + ;; + rhel) + REQUIRED_PKGS="${REQUIRED_PKGS} \ + bzip2-devel \ + ca-certificates \ + findutils \ + gcc \ + gnupg2 \ + libffi-devel \ + libxml2-devel \ + make \ + ncurses-devel \ + openssl-devel \ + shadow-utils \ + sqlite-devel \ + tar \ + which \ + xz-devel \ + xz \ + zlib-devel" + if ! type curl >/dev/null 2>&1; then + REQUIRED_PKGS="${REQUIRED_PKGS} \ + curl" + fi + # Mariner does not have tk-devel package available, RedHat ubi8 and ubi9 do not have tk-devel + if [ ${ID} != "mariner" ] && [ ${ID} != "rhel" ]; then + REQUIRED_PKGS="${REQUIRED_PKGS} \ + tk-devel" + fi + # Redhat ubi8 and ubi9 do not have some packages by default, only add them + # if we're not on RedHat ... + if [ ${ID} != "rhel" ]; then + REQUIRED_PKGS="${REQUIRED_PKGS} \ + gdbm-devel \ + readline-devel \ + uuid-devel \ + xmlsec1-devel" + fi + ;; +esac + +check_packages ${REQUIRED_PKGS} # Install Python from source if needed if [ "${PYTHON_VERSION}" != "none" ]; then @@ -410,12 +698,15 @@ if [ "${PYTHON_VERSION}" != "none" ]; then find "${PYTHON_INSTALL_PATH}" -type d -print0 | xargs -0 -n 1 chmod g+s PYTHON_SRC="${INSTALL_PATH}/bin/python3" + if ! type pip >/dev/null 2>&1 && type pip3 >/dev/null 2>&1; then + ln -s /usr/bin/pip3 /usr/bin/pip + fi else PYTHON_SRC=$(which python) fi # Install Python tools if needed -if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; then +if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then echo 'Installing Python tools...' export PIPX_BIN_DIR="${PIPX_HOME}/bin" PATH="${PATH}:${PIPX_BIN_DIR}" @@ -432,20 +723,23 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; find "${PIPX_HOME}" -type d -print0 | xargs -0 -n 1 chmod g+s # Update pip if not using os provided python - if [[ $(python --version) != "" ]] || [[ ${PYTHON_VERSION} != "os-provided" ]] && [[ ${PYTHON_VERSION} != "system" ]] && [[ ${PYTHON_VERSION} != "none" ]]; then + if [[ -n "${PYTHON_SRC}" ]] && [[ ${PYTHON_VERSION} != "os-provided" ]] && [[ ${PYTHON_VERSION} != "system" ]] && [[ ${PYTHON_VERSION} != "none" ]]; then echo "Updating pip..." - python -m pip install --no-cache-dir --upgrade pip + ${PYTHON_SRC} -m pip install --no-cache-dir --upgrade pip fi # Install tools - echo "Installing Python tools..." export PYTHONUSERBASE=/tmp/pip-tmp export PIP_CACHE_DIR=/tmp/pip-tmp/cache PIPX_DIR="" if ! type pipx > /dev/null 2>&1; then - pip3 install --disable-pip-version-check --no-cache-dir --user pipx 2>&1 - /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx - PIPX_DIR="/tmp/pip-tmp/bin/" + if python_is_externally_managed ${PYTHON_SRC}; then + check_packages pipx + else + pip3 install --disable-pip-version-check --no-cache-dir --user pipx 2>&1 + /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx + PIPX_DIR="/tmp/pip-tmp/bin/" + fi fi for util in "${DEFAULT_UTILS[@]}"; do if ! type ${util} > /dev/null 2>&1; then @@ -458,7 +752,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; # Temporary: Removes โ€œsetup toolsโ€ metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 VULNERABLE_VERSIONS=("3.10" "3.11") - RUN_TIME_PY_VER_DETECT=$(python --version 2>&1) + RUN_TIME_PY_VER_DETECT=$(${PYTHON_SRC} --version 2>&1) PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info @@ -502,6 +796,6 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then fi # Clean up -rm -rf /var/lib/apt/lists/* +clean_up echo "Done!" diff --git a/test/python/alma-8-minimal.sh b/test/python/alma-8-minimal.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-8-minimal.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/alma-8.sh b/test/python/alma-8.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-8.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/alma-9-minimal.sh b/test/python/alma-9-minimal.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-9-minimal.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/alma-9.sh b/test/python/alma-9.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-9.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/centos-7.sh b/test/python/centos-7.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/centos-7.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/fedora.sh b/test/python/fedora.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/fedora.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_additional_jupyterlab_rhel_family.sh b/test/python/install_additional_jupyterlab_rhel_family.sh new file mode 100644 index 000000000..58c4f7f54 --- /dev/null +++ b/test/python/install_additional_jupyterlab_rhel_family.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Check for correct JupyterLab configuration +check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py + +# Report result +reportResults diff --git a/test/python/install_additional_python_rhel_family.sh b/test/python/install_additional_python_rhel_family.sh new file mode 100644 index 000000000..d584f4be9 --- /dev/null +++ b/test/python/install_additional_python_rhel_family.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "python version 3.11 installed as default" bash -c "python --version | grep 3.11" +check "python3 version 3.11 installed as default" bash -c "python3 --version | grep 3.11" +check "python version 3.10.5 installed" bash -c "ls -l /usr/local/python | grep 3.10.5" +check "python version 3.8 installed" bash -c "ls -l /usr/local/python | grep 3.8" +check "python version 3.9.13 installed" bash -c "ls -l /usr/local/python | grep 3.9.13" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_alternate_tools.sh b/test/python/install_alternate_tools.sh new file mode 100755 index 000000000..b98d70997 --- /dev/null +++ b/test/python/install_alternate_tools.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "bandit" bandit --version +check "mypy" mypy --version +check "pipenv" pipenv --version +check "pytest" pytest --version +check "ruff" ruff --version +check "virtualenv" virtualenv --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pipenv" bash -c "which pipenv | grep /usr/local/py-utils/bin/pipenv" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" +check "which ruff" bash -c "which ruff | grep /usr/local/py-utils/bin/ruff" +check "which virtualenv" bash -c "which virtualenv | grep /usr/local/py-utils/bin/virtualenv" + +# Report result +reportResults diff --git a/test/python/install_alternate_tools_rhel_family.sh b/test/python/install_alternate_tools_rhel_family.sh new file mode 100644 index 000000000..b98d70997 --- /dev/null +++ b/test/python/install_alternate_tools_rhel_family.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "bandit" bandit --version +check "mypy" mypy --version +check "pipenv" pipenv --version +check "pytest" pytest --version +check "ruff" ruff --version +check "virtualenv" virtualenv --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pipenv" bash -c "which pipenv | grep /usr/local/py-utils/bin/pipenv" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" +check "which ruff" bash -c "which ruff | grep /usr/local/py-utils/bin/ruff" +check "which virtualenv" bash -c "which virtualenv | grep /usr/local/py-utils/bin/virtualenv" + +# Report result +reportResults diff --git a/test/python/install_jupyterlab_rhel_family.sh b/test/python/install_jupyterlab_rhel_family.sh new file mode 100644 index 000000000..58c4f7f54 --- /dev/null +++ b/test/python/install_jupyterlab_rhel_family.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Check for correct JupyterLab configuration +check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py + +# Report result +reportResults diff --git a/test/python/install_os_provided_python.sh b/test/python/install_os_provided_python_deb.sh similarity index 100% rename from test/python/install_os_provided_python.sh rename to test/python/install_os_provided_python_deb.sh diff --git a/test/python/install_os_provided_python_mariner.sh b/test/python/install_os_provided_python_mariner.sh new file mode 100755 index 000000000..e39b51412 --- /dev/null +++ b/test/python/install_os_provided_python_mariner.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "python3 is installed" python3 --version +check "python is installed" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_os_provided_python_rhel_family.sh b/test/python/install_os_provided_python_rhel_family.sh new file mode 100644 index 000000000..e39b51412 --- /dev/null +++ b/test/python/install_os_provided_python_rhel_family.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "python3 is installed" python3 --version +check "python is installed" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_python310_setuptools_vulnerability.sh b/test/python/install_python310_setuptools_vulnerability.sh index d6f68f1fb..0e065b1b3 100644 --- a/test/python/install_python310_setuptools_vulnerability.sh +++ b/test/python/install_python310_setuptools_vulnerability.sh @@ -78,10 +78,6 @@ checkVulnerableDir() fi } -bash -c "echo -e -n '\n'"; -bash -c "echo -e 'Files/Folders related to setuptools :-'"; -bash -c "find / -name \"*setuptools*\";" - # only for 3.10 checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.10/site-packages/setuptools-65.5.0.dist-info" "3.10" diff --git a/test/python/install_python311_setuptools_vulnerability.sh b/test/python/install_python311_setuptools_vulnerability.sh index b1e3983d2..04a10948b 100644 --- a/test/python/install_python311_setuptools_vulnerability.sh +++ b/test/python/install_python311_setuptools_vulnerability.sh @@ -79,10 +79,6 @@ checkVulnerableDir() fi } -bash -c "echo -e -n '\n'"; -bash -c "echo -e 'Files/Folders related to setuptools :-'"; -bash -c "find / -name \"*setuptools*\";" - # only for 3.11 checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.11/site-packages/setuptools-65.5.0.dist-info" "3.11" diff --git a/test/python/install_python_shared_lib_deb.sh b/test/python/install_python_shared_lib_deb.sh new file mode 100755 index 000000000..ee16c436a --- /dev/null +++ b/test/python/install_python_shared_lib_deb.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Check compiled with "--enable-shared" -- python truth is opposite of the shells +# therefore we negate the check to get a '0' exit code when the python +# has been compiled with --enable-shared +check "python enable shared" /usr/local/python/current/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' + +# Report result +reportResults diff --git a/test/python/install_python_shared_lib_rhel_family.sh b/test/python/install_python_shared_lib_rhel_family.sh new file mode 100755 index 000000000..ee16c436a --- /dev/null +++ b/test/python/install_python_shared_lib_rhel_family.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Check compiled with "--enable-shared" -- python truth is opposite of the shells +# therefore we negate the check to get a '0' exit code when the python +# has been compiled with --enable-shared +check "python enable shared" /usr/local/python/current/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 493f99f1f..d574c84e8 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -26,6 +26,15 @@ } } }, + "install_additional_python_rhel_family": { + "image": "almalinux:8", + "features": { + "python": { + "version": "3.11", + "additionalVersions": "3.8,3.9.13,3.10.5" + } + } + }, "install_jupyterlab": { "image": "mcr.microsoft.com/devcontainers/base:focal", "remoteUser": "vscode", @@ -37,6 +46,20 @@ } } }, + "install_jupyterlab_rhel_family": { + "image": "almalinux:8", + "remoteUser": "vscode", + "features": { + "common-utils": { + "username": "vscode" + }, + "python": { + "version": "latest", + "installJupyterlab": true, + "configureJupyterlabAllowOrigin": "*" + } + } + }, "install_additional_jupyterlab": { "image": "mcr.microsoft.com/devcontainers/base:focal", "remoteUser": "vscode", @@ -49,12 +72,39 @@ } } }, - "install_os_provided_python": { + "install_additional_jupyterlab_rhel_family": { + "image": "almalinux:8", + "remoteUser": "vscode", + "features": { + "common-utils": { + "username": "vscode" + }, + "python": { + "version": "latest", + "additionalVersions": "3.9", + "installJupyterlab": true, + "configureJupyterlabAllowOrigin": "*" + } + } + }, + "install_os_provided_python_deb": { "image": "mcr.microsoft.com/devcontainers/base:1-bullseye", "features": { "python": "os-provided" } }, + "install_os_provided_python_rhel_family": { + "image": "almalinux:8", + "features": { + "python": "os-provided" + } + }, + "install_os_provided_python_mariner": { + "image": "mcr.microsoft.com/cbl-mariner/base/core:2.0", + "features": { + "python": "os-provided" + } + }, "install_jupyterlab_from_python_image": { "image": "mcr.microsoft.com/devcontainers/python:3.10-bullseye", "remoteUser": "vscode", @@ -99,5 +149,89 @@ "version": "3.12" } } + }, + "install_alternate_tools": { + "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "features": { + "python": { + "version": "3.12", + "toolsToInstall": "ruff,mypy,bandit,pipenv,virtualenv,pytest" + } + } + }, + "install_alternate_tools_rhel_family": { + "image": "almalinux:8", + "features": { + "python": { + "version": "3.12", + "toolsToInstall": "ruff,mypy,bandit,pipenv,virtualenv,pytest" + } + } + }, + "centos-7": { + "image": "centos:centos7", + "features": { + "python": { + "version": "3.11" + } + } + }, + "alma-8": { + "image": "almalinux:8", + "features": { + "python": { + "version": "3.11" + } + } + }, + "alma-8-minimal": { + "image": "almalinux:8-minimal", + "features": { + "python": { + "version": "3.11" + } + } + }, + "alma-9": { + "image": "almalinux:9", + "features": { + "python": { + "version": "3.11" + } + } + }, + "alma-9-minimal": { + "image": "almalinux:9-minimal", + "features": { + "python": { + "version": "3.11" + } + } + }, + "fedora": { + "image": "fedora", + "features": { + "python": { + "version": "3.11" + } + } + }, + "install_python_shared_lib_deb": { + "image": "ubuntu:focal", + "features": { + "python": { + "version": "3.11", + "enableShared": true + } + } + }, + "install_python_shared_lib_rhel_family": { + "image": "almalinux:8", + "features": { + "python": { + "version": "3.11", + "enableShared": true + } + } } -} +} \ No newline at end of file From 42925ae702eca8c6b85c47ebb250cd92912e71a3 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Fri, 9 Feb 2024 12:23:29 -0800 Subject: [PATCH 030/285] Fix broken workflows ; adds permissions (#835) --- .github/workflows/release.yaml | 2 ++ .github/workflows/update-documentation.yml | 3 +++ .github/workflows/update-dotnet-install-script.yml | 3 +++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 485096b5f..a1ec773b9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,6 +9,8 @@ jobs: deploy: if: ${{ github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest + permissions: + packages: write steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/update-documentation.yml b/.github/workflows/update-documentation.yml index 6976e62ea..2452e0d71 100644 --- a/.github/workflows/update-documentation.yml +++ b/.github/workflows/update-documentation.yml @@ -9,6 +9,9 @@ jobs: generate: runs-on: ubuntu-latest environment: documentation + permissions: + contents: write + pull-requests: write if: "github.ref == 'refs/heads/main'" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/update-dotnet-install-script.yml b/.github/workflows/update-dotnet-install-script.yml index 72d06099b..ebd175cff 100644 --- a/.github/workflows/update-dotnet-install-script.yml +++ b/.github/workflows/update-dotnet-install-script.yml @@ -8,6 +8,9 @@ jobs: fetch-latest-dotnet-install: runs-on: ubuntu-latest environment: documentation # grants access to secrets.PAT, for creating pull requests + permissions: + contents: write + pull-requests: write steps: - uses: actions/checkout@v3 From aa982d2d6cf866cc0c3008e97d46a48b66342931 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:16:20 -0800 Subject: [PATCH 031/285] Automated documentation update (#836) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/python/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python/README.md b/src/python/README.md index 2dd092ae7..db6ad1039 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -16,8 +16,10 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select a Python version to install. | string | os-provided | -| installTools | Install common Python tools like pylint | boolean | true | +| installTools | Flag indicating whether or not to install the tools specified via the 'toolsToInstall' option. Default is 'true'. | boolean | true | +| toolsToInstall | Comma-separated list of tools to install when 'installTools' is true. Defaults to a set of common Python tools like pylint. | string | flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest,pylint | | optimize | Optimize Python for performance when compiled (slow) | boolean | false | +| enableShared | Enable building a shared Python library | boolean | false | | installPath | The path where python will be installed. | string | /usr/local/python | | installJupyterlab | Install JupyterLab, a web-based interactive development environment for notebooks | boolean | false | | configureJupyterlabAllowOrigin | Configure JupyterLab to accept HTTP requests from the specified origin | string | - | @@ -34,7 +36,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the apt, yum, dnf, or microdnf package manager installed. `bash` is required to execute the `install.sh` script. From 91ac77405725c0d6fcbc7a4cc15b7c94078540c1 Mon Sep 17 00:00:00 2001 From: Ben Menesini Date: Fri, 9 Feb 2024 16:43:13 -0800 Subject: [PATCH 032/285] Allow moby-buildx version to be specified in devcontainer.json (#838) * Allow moby-buildx version to be specified in devcontainer.json Fixes #837 * Revert changes to README.md * Address code review feedback - Specify mobyBuildxVersion in devcontainer-feature.json with pinned and commented default value - Bump minor version - Leverage default value in install.sh - Only set buildx_version_suffix when installing moby * Handle varying distributions * Fix typo / remove extra line --- .../devcontainer-feature.json | 7 +++- src/docker-in-docker/install.sh | 34 +++++++++++++++---- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 6b5c9840c..744bc1d81 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.8.1", + "version": "2.9.0", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", @@ -20,6 +20,11 @@ "default": true, "description": "Install OSS Moby build instead of Docker CE" }, + "mobyBuildxVersion": { + "type": "string", + "default": "0.12.0", + "description": "Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version)" + }, "dockerDashComposeVersion": { "type": "string", "enum": [ diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 8543b65dd..db3cefc6a 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -10,6 +10,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version USE_MOBY="${MOBY:-"true"}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL}" @@ -81,7 +82,7 @@ find_version_from_git_tags() { local repository=$2 local prefix=${3:-"tags/v"} local separator=${4:-"."} - local last_part_optional=${5:-"false"} + local last_part_optional=${5:-"false"} if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator=${separator//./\\.} local last_part @@ -198,6 +199,27 @@ else echo "cli_version_suffix ${cli_version_suffix}" fi +# Version matching for moby-buildx +if [ "${USE_MOBY}" = "true" ]; then + if [ "${MOBY_BUILDX_VERSION}" = "latest" ]; then + # Empty, meaning grab whatever "latest" is in apt repo + buildx_version_suffix="" + else + buildx_version_dot_escaped="${MOBY_BUILDX_VERSION//./\\.}" + buildx_version_dot_plus_escaped="${buildx_version_dot_escaped//+/\\+}" + buildx_version_regex="^(.+:)?${buildx_version_dot_plus_escaped}([\\.\\+ ~:-]|$)" + set +e + buildx_version_suffix="=$(apt-cache madison moby-buildx | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${buildx_version_regex}")" + set -e + if [ -z "${buildx_version_suffix}" ] || [ "${buildx_version_suffix}" = "=" ]; then + err "No full or partial moby-buildx version match found for \"${MOBY_BUILDX_VERSION}\" on OS ${ID} ${VERSION_CODENAME} (${architecture}). Available versions:" + apt-cache madison moby-buildx | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+' + exit 1 + fi + echo "buildx_version_suffix ${buildx_version_suffix}" + fi +fi + # Install Docker / Moby CLI if not already installed if type docker > /dev/null 2>&1 && type dockerd > /dev/null 2>&1; then echo "Docker / Moby CLI and Engine already installed." @@ -205,7 +227,7 @@ else if [ "${USE_MOBY}" = "true" ]; then # Install engine set +e # Handle error gracefully - apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx moby-engine${engine_version_suffix} + apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx${buildx_version_suffix} moby-engine${engine_version_suffix} if [ $? -ne 0 ]; then err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')." exit 1 @@ -399,7 +421,7 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU retry_cgroup_nesting=`expr $retry_cgroup_nesting + 1` set -e - done + done # -- End: dind wrapper script -- @@ -442,7 +464,7 @@ retry_docker_start_count=0 docker_ok="false" until [ "${docker_ok}" = "true" ] || [ "${retry_docker_start_count}" -eq "5" ]; -do +do # Start using sudo if not invoked as root if [ "$(id -u)" -ne 0 ]; then sudo /bin/sh -c "${dockerd_start}" @@ -460,7 +482,7 @@ do retry_count=`expr $retry_count + 1` done - + if [ "${docker_ok}" != "true" ] && [ "${retry_docker_start_count}" != "4" ]; then echo "(*) Failed to start docker, retrying..." set +e @@ -468,7 +490,7 @@ do sudo_if pkill containerd set -e fi - + retry_docker_start_count=`expr $retry_docker_start_count + 1` done From fc62e9abf47c5ea52e02de997c91c5d52a5edc3a Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Fri, 9 Feb 2024 16:47:43 -0800 Subject: [PATCH 033/285] Automated documentation update (#840) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/docker-in-docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/docker-in-docker/README.md b/src/docker-in-docker/README.md index bfca77357..41004b7e1 100644 --- a/src/docker-in-docker/README.md +++ b/src/docker-in-docker/README.md @@ -17,6 +17,7 @@ Create child containers *inside* a container, independent from the host's docker |-----|-----|-----|-----| | version | Select or enter a Docker/Moby Engine version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | +| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version) | string | 0.12.0 | | dockerDashComposeVersion | Default version of Docker Compose (v1 or v2 or none) | string | v1 | | azureDnsAutoDetection | Allow automatically setting the dockerd DNS server when the installation script detects it is running in Azure | boolean | true | | dockerDefaultAddressPool | Define default address pools for Docker networks. e.g. base=192.168.0.0/16,size=24 | string | - | From e05b0b85ae9ff5fac24b021177e353721d265e15 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 12 Feb 2024 12:40:31 -0800 Subject: [PATCH 034/285] Docker-outside-of-docker: Fix moby-buildx errors (#843) * Docker-outside-of-docker: Fix moby-buildx errors * test debugging * fix test * nit --- .../devcontainer-feature.json | 7 +++++- src/docker-outside-of-docker/install.sh | 24 ++++++++++++++++++- test/docker-outside-of-docker/docker_init.sh | 3 ++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 4e5eae721..78ad1162c 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.3.2", + "version": "1.4.0", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", @@ -20,6 +20,11 @@ "default": true, "description": "Install OSS Moby build instead of Docker CE" }, + "mobyBuildxVersion": { + "type": "string", + "default": "0.12.0", + "description": "Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version)" + }, "dockerDashComposeVersion": { "type": "string", "enum": [ diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 901dd7870..988b7e43c 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -9,6 +9,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" @@ -170,6 +171,27 @@ else echo "cli_version_suffix ${cli_version_suffix}" fi +# Version matching for moby-buildx +if [ "${USE_MOBY}" = "true" ]; then + if [ "${MOBY_BUILDX_VERSION}" = "latest" ]; then + # Empty, meaning grab whatever "latest" is in apt repo + buildx_version_suffix="" + else + buildx_version_dot_escaped="${MOBY_BUILDX_VERSION//./\\.}" + buildx_version_dot_plus_escaped="${buildx_version_dot_escaped//+/\\+}" + buildx_version_regex="^(.+:)?${buildx_version_dot_plus_escaped}([\\.\\+ ~:-]|$)" + set +e + buildx_version_suffix="=$(apt-cache madison moby-buildx | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${buildx_version_regex}")" + set -e + if [ -z "${buildx_version_suffix}" ] || [ "${buildx_version_suffix}" = "=" ]; then + err "No full or partial moby-buildx version match found for \"${MOBY_BUILDX_VERSION}\" on OS ${ID} ${VERSION_CODENAME} (${architecture}). Available versions:" + apt-cache madison moby-buildx | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+' + exit 1 + fi + echo "buildx_version_suffix ${buildx_version_suffix}" + fi +fi + # Install Docker / Moby CLI if not already installed if type docker > /dev/null 2>&1; then echo "Docker / Moby CLI already installed." @@ -177,7 +199,7 @@ else if [ "${USE_MOBY}" = "true" ]; then buildx=() if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then - buildx=(moby-buildx) + buildx=(moby-buildx${buildx_version_suffix}) fi apt-get -y install --no-install-recommends ${cli_package_name}${cli_version_suffix} "${buildx[@]}" apt-get -y install --no-install-recommends moby-compose || echo "(*) Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping." diff --git a/test/docker-outside-of-docker/docker_init.sh b/test/docker-outside-of-docker/docker_init.sh index b47285ead..88153c1e0 100644 --- a/test/docker-outside-of-docker/docker_init.sh +++ b/test/docker-outside-of-docker/docker_init.sh @@ -12,7 +12,8 @@ check "docker-compose" bash -c "docker-compose --version" check "docker-init-exists" bash -c "ls /usr/local/share/docker-init.sh" check "log-exists" bash -c "ls /tmp/vscr-docker-from-docker.log" check "log-contents-for-success" bash -c "cat /tmp/vscr-docker-from-docker.log | grep 'Success'" -check "log-contents" bash -c "cat /tmp/vscr-docker-from-docker.log | grep 'Proxying /var/run/docker-host.sock to /var/run/docker.sock for vscode'" + +check "log-contents" bash -c "cat /tmp/vscr-docker-from-docker.log | grep 'Ensuring vscode has access to /var/run/docker-host.sock via /var/run/docker.sock'" check "docker-ps" bash -c "docker ps >/dev/null" # Report result From eea8ec3a3702057ce3c0c3e54bbb205ed580867b Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:42:42 -0800 Subject: [PATCH 035/285] Automated documentation update (#844) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/docker-outside-of-docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/docker-outside-of-docker/README.md b/src/docker-outside-of-docker/README.md index 74dca7ae4..7e352af05 100644 --- a/src/docker-outside-of-docker/README.md +++ b/src/docker-outside-of-docker/README.md @@ -19,6 +19,7 @@ Re-use the host docker socket, adding the Docker CLI to a container. Feature inv |-----|-----|-----|-----| | version | Select or enter a Docker/Moby CLI version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | +| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version) | string | 0.12.0 | | dockerDashComposeVersion | Compose version to use for docker-compose (v1 or v2 or none) | string | v2 | | installDockerBuildx | Install Docker Buildx | boolean | true | From 28b69c5f97c77943efb71206f99afae8d62cc257 Mon Sep 17 00:00:00 2001 From: Jeff Putsch <150963136+jdputschadi@users.noreply.github.com> Date: Wed, 14 Feb 2024 09:37:43 -0800 Subject: [PATCH 036/285] Add RHEL support to go feature (#846) * Support RHEL with tests * update per PR feedback * Bump minor version --------- Co-authored-by: Jeff Putsch --- src/go/NOTES.md | 2 +- src/go/devcontainer-feature.json | 4 +- src/go/install.sh | 175 ++++++++++++++---- test/go/install_go_alma-8-minimal.sh | 28 +++ test/go/install_go_alma-8.sh | 28 +++ test/go/install_go_alma-9-minimal.sh | 28 +++ test/go/install_go_alma-9.sh | 28 +++ test/go/install_go_centos-7.sh | 28 +++ test/go/install_go_fedora.sh | 28 +++ test/go/install_go_mariner.sh | 28 +++ test/go/install_go_tool_in_postCreate_rhel.sh | 13 ++ test/go/scenarios.json | 75 +++++++- 12 files changed, 425 insertions(+), 40 deletions(-) create mode 100755 test/go/install_go_alma-8-minimal.sh create mode 100755 test/go/install_go_alma-8.sh create mode 100755 test/go/install_go_alma-9-minimal.sh create mode 100755 test/go/install_go_alma-9.sh create mode 100755 test/go/install_go_centos-7.sh create mode 100755 test/go/install_go_fedora.sh create mode 100755 test/go/install_go_mariner.sh create mode 100644 test/go/install_go_tool_in_postCreate_rhel.sh diff --git a/src/go/NOTES.md b/src/go/NOTES.md index 19fe92f31..79a308cf5 100644 --- a/src/go/NOTES.md +++ b/src/go/NOTES.md @@ -2,6 +2,6 @@ ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the apt, yum, dnf, or microdnf package manager installed. `bash` is required to execute the `install.sh` script. diff --git a/src/go/devcontainer-feature.json b/src/go/devcontainer-feature.json index 307ab217b..58415ce6b 100644 --- a/src/go/devcontainer-feature.json +++ b/src/go/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "go", - "version": "1.2.2", + "version": "1.3.0", "name": "Go", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/go", "description": "Installs Go and common Go utilities. Auto-detects latest version and installs needed dependencies.", @@ -44,4 +44,4 @@ "installsAfter": [ "ghcr.io/devcontainers/features/common-utils" ] -} +} \ No newline at end of file diff --git a/src/go/install.sh b/src/go/install.sh index 90e0973f5..ca01d34f5 100755 --- a/src/go/install.sh +++ b/src/go/install.sh @@ -20,36 +20,60 @@ GO_GPG_KEY_URI="https://dl.google.com/linux/linux_signing_key.pub" set -e -# Clean up -rm -rf /var/lib/apt/lists/* - if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 fi -# Ensure that login shells get the correct path if the user updated the PATH using ENV. -rm -f /etc/profile.d/00-restore-env.sh -echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh -chmod +x /etc/profile.d/00-restore-env.sh - -# Determine the appropriate non-root user -if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then - USERNAME="" - POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") - for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do - if id -u ${CURRENT_USER} > /dev/null 2>&1; then - USERNAME=${CURRENT_USER} - break - fi - done - if [ "${USERNAME}" = "" ]; then - USERNAME=root +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +MAJOR_VERSION_ID=$(echo ${VERSION_ID} | cut -d . -f 1) +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + if [[ "${ID}" = "rhel" ]] || [[ "${ID}" = *"alma"* ]] || [[ "${ID}" = *"rocky"* ]]; then + VERSION_CODENAME="rhel${MAJOR_VERSION_ID}" + else + VERSION_CODENAME="${ID}${MAJOR_VERSION_ID}" fi -elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then - USERNAME=root +else + echo "Linux distro ${ID} not supported." + exit 1 fi +# Setup INSTALL_CMD & PKG_MGR_CMD +if type apt-get > /dev/null 2>&1; then + PKG_MGR_CMD=apt-get + INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" +elif type microdnf > /dev/null 2>&1; then + PKG_MGR_CMD=microdnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +else + PKG_MGR_CMD=yum + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --noplugins --setopt=install_weak_deps=0" +fi + +# Clean up +clean_up() { + case ${ADJUSTED_ID} in + debian) + rm -rf /var/lib/apt/lists/* + ;; + rhel) + rm -rf /var/cache/dnf/* /var/cache/yum/* + rm -rf /tmp/yum.log + rm -rf ${GPG_INSTALL_PATH} + ;; + esac +} +clean_up + + # Figure out correct version of a three part version number is not passed find_version_from_git_tags() { local variable_name=$1 @@ -84,29 +108,108 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - apt-get update -y - fi +pkg_mgr_update() { + case $ADJUSTED_ID in + debian) + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${PKG_MGR_CMD} update -y + fi + ;; + rhel) + if [ ${PKG_MGR_CMD} = "microdnf" ]; then + if [ "$(ls /var/cache/yum/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} makecache ..." + ${PKG_MGR_CMD} makecache + fi + else + if [ "$(ls /var/cache/${PKG_MGR_CMD}/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} check-update ..." + set +e + ${PKG_MGR_CMD} check-update + rc=$? + if [ $rc != 0 ] && [ $rc != 100 ]; then + exit 1 + fi + set -e + fi + fi + ;; + esac } # Checks if packages are installed and installs them if not check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - apt_get_update - apt-get -y install --no-install-recommends "$@" - fi + case ${ADJUSTED_ID} in + debian) + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + rhel) + if ! rpm -q "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + esac } +# Ensure that login shells get the correct path if the user updated the PATH using ENV. +rm -f /etc/profile.d/00-restore-env.sh +echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh +chmod +x /etc/profile.d/00-restore-env.sh + +# Some distributions do not install awk by default (e.g. Mariner) +if ! type awk >/dev/null 2>&1; then + check_packages awk +fi + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + export DEBIAN_FRONTEND=noninteractive -# Install curl, tar, git, other dependencies if missing -check_packages curl ca-certificates gnupg2 tar g++ gcc libc6-dev make pkg-config +check_packages ca-certificates gnupg2 tar gcc make pkg-config + +if [ $ADJUSTED_ID = "debian" ]; then + check_packages g++ libc6-dev +else + check_packages gcc-c++ glibc-devel +fi +# Install curl, git, other dependencies if missing +if ! type curl > /dev/null 2>&1; then + check_packages curl +fi if ! type git > /dev/null 2>&1; then check_packages git fi +# Some systems, e.g. Mariner, still a few more packages +if ! type as > /dev/null 2>&1; then + check_packages binutils +fi +if ! [ -f /usr/include/linux/errno.h ]; then + check_packages kernel-headers +fi +# Minimal RHEL install may need findutils installed +if ! [ -f /usr/bin/find ]; then + check_packages findutils +fi # Get closest match for version number specified find_version_from_git_tags TARGET_GO_VERSION "https://go.googlesource.com/go" "tags/go" "." "true" @@ -128,7 +231,7 @@ fi usermod -a -G golang "${USERNAME}" mkdir -p "${TARGET_GOROOT}" "${TARGET_GOPATH}" -if [[ "${TARGET_GO_VERSION}" != "none" ]] && [[ "$(go version)" != *"${TARGET_GO_VERSION}"* ]]; then +if [[ "${TARGET_GO_VERSION}" != "none" ]] && [[ "$(go version 2>/dev/null)" != *"${TARGET_GO_VERSION}"* ]]; then # Use a temporary location for gpg keys to avoid polluting image export GNUPGHOME="/tmp/tmp-gnupg" mkdir -p ${GNUPGHOME} @@ -230,6 +333,6 @@ find "${TARGET_GOROOT}" -type d -print0 | xargs -n 1 -0 chmod g+s find "${TARGET_GOPATH}" -type d -print0 | xargs -n 1 -0 chmod g+s # Clean up -rm -rf /var/lib/apt/lists/* +clean_up echo "Done!" diff --git a/test/go/install_go_alma-8-minimal.sh b/test/go/install_go_alma-8-minimal.sh new file mode 100755 index 000000000..33a95dd32 --- /dev/null +++ b/test/go/install_go_alma-8-minimal.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# go +check "version" go version + +# revive +check "revive version" revive --version +check "revive is installed at correct path" bash -c "type revive | grep /go/bin/revive" + +# gomodifytags +check "gomodifytags is installed at correct path" bash -c "type gomodifytags | grep /go/bin/gomodifytags" + +# goplay +check "goplay is installed at correct path" bash -c "type goplay | grep /go/bin/goplay" + +# gotests +check "gotests is installed at correct path" bash -c "type gotests | grep /go/bin/gotests" + +# impl +check "impl is installed at correct path" bash -c "type impl | grep /go/bin/impl" + +# Report result +reportResults \ No newline at end of file diff --git a/test/go/install_go_alma-8.sh b/test/go/install_go_alma-8.sh new file mode 100755 index 000000000..33a95dd32 --- /dev/null +++ b/test/go/install_go_alma-8.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# go +check "version" go version + +# revive +check "revive version" revive --version +check "revive is installed at correct path" bash -c "type revive | grep /go/bin/revive" + +# gomodifytags +check "gomodifytags is installed at correct path" bash -c "type gomodifytags | grep /go/bin/gomodifytags" + +# goplay +check "goplay is installed at correct path" bash -c "type goplay | grep /go/bin/goplay" + +# gotests +check "gotests is installed at correct path" bash -c "type gotests | grep /go/bin/gotests" + +# impl +check "impl is installed at correct path" bash -c "type impl | grep /go/bin/impl" + +# Report result +reportResults \ No newline at end of file diff --git a/test/go/install_go_alma-9-minimal.sh b/test/go/install_go_alma-9-minimal.sh new file mode 100755 index 000000000..33a95dd32 --- /dev/null +++ b/test/go/install_go_alma-9-minimal.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# go +check "version" go version + +# revive +check "revive version" revive --version +check "revive is installed at correct path" bash -c "type revive | grep /go/bin/revive" + +# gomodifytags +check "gomodifytags is installed at correct path" bash -c "type gomodifytags | grep /go/bin/gomodifytags" + +# goplay +check "goplay is installed at correct path" bash -c "type goplay | grep /go/bin/goplay" + +# gotests +check "gotests is installed at correct path" bash -c "type gotests | grep /go/bin/gotests" + +# impl +check "impl is installed at correct path" bash -c "type impl | grep /go/bin/impl" + +# Report result +reportResults \ No newline at end of file diff --git a/test/go/install_go_alma-9.sh b/test/go/install_go_alma-9.sh new file mode 100755 index 000000000..33a95dd32 --- /dev/null +++ b/test/go/install_go_alma-9.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# go +check "version" go version + +# revive +check "revive version" revive --version +check "revive is installed at correct path" bash -c "type revive | grep /go/bin/revive" + +# gomodifytags +check "gomodifytags is installed at correct path" bash -c "type gomodifytags | grep /go/bin/gomodifytags" + +# goplay +check "goplay is installed at correct path" bash -c "type goplay | grep /go/bin/goplay" + +# gotests +check "gotests is installed at correct path" bash -c "type gotests | grep /go/bin/gotests" + +# impl +check "impl is installed at correct path" bash -c "type impl | grep /go/bin/impl" + +# Report result +reportResults \ No newline at end of file diff --git a/test/go/install_go_centos-7.sh b/test/go/install_go_centos-7.sh new file mode 100755 index 000000000..33a95dd32 --- /dev/null +++ b/test/go/install_go_centos-7.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# go +check "version" go version + +# revive +check "revive version" revive --version +check "revive is installed at correct path" bash -c "type revive | grep /go/bin/revive" + +# gomodifytags +check "gomodifytags is installed at correct path" bash -c "type gomodifytags | grep /go/bin/gomodifytags" + +# goplay +check "goplay is installed at correct path" bash -c "type goplay | grep /go/bin/goplay" + +# gotests +check "gotests is installed at correct path" bash -c "type gotests | grep /go/bin/gotests" + +# impl +check "impl is installed at correct path" bash -c "type impl | grep /go/bin/impl" + +# Report result +reportResults \ No newline at end of file diff --git a/test/go/install_go_fedora.sh b/test/go/install_go_fedora.sh new file mode 100755 index 000000000..33a95dd32 --- /dev/null +++ b/test/go/install_go_fedora.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# go +check "version" go version + +# revive +check "revive version" revive --version +check "revive is installed at correct path" bash -c "type revive | grep /go/bin/revive" + +# gomodifytags +check "gomodifytags is installed at correct path" bash -c "type gomodifytags | grep /go/bin/gomodifytags" + +# goplay +check "goplay is installed at correct path" bash -c "type goplay | grep /go/bin/goplay" + +# gotests +check "gotests is installed at correct path" bash -c "type gotests | grep /go/bin/gotests" + +# impl +check "impl is installed at correct path" bash -c "type impl | grep /go/bin/impl" + +# Report result +reportResults \ No newline at end of file diff --git a/test/go/install_go_mariner.sh b/test/go/install_go_mariner.sh new file mode 100755 index 000000000..33a95dd32 --- /dev/null +++ b/test/go/install_go_mariner.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# go +check "version" go version + +# revive +check "revive version" revive --version +check "revive is installed at correct path" bash -c "type revive | grep /go/bin/revive" + +# gomodifytags +check "gomodifytags is installed at correct path" bash -c "type gomodifytags | grep /go/bin/gomodifytags" + +# goplay +check "goplay is installed at correct path" bash -c "type goplay | grep /go/bin/goplay" + +# gotests +check "gotests is installed at correct path" bash -c "type gotests | grep /go/bin/gotests" + +# impl +check "impl is installed at correct path" bash -c "type impl | grep /go/bin/impl" + +# Report result +reportResults \ No newline at end of file diff --git a/test/go/install_go_tool_in_postCreate_rhel.sh b/test/go/install_go_tool_in_postCreate_rhel.sh new file mode 100644 index 000000000..dcd7311a6 --- /dev/null +++ b/test/go/install_go_tool_in_postCreate_rhel.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "mkcert version" bash -c "mkcert --version | grep v1.4.2" +check "mkcert is installed at correct path" bash -c "type mkcert | grep /go/bin/mkcert" +check "golangci-lint version" bash -c "golangci-lint --version | grep 'golangci-lint has version 1.50.0'" + +# Report result +reportResults diff --git a/test/go/scenarios.json b/test/go/scenarios.json index 86e127020..b6d8d1d0b 100644 --- a/test/go/scenarios.json +++ b/test/go/scenarios.json @@ -1,4 +1,67 @@ { + "install_go_alma-8": { + "image": "almalinux:8", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + } + }, + "install_go_alma-8-minimal": { + "image": "almalinux:8-minimal", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + } + }, + "install_go_alma-9": { + "image": "almalinux:9", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + } + }, + "install_go_alma-9-minimal": { + "image": "almalinux:9-minimal", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + } + }, + "install_go_centos-7": { + "image": "centos:centos7", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + } + }, + "install_go_fedora": { + "image": "fedora", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + } + }, + "install_go_mariner": { + "image": "mcr.microsoft.com/cbl-mariner/base/core:2.0", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + } + }, "install_go_tool_in_postCreate": { "image": "ubuntu:focal", "features": { @@ -9,6 +72,16 @@ }, "postCreateCommand": "go install filippo.io/mkcert@v1.4.2" }, + "install_go_tool_in_postCreate_rhel": { + "image": "almalinux:8", + "features": { + "go": { + "version": "latest", + "golangciLintVersion": "1.50.0" + } + }, + "postCreateCommand": "go install filippo.io/mkcert@v1.4.2" + }, "install_go_twice": { "image": "mcr.microsoft.com/devcontainers/go:1.18", "features": { @@ -17,4 +90,4 @@ } } } -} +} \ No newline at end of file From f91ff3cd8de5d33a52d21887963ba7166600cc44 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Wed, 14 Feb 2024 09:42:15 -0800 Subject: [PATCH 037/285] Automated documentation update (#851) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/go/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/go/README.md b/src/go/README.md index be035ee85..5f35da948 100644 --- a/src/go/README.md +++ b/src/go/README.md @@ -28,7 +28,7 @@ Installs Go and common Go utilities. Auto-detects latest version and installs ne ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the apt, yum, dnf, or microdnf package manager installed. `bash` is required to execute the `install.sh` script. From 7d807c2abcfaf54f1f7b9ab939cf4e79be01b416 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Wed, 21 Feb 2024 16:44:21 -0800 Subject: [PATCH 038/285] [Updates] Automated vendor dotnet-install script (#868) * Automated dotnet-install script update * Bump version --------- Co-authored-by: github-actions --- src/dotnet/devcontainer-feature.json | 2 +- src/dotnet/scripts/vendor/dotnet-install.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 7abfc2d30..878a95d3f 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.3", + "version": "2.0.4", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", diff --git a/src/dotnet/scripts/vendor/dotnet-install.sh b/src/dotnet/scripts/vendor/dotnet-install.sh index 86a2d0bad..f6b08d1f8 100755 --- a/src/dotnet/scripts/vendor/dotnet-install.sh +++ b/src/dotnet/scripts/vendor/dotnet-install.sh @@ -303,6 +303,11 @@ get_machine_architecture() { return 0 ;; aarch64|arm64) + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + echo "arm" + return 0 + fi echo "arm64" return 0 ;; From 01aad64dbdcc2a02583d9ea1ca719f60661eced0 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 23 Feb 2024 01:23:44 +0530 Subject: [PATCH 039/285] [python]-setuptools-GHSA-r9hx-vwmv-q579-remove setuptools-65.5.0-py3-none-any.whl (#866) * [python]-setuptools-GHSA-r9hx-vwmv-q579-remove setuptools-65.5.0-py3-none-any.whl * changes based on comments on pr #866 * Tried to answer acc. to last comment * Added tests to validate when skipVulnerabilityPatching=true --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 41 +++++- ...tall_python310_setuptools_vulnerability.sh | 51 ++++++-- ...ython310_skipVulnerabilityPatching_true.sh | 117 ++++++++++++++++++ ...tall_python311_setuptools_vulnerability.sh | 51 ++++++-- test/python/scenarios.json | 16 ++- 6 files changed, 247 insertions(+), 31 deletions(-) create mode 100644 test/python/install_python310_skipVulnerabilityPatching_true.sh diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 26f2e5a52..c14dd7b97 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.4.0", + "version": "1.4.1", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index 9094191f3..fcec73b0c 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -9,6 +9,7 @@ PYTHON_VERSION="${VERSION:-"latest"}" # 'system' or 'os-provided' checks the base image first, else installs 'latest' INSTALL_PYTHON_TOOLS="${INSTALLTOOLS:-"true"}" +SKIP_VULNERABILITY_PATCHING="${SKIPVULNERABILITYPATCHING:-"false"}" OPTIMIZE_BUILD_FROM_SOURCE="${OPTIMIZE:-"false"}" ENABLE_SHARED_FROM_SOURCE="${ENABLESHARED:-"false"}" PYTHON_INSTALL_PATH="${INSTALLPATH:-"/usr/local/python"}" @@ -750,12 +751,40 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then done # Temporary: Removes โ€œsetup toolsโ€ metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 - - VULNERABLE_VERSIONS=("3.10" "3.11") - RUN_TIME_PY_VER_DETECT=$(${PYTHON_SRC} --version 2>&1) - PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; - if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then - rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info + if [[ $SKIP_VULNERABILITY_PATCHING = "false" ]]; then + VULNERABLE_VERSIONS=("3.10" "3.11") + RUN_TIME_PY_VER_DETECT=$(${PYTHON_SRC} --version 2>&1) + PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; + if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then + rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info + if [[ -e "/usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" ]]; then + # remove the vulnerable setuptools-65.5.0-py3-none-any.whl file + rm /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl + # create and change to the setuptools_downloaded directory + mkdir -p /tmp/setuptools_downloaded + cd /tmp/setuptools_downloaded + # download the source distribution for setuptools using pip + pip download setuptools==65.5.1 --no-binary :all: + # extract the filename of the setuptools-*.tar.gz file + filename=$(find . -maxdepth 1 -type f) + # create a directory to store unpacked contents of the source distribution + mkdir -p /tmp/setuptools_src_dist + # extract the contents inside the new directory + tar -xzf $filename -C /tmp/setuptools_src_dist + # move to the setuptools-* directory inside /setuptools_src_dist + cd /tmp/setuptools_src_dist/setuptools-65.5.1/ + # look for setup.py file in the current directory and create a wheel file + python setup.py bdist_wheel + # move inside the dist directory in pwd + cd dist + # copy this file to the ensurepip/_bundled directory + cp setuptools-65.5.1-py3-none-any.whl /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/ + # replace the version in __init__.py file with the installed version + sed -i 's/_SETUPTOOLS_VERSION = \"65\.5\.0\"/_SETUPTOOLS_VERSION = "65.5.1"/g' /usr/local/lib/"python${PY_MAJOR_MINOR_VER}"/ensurepip/__init__.py + # cleanup created dir's + rm -rf /tmp/setuptools_downloaded /tmp/setuptools_src_dist + fi + fi fi rm -rf /tmp/pip-tmp diff --git a/test/python/install_python310_setuptools_vulnerability.sh b/test/python/install_python310_setuptools_vulnerability.sh index 0e065b1b3..d097f2113 100644 --- a/test/python/install_python310_setuptools_vulnerability.sh +++ b/test/python/install_python310_setuptools_vulnerability.sh @@ -64,22 +64,51 @@ check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bi check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" -checkVulnerableDir() +checkVulnerableFile_OR_DIR() { - DIRECTORY=$1 - VERSION=$2 - - if [[ -d $DIRECTORY ]] ; then - echoStderr "โŒ check for vulnerable setuptools version failed for python ${VERSION}." - return 1 + for arg in "$@"; do + if [[ -e $arg ]]; then + echo -e "\nโœ… Vulnerable:- ${arg} - exists in v3.10 as Vulnerability Patching has been skipped." + else + echo -e "\nโŒ Vulnerable:- ${arg} - don't exist in v3.10 as Vulnerability Patching has not been skipped." + fi + done +} + +# print setuptools +check "Show All Files/Folders which include setuptools" bash -c 'find / -name "*setuptools*"' + +# only for 3.10 +checkVulnerableFile_OR_DIR "/usr/local/py-utils/shared/lib/python3.10/site-packages/setuptools-65.5.0.dist-info" "/usr/local/lib/python3.10/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" + +# Function to check if a package is installed +checkPackageInstalled() { + if python -c "import $1" &>/dev/null; then + echo -e "\nโœ… Passed! \n$1 is installed" else - echo "โœ… Passed! Either the container does not have vulnerable version or vulnerable version specific directory got removed." - return 0 + echo -e "$1 is NOT installed\n" + echoStderr "โŒ check failed." fi } -# only for 3.10 -checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.10/site-packages/setuptools-65.5.0.dist-info" "3.10" +# Function to install a package using pip +installPackage() { + python3 -m pip install "$1" +} + +checkPipWorkingCorrectly() { + echo -e "\n๐Ÿงช Testing whether pip install works fine \n" + # List of packages to install via pip + packages=("numpy" "requests" "matplotlib") + # Install packages and check if installation was successful + for package in "${packages[@]}"; do + echo -e "\n๐Ÿงช Testing pip install $package\n" + installPackage "$package" + checkPackageInstalled "$package" + done +} + +checkPipWorkingCorrectly # Report result reportResults diff --git a/test/python/install_python310_skipVulnerabilityPatching_true.sh b/test/python/install_python310_skipVulnerabilityPatching_true.sh new file mode 100644 index 000000000..a6d582e9d --- /dev/null +++ b/test/python/install_python310_skipVulnerabilityPatching_true.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +FAILED=() + +echoStderr() +{ + echo "$@" 1>&2 +} + +check-version-ge() { + LABEL=$1 + CURRENT_VERSION=$2 + REQUIRED_VERSION=$3 + shift + echo -e "\n๐Ÿงช Testing $LABEL: '$CURRENT_VERSION' is >= '$REQUIRED_VERSION'" + local GREATER_VERSION=$((echo ${CURRENT_VERSION}; echo ${REQUIRED_VERSION}) | sort -V | tail -1) + if [ "${CURRENT_VERSION}" == "${GREATER_VERSION}" ]; then + echo "โœ… Passed!" + return 0 + else + echoStderr "โŒ $LABEL check failed." + FAILED+=("$LABEL") + return 1 + fi +} + +checkPythonPackageVersion() +{ + PACKAGE=$1 + REQUIRED_VERSION=$2 + + current_version=$(python -c "import importlib.metadata; print(importlib.metadata.version('${PACKAGE}'))") + check-version-ge "${PACKAGE}-requirement" "${current_version}" "${REQUIRED_VERSION}" +} + +checkPythonPackageVersion "setuptools" "65.5.1" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version +check "setuptools" pip list | grep setuptools + +# Check paths in settings +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +HL="\033[1;33m" +N="\033[0;37m" + +checkVulnerableFile_OR_DIR() +{ + for arg in "$@"; do + if [[ -e $arg ]]; then + echo -e "\nโœ…${HL} Vulnerable:- ${N}${arg} - ${HL}EXISTS ${N}in v3.10 as skipVulnerabilityPatching=true" + else + echo -e "\nโŒ${HL} Vulnerable:- ${arg} - ${N}don't ${HL}EXISTS ${N}in v3.10 as skipVulnerabilityPatching=false" + fi + done +} + +# print setuptools +check "Show All Files/Folders which include setuptools" bash -c 'find / -name "*setuptools*"' + +# only for 3.10 +checkVulnerableFile_OR_DIR "/usr/local/py-utils/shared/lib/python3.10/site-packages/setuptools-65.5.0.dist-info" "/usr/local/lib/python3.10/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" + +# Function to check if a package is installed +checkPackageInstalled() { + if python -c "import $1" &>/dev/null; then + echo -e "\nโœ… Passed! \n$1 is installed" + else + echo -e "$1 is NOT installed\n" + echoStderr "โŒ check failed." + fi +} + +# Function to install a package using pip +installPackage() { + python3 -m pip install "$1" +} + +checkPipWorkingCorrectly() { + echo -e "\n๐Ÿงช Testing whether pip install works fine \n" + # List of packages to install via pip + packages=("numpy" "requests" "matplotlib") + # Install packages and check if installation was successful + for package in "${packages[@]}"; do + echo -e "\n๐Ÿงช Testing pip install $package\n" + installPackage "$package" + checkPackageInstalled "$package" + done +} + +checkPipWorkingCorrectly + +# Report result +reportResults diff --git a/test/python/install_python311_setuptools_vulnerability.sh b/test/python/install_python311_setuptools_vulnerability.sh index 04a10948b..969c0ba8d 100644 --- a/test/python/install_python311_setuptools_vulnerability.sh +++ b/test/python/install_python311_setuptools_vulnerability.sh @@ -65,22 +65,51 @@ check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bi check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" -checkVulnerableDir() +checkVulnerableFile_OR_DIR() { - DIRECTORY=$1 - VERSION=$2 - - if [[ -d $DIRECTORY ]] ; then - echoStderr "โŒ check for vulnerable setuptools version failed for python ${VERSION}." - return 1 + for arg in "$@"; do + if [[ -e $arg ]]; then + echo -e "\nโœ… Vulnerable:- ${arg} - exists in v3.11 as Vulnerability Patching has been skipped." + else + echo -e "\nโŒ Vulnerable:- ${arg} - don't exist in v3.11 as Vulnerability Patching has not been skipped." + fi + done +} + +# print setuptools +check "Show All Files/Folders which include setuptools" bash -c 'find / -name "*setuptools*"' + +# only for 3.11 +checkVulnerableFile_OR_DIR "/usr/local/py-utils/shared/lib/python3.11/site-packages/setuptools-65.5.0.dist-info" "/usr/local/lib/python3.11/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" + +# Function to check if a package is installed +checkPackageInstalled() { + if python -c "import $1" &>/dev/null; then + echo -e "\nโœ… Passed! \n$1 is installed" else - echo "โœ… Passed! Either the container does not have vulnerable version or vulnerable version specific directory got removed." - return 0 + echo -e "$1 is NOT installed\n" + echoStderr "โŒ check failed." fi } -# only for 3.11 -checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.11/site-packages/setuptools-65.5.0.dist-info" "3.11" +# Function to install a package using pip +installPackage() { + python3 -m pip install "$1" +} + +checkPipWorkingCorrectly() { + echo -e "\n๐Ÿงช Testing whether pip install works fine \n" + # List of packages to install via pip + packages=("numpy" "requests" "matplotlib") + # Install packages and check if installation was successful + for package in "${packages[@]}"; do + echo -e "\n๐Ÿงช Testing pip install $package\n" + installPackage "$package" + checkPackageInstalled "$package" + done +} + +checkPipWorkingCorrectly # Report result reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index d574c84e8..920e4f541 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -1,10 +1,21 @@ { + "install_python310_skipVulnerabilityPatching_true": { + "image": "python:3.10", + "features": { + "python": { + "version": "none", + "installTools": true, + "skipVulnerabilityPatching": true + } + } + }, "install_python310_setuptools_vulnerability": { "image": "python:3.10", "features": { "python": { "version": "none", - "installTools": true + "installTools": true, + "skipVulnerabilityPatching": false } } }, @@ -13,7 +24,8 @@ "features": { "python": { "version": "none", - "installTools": true + "installTools": true, + "skipVulnerabilityPatching": false } } }, From 403c8964de8b1ec123c23ccef95b2a449b13ab16 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 23 Feb 2024 05:27:16 +0530 Subject: [PATCH 040/285] [ docker-in-docker ] - buildx can fallback to previous version if latest artifact not found (#869) * [ docker-in-docker ] - buildx fallback to prev. version * corrected the command inside get_previous_version fn to get the second to latest version instead * changes in response to comments by @samruddhikhandale * minor change * changes as requested in comments by @samruddhikhandale --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 30 +++++- .../docker_build_fallback_buildx.sh | 95 +++++++++++++++++++ test/docker-in-docker/scenarios.json | 11 +++ 4 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 test/docker-in-docker/docker_build_fallback_buildx.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 744bc1d81..5c62a057d 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.9.0", + "version": "2.9.1", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index db3cefc6a..8943c91a6 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -139,7 +139,7 @@ else fi # Install dependencies -check_packages apt-transport-https curl ca-certificates pigz iptables gnupg2 dirmngr wget +check_packages apt-transport-https curl ca-certificates pigz iptables gnupg2 dirmngr wget jq if ! type git > /dev/null 2>&1; then check_packages git fi @@ -332,14 +332,36 @@ fi usermod -aG docker ${USERNAME} +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects +} + +install_previous_version_artifacts() { + wget_exit_code=$? + if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL + previous_version=$(get_previous_version "${repo_url}") + buildx_file_name="buildx-${previous_version}.linux-${architecture}" + echo -e "\nAttempting to install ${previous_version}" + wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} + else + echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" + exit 1 + fi +} + if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then buildx_version="latest" find_version_from_git_tags buildx_version "https://github.com/docker/buildx" "refs/tags/v" - echo "(*) Installing buildx ${buildx_version}..." buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" - cd /tmp && wget "https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name}" - + + cd /tmp + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" diff --git a/test/docker-in-docker/docker_build_fallback_buildx.sh b/test/docker-in-docker/docker_build_fallback_buildx.sh new file mode 100644 index 000000000..8d30d2a46 --- /dev/null +++ b/test/docker-in-docker/docker_build_fallback_buildx.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests before test for fallback +HL="\033[1;33m" +N="\033[0;37m" +echo -e "\n๐Ÿ‘‰${HL} docker/buildx version as installed by docker-in-docker feature${N}" +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +echo -e "\n๐Ÿ‘‰${HL} Creating a scenario for fallback${N}\n" +# Code to test the made up scenario when latest version of docker/buildx fails on wget command for fetching the artifacts +repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL +architecture="$(dpkg --print-architecture)" + +# Function to fetch the latest version of the plugin +get_latest_version() { + curl -s "$repo_url/latest" | jq -r '.tag_name' +} + +# Function to fetch the previous version of the plugin +get_previous_version() { + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects +} + +# Function to change the patch number in a semver version +change_patch_number() { + local version="$1" # Input version + local new_patch="$2" # New patch number + # Extract major, minor, and current patch numbers + local major=$(echo "$version" | cut -d. -f1) + local minor=$(echo "$version" | cut -d. -f2) + local current_patch=$(echo "$version" | cut -d. -f3) + # Construct the new version with the updated patch number + local new_version="$major.$minor.$new_patch" + echo "$new_version" +} + +change_version_to_fail() { + latest_version=$1 + new_patch_number="xyz" # for testing a tag not found scenario for docker/buildx plugin + latest_version=$(get_latest_version) # can take latest_version from fn get_latest_version + buildx_version_fallback_test=$(change_patch_number "$latest_version" "$new_patch_number") # for testing a tag not found scenario for docker/buildx plugin + echo "${buildx_version_fallback_test}" +} + +install_previous_version_artifacts() { + wget_exit_code=$? + if [ $wget_exit_code -ne 0 ]; then # means wget command to fetch latest version failed + if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx ${buildx_version}..." + previous_version=$(get_previous_version) + echo -e "\nAttempting to install ${previous_version}" + buildx_file_name="buildx-${previous_version}.linux-${architecture}" + wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} + else + echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" + exit 1 + fi + fi +} + +test_version=$(change_version_to_fail "$(get_latest_version)") +buildx_file_name="buildx-${test_version}.linux-${architecture}" +buildx_version=$test_version + +# This wget command will fail as the wrong version won't fetch artifact +wget https://github.com/docker/buildx/releases/download/${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + +mkdir -p ${cli_plugins_dir} +mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx +chmod +x ${cli_plugins_dir}/docker-buildx + +chown -R "${USERNAME}:docker" "${docker_home}" +chmod -R g+r+w "${docker_home}" +find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s + +# Definition specific tests after test for fallback +echo -e "\n๐Ÿ‘‰${HL} docker/buildx version as installed by test for fallback${N}" +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index 6de695555..df27d6c28 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -87,6 +87,17 @@ } } }, + "docker_build_fallback_buildx": { + "image": "ubuntu:focal", + "features": { + "docker-in-docker": { + "version": "latest", + "installDockerBuildx": true, + "moby": "false", + "dockerDashComposeVersion": "v2" + } + } + }, // DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow "docker_with_on_create_command": { "image": "mcr.microsoft.com/devcontainers/base:debian", From c04ead01d2dd154dc782d3bb610e08e6d740b77c Mon Sep 17 00:00:00 2001 From: Nebula <40148908+nebula-it@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:51:04 -0800 Subject: [PATCH 041/285] Add support for including PowerShell profile (#804) * Add support for including PowerShell profile * Update version in devcontainer-feature.json * Add powershellProfileURL to options * Update src/powershell/devcontainer-feature.json Co-authored-by: Samruddhi Khandale * Update README.md * Create testProfile.ps1 Added a Profile for testing * Add powershell profile test * Update install_modules.sh * Undo README.md changes * Update scenarios.json * Delete test/powershell/testProfile.ps1 --------- Co-authored-by: Samruddhi Khandale Co-authored-by: Samruddhi Khandale --- src/powershell/devcontainer-feature.json | 9 +++++++-- src/powershell/install.sh | 7 +++++++ test/powershell/install_modules.sh | 1 + test/powershell/scenarios.json | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index c72923fdc..0c15b6888 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.2.0", + "version": "1.3.0", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", @@ -19,6 +19,11 @@ "type": "string", "default": "", "description": "Optional comma separated list of PowerShell modules to install." + }, + "powershellProfileURL ": { + "type": "string", + "default": "", + "description": "Optional (publicly accessible) URL to download PowerShell profile." } }, "customizations": { @@ -31,4 +36,4 @@ "installsAfter": [ "ghcr.io/devcontainers/features/common-utils" ] -} \ No newline at end of file +} diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 666071f55..dadbc306f 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -14,6 +14,7 @@ rm -rf /var/lib/apt/lists/* POWERSHELL_VERSION=${VERSION:-"latest"} POWERSHELL_MODULES="${MODULES}" +POWERSHELL_PROFILE_URL="${PROFILE_URL}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" POWERSHELL_ARCHIVE_ARCHITECTURES="amd64" @@ -162,6 +163,12 @@ if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then done fi +# If URL for powershell profile is provided, download it to '/opt/microsoft/powershell/7/profile.ps1' +if [ -n "$POWERSHELL_PROFILE_URL" ]; then + echo "Downloading PowerShell Profile from: $POWERSHELL_PROFILE_URL" + curl -sSL -o "/opt/microsoft/powershell/7/profile.ps1" "$POWERSHELL_PROFILE_URL" +fi + # Clean up rm -rf /var/lib/apt/lists/* diff --git a/test/powershell/install_modules.sh b/test/powershell/install_modules.sh index 0bb0172e6..1415af2c1 100644 --- a/test/powershell/install_modules.sh +++ b/test/powershell/install_modules.sh @@ -8,6 +8,7 @@ source dev-container-features-test-lib # Extension-specific tests check "az.resources" pwsh -Command "(Get-Module -ListAvailable -Name Az.Resources).Version.ToString()" check "az.storage" pwsh -Command "(Get-Module -ListAvailable -Name Az.Storage).Version.ToString()" +check "profile" pwsh -Command "(Get-Variable $env:ProfileLoaded).Value" # Report result reportResults diff --git a/test/powershell/scenarios.json b/test/powershell/scenarios.json index 8ced96498..b2659d7ad 100644 --- a/test/powershell/scenarios.json +++ b/test/powershell/scenarios.json @@ -3,7 +3,8 @@ "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { "powershell": { - "modules": "az.resources, az.storage" + "modules": "az.resources, az.storage", + "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" } } } From f4f492ce2b9c11afc441870424aef560de4f092c Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:57:10 -0800 Subject: [PATCH 042/285] Automated documentation update (#874) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/powershell/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/powershell/README.md b/src/powershell/README.md index 6112de79e..a44614604 100644 --- a/src/powershell/README.md +++ b/src/powershell/README.md @@ -17,6 +17,7 @@ Installs PowerShell along with needed dependencies. Useful for base Dockerfiles |-----|-----|-----|-----| | version | Select or enter a version of PowerShell. | string | latest | | modules | Optional comma separated list of PowerShell modules to install. | string | - | +| powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | ## Customizations From 758b0322d412df9d43274a4ac882e312d56a2d1d Mon Sep 17 00:00:00 2001 From: Lucas Fernando Cardoso Nunes Date: Fri, 23 Feb 2024 16:42:24 -0300 Subject: [PATCH 043/285] sync `install.sh` to `devcontainer-feature.json` default values (#867) * add default value to MOBYBUILDXVERSION Signed-off-by: Lucas Fernando Cardoso Nunes * bump updated features patch versions Signed-off-by: Lucas Fernando Cardoso Nunes * sync missing default values to `install.sh` find `^[A-Z0-9_]+="\$\{[^:]+?\}` on `install.sh` files Signed-off-by: Lucas Fernando Cardoso Nunes --------- Signed-off-by: Lucas Fernando Cardoso Nunes --- src/docker-in-docker/install.sh | 4 ++-- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 2 +- src/dotnet/devcontainer-feature.json | 2 +- src/dotnet/install.sh | 6 +++--- src/java/devcontainer-feature.json | 2 +- src/java/install.sh | 2 +- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 8943c91a6..889d1dbb2 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -10,10 +10,10 @@ DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" -DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL}" +DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL:-""}" USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 78ad1162c..a17757c06 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.0", + "version": "1.4.1", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 988b7e43c..da936235c 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -9,7 +9,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 878a95d3f..78a061d23 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.4", + "version": "2.0.5", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", diff --git a/src/dotnet/install.sh b/src/dotnet/install.sh index ff9244d3a..237a8a0be 100644 --- a/src/dotnet/install.sh +++ b/src/dotnet/install.sh @@ -7,9 +7,9 @@ # Docs: https://github.com/devcontainers/features/tree/main/src/dotnet # Maintainer: The Dev Container spec maintainers DOTNET_VERSION="${VERSION:-"latest"}" -ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS}" -DOTNET_RUNTIME_VERSIONS="${DOTNETRUNTIMEVERSIONS}" -ASPNETCORE_RUNTIME_VERSIONS="${ASPNETCORERUNTIMEVERSIONS}" +ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" +DOTNET_RUNTIME_VERSIONS="${DOTNETRUNTIMEVERSIONS:-""}" +ASPNETCORE_RUNTIME_VERSIONS="${ASPNETCORERUNTIMEVERSIONS:-""}" set -e diff --git a/src/java/devcontainer-feature.json b/src/java/devcontainer-feature.json index f65a8f360..9ed72ad59 100644 --- a/src/java/devcontainer-feature.json +++ b/src/java/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "java", - "version": "1.4.0", + "version": "1.4.1", "name": "Java (via SDKMAN!)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/java", "description": "Installs Java, SDKMAN! (if not installed), and needed dependencies.", diff --git a/src/java/install.sh b/src/java/install.sh index 37cb9e52b..30b6cb10c 100644 --- a/src/java/install.sh +++ b/src/java/install.sh @@ -18,7 +18,7 @@ INSTALL_ANT="${INSTALLANT:-"false"}" ANT_VERSION="${ANTVERSION:-"latest"}" INSTALL_GROOVY="${INSTALLGROOVY:-"false"}" GROOVY_VERSION="${GROOVYVERSION:-"latest"}" -JDK_DISTRO="${JDKDISTRO}" +JDK_DISTRO="${JDKDISTRO:-"ms"}" export SDKMAN_DIR="${SDKMAN_DIR:-"/usr/local/sdkman"}" USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 0c15b6888..5a831f7e3 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.0", + "version": "1.3.1", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index dadbc306f..2c052d7cd 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -13,7 +13,7 @@ set -e rm -rf /var/lib/apt/lists/* POWERSHELL_VERSION=${VERSION:-"latest"} -POWERSHELL_MODULES="${MODULES}" +POWERSHELL_MODULES="${MODULES:-""}" POWERSHELL_PROFILE_URL="${PROFILE_URL}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" From a8dc2dbb5909de35662067bb9428a7aafc10d3fe Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 26 Feb 2024 09:28:00 -0800 Subject: [PATCH 044/285] Docker: Unpin "mobyBuildxVersion" (#877) --- .../devcontainer-feature.json | 6 +++--- src/docker-in-docker/install.sh | 14 +++++++------ .../devcontainer-feature.json | 6 +++--- src/docker-outside-of-docker/install.sh | 2 +- .../docker_specific_moby_buildx.sh | 20 +++++++++++++++++++ test/docker-in-docker/scenarios.json | 8 ++++++++ test/docker-in-docker/test.sh | 1 + .../docker_specific_moby_buildx.sh | 19 ++++++++++++++++++ test/docker-outside-of-docker/scenarios.json | 8 ++++++++ test/docker-outside-of-docker/test.sh | 1 + 10 files changed, 72 insertions(+), 13 deletions(-) create mode 100755 test/docker-in-docker/docker_specific_moby_buildx.sh create mode 100755 test/docker-outside-of-docker/docker_specific_moby_buildx.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 5c62a057d..d74f205a1 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.9.1", + "version": "2.9.2", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", @@ -22,8 +22,8 @@ }, "mobyBuildxVersion": { "type": "string", - "default": "0.12.0", - "description": "Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version)" + "default": "latest", + "description": "Install a specific version of moby-buildx when using Moby" }, "dockerDashComposeVersion": { "type": "string", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 889d1dbb2..9f0e18cee 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -10,7 +10,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL:-""}" @@ -228,11 +228,13 @@ else # Install engine set +e # Handle error gracefully apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx${buildx_version_suffix} moby-engine${engine_version_suffix} - if [ $? -ne 0 ]; then - err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')." - exit 1 - fi - set -e + exit_code=$? + set -e + + if [ ${exit_code} -ne 0 ]; then + err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')." + exit 1 + fi # Install compose apt-get -y install --no-install-recommends moby-compose || err "Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping." diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index a17757c06..d4c1447ba 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.1", + "version": "1.4.2", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", @@ -22,8 +22,8 @@ }, "mobyBuildxVersion": { "type": "string", - "default": "0.12.0", - "description": "Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version)" + "default": "latest", + "description": "Install a specific version of moby-buildx when using Moby" }, "dockerDashComposeVersion": { "type": "string", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index da936235c..65424740e 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -9,7 +9,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" diff --git a/test/docker-in-docker/docker_specific_moby_buildx.sh b/test/docker-in-docker/docker_specific_moby_buildx.sh new file mode 100755 index 000000000..1ca2b20bd --- /dev/null +++ b/test/docker-in-docker/docker_specific_moby_buildx.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib +# Definition specific tests +check "moby-buildx" bash -c "dpkg-query -W moby-buildx | grep -E '0.12.0'" + +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ + +check "installs docker-compose v1 install" bash -c "type docker-compose" +check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index df27d6c28..ccf57b188 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -98,6 +98,14 @@ } } }, + "docker_specific_moby_buildx": { + "image": "ubuntu:focal", + "features": { + "docker-in-docker": { + "mobyBuildxVersion": "0.12.0" + } + } + }, // DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow "docker_with_on_create_command": { "image": "mcr.microsoft.com/devcontainers/base:debian", diff --git a/test/docker-in-docker/test.sh b/test/docker-in-docker/test.sh index e86a841c2..10a7232b6 100755 --- a/test/docker-in-docker/test.sh +++ b/test/docker-in-docker/test.sh @@ -12,6 +12,7 @@ check "docker-ps" bash -c "docker ps" check "log-exists" bash -c "ls /tmp/dockerd.log" check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'" check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'" +check "moby-buildx" bash -c "dpkg-query -W moby-buildx" # Report result reportResults \ No newline at end of file diff --git a/test/docker-outside-of-docker/docker_specific_moby_buildx.sh b/test/docker-outside-of-docker/docker_specific_moby_buildx.sh new file mode 100755 index 000000000..929fa6080 --- /dev/null +++ b/test/docker-outside-of-docker/docker_specific_moby_buildx.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib +# Definition specific tests +check "moby-buildx" bash -c "dpkg-query -W moby-buildx | grep -E '0.12.0'" + +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ + +check "installs docker-compose v1 install" bash -c "type docker-compose" + +# Report result +reportResults diff --git a/test/docker-outside-of-docker/scenarios.json b/test/docker-outside-of-docker/scenarios.json index 3b82c6cc5..61f1ab402 100644 --- a/test/docker-outside-of-docker/scenarios.json +++ b/test/docker-outside-of-docker/scenarios.json @@ -124,5 +124,13 @@ } }, "remoteUser": "node" + }, + "docker_specific_moby_buildx": { + "image": "ubuntu:focal", + "features": { + "docker-outside-of-docker": { + "mobyBuildxVersion": "0.12.0" + } + } } } diff --git a/test/docker-outside-of-docker/test.sh b/test/docker-outside-of-docker/test.sh index fe9098e10..5206f5977 100644 --- a/test/docker-outside-of-docker/test.sh +++ b/test/docker-outside-of-docker/test.sh @@ -10,6 +10,7 @@ check "docker compose" bash -c "docker compose version" check "docker-compose" bash -c "docker-compose --version" check "docker-ps" bash -c "docker ps >/dev/null" +check "moby-buildx" bash -c "dpkg-query -W moby-buildx" # Report result reportResults \ No newline at end of file From 979c12bd966bb4ded8e9ac74a2b656f3eeaa77d0 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:30:24 -0800 Subject: [PATCH 045/285] Automated documentation update (#881) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/docker-in-docker/README.md | 2 +- src/docker-outside-of-docker/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docker-in-docker/README.md b/src/docker-in-docker/README.md index 41004b7e1..2ab700118 100644 --- a/src/docker-in-docker/README.md +++ b/src/docker-in-docker/README.md @@ -17,7 +17,7 @@ Create child containers *inside* a container, independent from the host's docker |-----|-----|-----|-----| | version | Select or enter a Docker/Moby Engine version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | -| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version) | string | 0.12.0 | +| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby | string | latest | | dockerDashComposeVersion | Default version of Docker Compose (v1 or v2 or none) | string | v1 | | azureDnsAutoDetection | Allow automatically setting the dockerd DNS server when the installation script detects it is running in Azure | boolean | true | | dockerDefaultAddressPool | Define default address pools for Docker networks. e.g. base=192.168.0.0/16,size=24 | string | - | diff --git a/src/docker-outside-of-docker/README.md b/src/docker-outside-of-docker/README.md index 7e352af05..18fe79573 100644 --- a/src/docker-outside-of-docker/README.md +++ b/src/docker-outside-of-docker/README.md @@ -19,7 +19,7 @@ Re-use the host docker socket, adding the Docker CLI to a container. Feature inv |-----|-----|-----|-----| | version | Select or enter a Docker/Moby CLI version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | -| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version) | string | 0.12.0 | +| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby | string | latest | | dockerDashComposeVersion | Compose version to use for docker-compose (v1 or v2 or none) | string | v2 | | installDockerBuildx | Install Docker Buildx | boolean | true | From 0604e45e6bcc6f4fa0fa6b4ea5ab58f47b2acba8 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 27 Feb 2024 09:20:42 -0800 Subject: [PATCH 046/285] Oryx: Fix build failures ; pin to .NET 8.0.101 (#882) * Oryx: Fix build failures ; pin to .NET 8.0.101 * fix build * fix "install_dotnet_and_oryx" --- .../update-dotnet-install-script.yml | 9 + src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 43 +- src/oryx/scripts/vendor/README.md | 27 + src/oryx/scripts/vendor/dotnet-install.sh | 1844 +++++++++++++++++ 5 files changed, 1921 insertions(+), 4 deletions(-) create mode 100644 src/oryx/scripts/vendor/README.md create mode 100755 src/oryx/scripts/vendor/dotnet-install.sh diff --git a/.github/workflows/update-dotnet-install-script.yml b/.github/workflows/update-dotnet-install-script.yml index ebd175cff..85a9ea4d8 100644 --- a/.github/workflows/update-dotnet-install-script.yml +++ b/.github/workflows/update-dotnet-install-script.yml @@ -25,6 +25,9 @@ jobs: set -e echo "Start." + # Update dotnet-install for Oryx Feature as well + cp src/dotnet/scripts/vendor/dotnet-install.sh src/oryx/scripts/vendor/dotnet-install.sh + # Configure git and Push updates git config --global user.email github-actions@github.com git config --global user.name github-actions @@ -36,12 +39,18 @@ jobs: # Add / update and commit git add src/dotnet/scripts/vendor/dotnet-install.sh + git add src/dotnet/scripts/vendor/dotnet-install.sh + git commit -m 'Automated dotnet-install script update' || export NO_UPDATES=true # Bump version and push if [ "$NO_UPDATES" != "true" ] ; then echo "$(jq --indent 4 '.version = (.version | split(".") | map(tonumber) | .[2] += 1 | join("."))' src/dotnet/devcontainer-feature.json)" > src/dotnet/devcontainer-feature.json git add src/dotnet/devcontainer-feature.json + + echo "$(jq --indent 4 '.version = (.version | split(".") | map(tonumber) | .[2] += 1 | join("."))' src/oryx/devcontainer-feature.json)" > src/oryx/devcontainer-feature.json + git add src/oryx/devcontainer-feature.json + git commit -m 'Bump version' git push origin "$branch" gh api \ diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index aae6d43db..fe13e2fcb 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.2.0", + "version": "1.3.0", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index a16890bef..f4c547fb5 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -70,6 +70,25 @@ check_packages() { fi } +install_dotnet_with_script() +{ + local version="$1" + CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + DOTNET_INSTALL_SCRIPT="$CURRENT_DIR/scripts/vendor/dotnet-install.sh" + DOTNET_INSTALL_DIR='/usr/share/dotnet' + + check_packages icu-devtools + + "$DOTNET_INSTALL_SCRIPT" \ + --version "$version" \ + --install-dir "$DOTNET_INSTALL_DIR" \ + --no-path + + DOTNET_BINARY="dotnet" + export PATH="${PATH}:/usr/share/dotnet" + DOTNET_BINARY_INSTALLATION="/usr/share/dotnet/sdk/${version}" +} + install_dotnet_using_apt() { echo "Attempting to auto-install dotnet..." install_from_microsoft_feed=false @@ -86,6 +105,7 @@ install_dotnet_using_apt() { DOTNET_SKIP_FIRST_TIME_EXPERIENCE="true" apt-get install -yq $DOTNET_INSTALLATION_PACKAGE fi + DOTNET_BINARY="/usr/bin/dotnet" echo -e "Finished attempt to install dotnet. Sdks installed:\n" dotnet --list-sdks @@ -126,6 +146,7 @@ usermod -a -G oryx "${USERNAME}" # Required to decide if we want to clean up dotnet later. DOTNET_INSTALLATION_PACKAGE="" +DOTNET_BINARY_INSTALLATION="" DOTNET_BINARY="" if dotnet --version > /dev/null ; then @@ -133,18 +154,23 @@ if dotnet --version > /dev/null ; then fi MAJOR_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 1) +PATCH_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 3) # Oryx needs to be built with .NET 8 -if [[ "${DOTNET_BINARY}" = "" ]] || [[ $MAJOR_VERSION_ID != "8" ]] ; then +if [[ "${DOTNET_BINARY}" = "" ]] || [[ $MAJOR_VERSION_ID != "8" ]] || [[ $MAJOR_VERSION_ID = "8" && ${PATCH_VERSION_ID} -ge "101" ]] ; then echo "'dotnet 8' was not detected. Attempting to install .NET 8 to build oryx." - install_dotnet_using_apt + + # The oryx build fails with .Net 8.0.201, see https://github.com/devcontainers/images/issues/974 + # Pinning it to a working version until the upstream Oryx repo updates the dependency + # install_dotnet_using_apt + PINNED_SDK_VERSION="8.0.101" + install_dotnet_with_script ${PINNED_SDK_VERSION} if ! dotnet --version > /dev/null ; then echo "(!) Please install Dotnet before installing Oryx" exit 1 fi - DOTNET_BINARY="/usr/bin/dotnet" fi BUILD_SCRIPT_GENERATOR=/usr/local/buildscriptgen @@ -156,6 +182,11 @@ mkdir -p ${ORYX} git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX +if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then + cd $GIT_ORYX + dotnet new globaljson --sdk-version ${PINNED_SDK_VERSION} +fi + SOLUTION_FILE_NAME="Oryx.sln" echo "Building solution '$SOLUTION_FILE_NAME'..." @@ -203,6 +234,12 @@ if [[ "${DOTNET_INSTALLATION_PACKAGE}" != "" ]]; then apt purge -yq $DOTNET_INSTALLATION_PACKAGE fi +if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then + rm -f ${GIT_ORYX}/global.json + rm -rf ${DOTNET_BINARY_INSTALLATION} +fi + + # Clean up rm -rf /var/lib/apt/lists/* diff --git a/src/oryx/scripts/vendor/README.md b/src/oryx/scripts/vendor/README.md new file mode 100644 index 000000000..181b53781 --- /dev/null +++ b/src/oryx/scripts/vendor/README.md @@ -0,0 +1,27 @@ +### **IMPORTANT NOTE** + +Scripts in this directory are sourced externally and not maintained by the Dev Container spec maintainers. Do not make changes directly as they might be overwritten at any moment. + +## dotnet-install.sh + +`dotnet-install.sh` is a copy of . ([Script reference](https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script)) + +Quick options reminder for `dotnet-install.sh`: + +- `--version`: `"latest"` (default) or an exact version in the form A.B.C like `"6.0.413"` +- `--channel`: `"LTS"` (default), `"STS"`, a two-part version in the form A.B like `"6.0"` or three-part form A.B.Cxx like `"6.0.1xx"` +- `--quality`: `"daily"`, `"preview"` or `"GA"` +- The channel option is only used when version is 'latest' because an exact version overrides the channel option +- The quality option is only used when channel is 'A.B' or 'A.B.Cxx' because it can't be used with STS or LTS + +Examples + +``` +dotnet-install.sh [--version latest] [--channel LTS] +dotnet-install.sh [--version latest] --channel STS +dotnet-install.sh [--version latest] --channel 6.0 [--quality GA] +dotnet-install.sh [--version latest] --channel 6.0.4xx [--quality GA] +dotnet-install.sh [--version latest] --channel 8.0 --quality preview +dotnet-install.sh [--version latest] --channel 8.0 --quality daily +dotnet-install.sh --version 6.0.413 +``` \ No newline at end of file diff --git a/src/oryx/scripts/vendor/dotnet-install.sh b/src/oryx/scripts/vendor/dotnet-install.sh new file mode 100755 index 000000000..f6b08d1f8 --- /dev/null +++ b/src/oryx/scripts/vendor/dotnet-install.sh @@ -0,0 +1,1844 @@ +#!/usr/bin/env bash +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Stop script on NZEC +set -e +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u +# By default cmd1 | cmd2 returns exit code of cmd2 regardless of cmd1 success +# This is causing it to fail +set -o pipefail + +# Use in the the functions: eval $invocation +invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"' + +# standard output may be used as a return value in the functions +# we need a way to write text on the screen in the functions so that +# it won't interfere with the return value. +# Exposing stream 3 as a pipe to standard output of the script itself +exec 3>&1 + +# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors. +# See if stdout is a terminal +if [ -t 1 ] && command -v tput > /dev/null; then + # see if it supports colors + ncolors=$(tput colors || echo 0) + if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then + bold="$(tput bold || echo)" + normal="$(tput sgr0 || echo)" + black="$(tput setaf 0 || echo)" + red="$(tput setaf 1 || echo)" + green="$(tput setaf 2 || echo)" + yellow="$(tput setaf 3 || echo)" + blue="$(tput setaf 4 || echo)" + magenta="$(tput setaf 5 || echo)" + cyan="$(tput setaf 6 || echo)" + white="$(tput setaf 7 || echo)" + fi +fi + +say_warning() { + printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" >&3 +} + +say_err() { + printf "%b\n" "${red:-}dotnet_install: Error: $1${normal:-}" >&2 +} + +say() { + # using stream 3 (defined in the beginning) to not interfere with stdout of functions + # which may be used as return value + printf "%b\n" "${cyan:-}dotnet-install:${normal:-} $1" >&3 +} + +say_verbose() { + if [ "$verbose" = true ]; then + say "$1" + fi +} + +# This platform list is finite - if the SDK/Runtime has supported Linux distribution-specific assets, +# then and only then should the Linux distribution appear in this list. +# Adding a Linux distribution to this list does not imply distribution-specific support. +get_legacy_os_name_from_platform() { + eval $invocation + + platform="$1" + case "$platform" in + "centos.7") + echo "centos" + return 0 + ;; + "debian.8") + echo "debian" + return 0 + ;; + "debian.9") + echo "debian.9" + return 0 + ;; + "fedora.23") + echo "fedora.23" + return 0 + ;; + "fedora.24") + echo "fedora.24" + return 0 + ;; + "fedora.27") + echo "fedora.27" + return 0 + ;; + "fedora.28") + echo "fedora.28" + return 0 + ;; + "opensuse.13.2") + echo "opensuse.13.2" + return 0 + ;; + "opensuse.42.1") + echo "opensuse.42.1" + return 0 + ;; + "opensuse.42.3") + echo "opensuse.42.3" + return 0 + ;; + "rhel.7"*) + echo "rhel" + return 0 + ;; + "ubuntu.14.04") + echo "ubuntu" + return 0 + ;; + "ubuntu.16.04") + echo "ubuntu.16.04" + return 0 + ;; + "ubuntu.16.10") + echo "ubuntu.16.10" + return 0 + ;; + "ubuntu.18.04") + echo "ubuntu.18.04" + return 0 + ;; + "alpine.3.4.3") + echo "alpine" + return 0 + ;; + esac + return 1 +} + +get_legacy_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ -n "$runtime_id" ]; then + echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") + if [ -n "$os" ]; then + echo "$os" + return 0 + fi + fi + fi + + say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" + return 1 +} + +get_linux_platform_name() { + eval $invocation + + if [ -n "$runtime_id" ]; then + echo "${runtime_id%-*}" + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + echo "$ID${VERSION_ID:+.${VERSION_ID}}" + return 0 + elif [ -e /etc/redhat-release ]; then + local redhatRelease=$(&1 || true) | grep -q musl +} + +get_current_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ "$uname" = "FreeBSD" ]; then + echo "freebsd" + return 0 + elif [ "$uname" = "Linux" ]; then + local linux_platform_name="" + linux_platform_name="$(get_linux_platform_name)" || true + + if [ "$linux_platform_name" = "rhel.6" ]; then + echo $linux_platform_name + return 0 + elif is_musl_based_distro; then + echo "linux-musl" + return 0 + elif [ "$linux_platform_name" = "linux-musl" ]; then + echo "linux-musl" + return 0 + else + echo "linux" + return 0 + fi + fi + + say_err "OS name could not be detected: UName = $uname" + return 1 +} + +machine_has() { + eval $invocation + + command -v "$1" > /dev/null 2>&1 + return $? +} + +check_min_reqs() { + local hasMinimum=false + if machine_has "curl"; then + hasMinimum=true + elif machine_has "wget"; then + hasMinimum=true + fi + + if [ "$hasMinimum" = "false" ]; then + say_err "curl (recommended) or wget are required to download dotnet. Install missing prerequisite to proceed." + return 1 + fi + return 0 +} + +# args: +# input - $1 +to_lowercase() { + #eval $invocation + + echo "$1" | tr '[:upper:]' '[:lower:]' + return 0 +} + +# args: +# input - $1 +remove_trailing_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input%/}" + return 0 +} + +# args: +# input - $1 +remove_beginning_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input#/}" + return 0 +} + +# args: +# root_path - $1 +# child_path - $2 - this parameter can be empty +combine_paths() { + eval $invocation + + # TODO: Consider making it work with any number of paths. For now: + if [ ! -z "${3:-}" ]; then + say_err "combine_paths: Function takes two parameters." + return 1 + fi + + local root_path="$(remove_trailing_slash "$1")" + local child_path="$(remove_beginning_slash "${2:-}")" + say_verbose "combine_paths: root_path=$root_path" + say_verbose "combine_paths: child_path=$child_path" + echo "$root_path/$child_path" + return 0 +} + +get_machine_architecture() { + eval $invocation + + if command -v uname > /dev/null; then + CPUName=$(uname -m) + case $CPUName in + armv*l) + echo "arm" + return 0 + ;; + aarch64|arm64) + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + echo "arm" + return 0 + fi + echo "arm64" + return 0 + ;; + s390x) + echo "s390x" + return 0 + ;; + ppc64le) + echo "ppc64le" + return 0 + ;; + loongarch64) + echo "loongarch64" + return 0 + ;; + esac + fi + + # Always default to 'x64' + echo "x64" + return 0 +} + +# args: +# architecture - $1 +get_normalized_architecture_from_architecture() { + eval $invocation + + local architecture="$(to_lowercase "$1")" + + if [[ $architecture == \ ]]; then + echo "$(get_machine_architecture)" + return 0 + fi + + case "$architecture" in + amd64|x64) + echo "x64" + return 0 + ;; + arm) + echo "arm" + return 0 + ;; + arm64) + echo "arm64" + return 0 + ;; + s390x) + echo "s390x" + return 0 + ;; + ppc64le) + echo "ppc64le" + return 0 + ;; + loongarch64) + echo "loongarch64" + return 0 + ;; + esac + + say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" + return 1 +} + +# args: +# version - $1 +# channel - $2 +# architecture - $3 +get_normalized_architecture_for_specific_sdk_version() { + eval $invocation + + local is_version_support_arm64="$(is_arm64_supported "$1")" + local is_channel_support_arm64="$(is_arm64_supported "$2")" + local architecture="$3"; + local osname="$(get_current_os_name)" + + if [ "$osname" == "osx" ] && [ "$architecture" == "arm64" ] && { [ "$is_version_support_arm64" = false ] || [ "$is_channel_support_arm64" = false ]; }; then + #check if rosetta is installed + if [ "$(/usr/bin/pgrep oahd >/dev/null 2>&1;echo $?)" -eq 0 ]; then + say_verbose "Changing user architecture from '$architecture' to 'x64' because .NET SDKs prior to version 6.0 do not support arm64." + echo "x64" + return 0; + else + say_err "Architecture \`$architecture\` is not supported for .NET SDK version \`$version\`. Please install Rosetta to allow emulation of the \`$architecture\` .NET SDK on this platform" + return 1 + fi + fi + + echo "$architecture" + return 0 +} + +# args: +# version or channel - $1 +is_arm64_supported() { + #any channel or version that starts with the specified versions + case "$1" in + ( "1"* | "2"* | "3"* | "4"* | "5"*) + echo false + return 0 + esac + + echo true + return 0 +} + +# args: +# user_defined_os - $1 +get_normalized_os() { + eval $invocation + + local osname="$(to_lowercase "$1")" + if [ ! -z "$osname" ]; then + case "$osname" in + osx | freebsd | rhel.6 | linux-musl | linux) + echo "$osname" + return 0 + ;; + macos) + osname='osx' + echo "$osname" + return 0 + ;; + *) + say_err "'$user_defined_os' is not a supported value for --os option, supported values are: osx, macos, linux, linux-musl, freebsd, rhel.6. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues." + return 1 + ;; + esac + else + osname="$(get_current_os_name)" || return 1 + fi + echo "$osname" + return 0 +} + +# args: +# quality - $1 +get_normalized_quality() { + eval $invocation + + local quality="$(to_lowercase "$1")" + if [ ! -z "$quality" ]; then + case "$quality" in + daily | signed | validated | preview) + echo "$quality" + return 0 + ;; + ga) + #ga quality is available without specifying quality, so normalizing it to empty + return 0 + ;; + *) + say_err "'$quality' is not a supported value for --quality option. Supported values are: daily, signed, validated, preview, ga. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues." + return 1 + ;; + esac + fi + return 0 +} + +# args: +# channel - $1 +get_normalized_channel() { + eval $invocation + + local channel="$(to_lowercase "$1")" + + if [[ $channel == current ]]; then + say_warning 'Value "Current" is deprecated for -Channel option. Use "STS" instead.' + fi + + if [[ $channel == release/* ]]; then + say_warning 'Using branch name with -Channel option is no longer supported with newer releases. Use -Quality option with a channel in X.Y format instead.'; + fi + + if [ ! -z "$channel" ]; then + case "$channel" in + lts) + echo "LTS" + return 0 + ;; + sts) + echo "STS" + return 0 + ;; + current) + echo "STS" + return 0 + ;; + *) + echo "$channel" + return 0 + ;; + esac + fi + + return 0 +} + +# args: +# runtime - $1 +get_normalized_product() { + eval $invocation + + local product="" + local runtime="$(to_lowercase "$1")" + if [[ "$runtime" == "dotnet" ]]; then + product="dotnet-runtime" + elif [[ "$runtime" == "aspnetcore" ]]; then + product="aspnetcore-runtime" + elif [ -z "$runtime" ]; then + product="dotnet-sdk" + fi + echo "$product" + return 0 +} + +# The version text returned from the feeds is a 1-line or 2-line string: +# For the SDK and the dotnet runtime (2 lines): +# Line 1: # commit_hash +# Line 2: # 4-part version +# For the aspnetcore runtime (1 line): +# Line 1: # 4-part version + +# args: +# version_text - stdin +get_version_from_latestversion_file_content() { + eval $invocation + + cat | tail -n 1 | sed 's/\r$//' + return 0 +} + +# args: +# install_root - $1 +# relative_path_to_package - $2 +# specific_version - $3 +is_dotnet_package_installed() { + eval $invocation + + local install_root="$1" + local relative_path_to_package="$2" + local specific_version="${3//[$'\t\r\n']}" + + local dotnet_package_path="$(combine_paths "$(combine_paths "$install_root" "$relative_path_to_package")" "$specific_version")" + say_verbose "is_dotnet_package_installed: dotnet_package_path=$dotnet_package_path" + + if [ -d "$dotnet_package_path" ]; then + return 0 + else + return 1 + fi +} + +# args: +# downloaded file - $1 +# remote_file_size - $2 +validate_remote_local_file_sizes() +{ + eval $invocation + + local downloaded_file="$1" + local remote_file_size="$2" + local file_size='' + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + file_size="$(stat -c '%s' "$downloaded_file")" + elif [[ "$OSTYPE" == "darwin"* ]]; then + # hardcode in order to avoid conflicts with GNU stat + file_size="$(/usr/bin/stat -f '%z' "$downloaded_file")" + fi + + if [ -n "$file_size" ]; then + say "Downloaded file size is $file_size bytes." + + if [ -n "$remote_file_size" ] && [ -n "$file_size" ]; then + if [ "$remote_file_size" -ne "$file_size" ]; then + say "The remote and local file sizes are not equal. The remote file size is $remote_file_size bytes and the local size is $file_size bytes. The local package may be corrupted." + else + say "The remote and local file sizes are equal." + fi + fi + + else + say "Either downloaded or local package size can not be measured. One of them may be corrupted." + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +get_version_from_latestversion_file() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + + local version_file_url=null + if [[ "$runtime" == "dotnet" ]]; then + version_file_url="$azure_feed/Runtime/$channel/latest.version" + elif [[ "$runtime" == "aspnetcore" ]]; then + version_file_url="$azure_feed/aspnetcore/Runtime/$channel/latest.version" + elif [ -z "$runtime" ]; then + version_file_url="$azure_feed/Sdk/$channel/latest.version" + else + say_err "Invalid value for \$runtime" + return 1 + fi + say_verbose "get_version_from_latestversion_file: latest url: $version_file_url" + + download "$version_file_url" || return $? + return 0 +} + +# args: +# json_file - $1 +parse_globaljson_file_for_version() { + eval $invocation + + local json_file="$1" + if [ ! -f "$json_file" ]; then + say_err "Unable to find \`$json_file\`" + return 1 + fi + + sdk_section=$(cat $json_file | tr -d "\r" | awk '/"sdk"/,/}/') + if [ -z "$sdk_section" ]; then + say_err "Unable to parse the SDK node in \`$json_file\`" + return 1 + fi + + sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}') + sdk_list=${sdk_list//[\" ]/} + sdk_list=${sdk_list//,/$'\n'} + + local version_info="" + while read -r line; do + IFS=: + while read -r key value; do + if [[ "$key" == "version" ]]; then + version_info=$value + fi + done <<< "$line" + done <<< "$sdk_list" + if [ -z "$version_info" ]; then + say_err "Unable to find the SDK:version node in \`$json_file\`" + return 1 + fi + + unset IFS; + echo "$version_info" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# version - $4 +# json_file - $5 +get_specific_version_from_version() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local version="$(to_lowercase "$4")" + local json_file="$5" + + if [ -z "$json_file" ]; then + if [[ "$version" == "latest" ]]; then + local version_info + version_info="$(get_version_from_latestversion_file "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 + say_verbose "get_specific_version_from_version: version_info=$version_info" + echo "$version_info" | get_version_from_latestversion_file_content + return 0 + else + echo "$version" + return 0 + fi + else + local version_info + version_info="$(parse_globaljson_file_for_version "$json_file")" || return 1 + echo "$version_info" + return 0 + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +# normalized_os - $5 +construct_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + local specific_product_version="$(get_specific_product_version "$1" "$4")" + local osname="$5" + + local download_link=null + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz" + else + return 1 + fi + + echo "$download_link" + return 0 +} + +# args: +# azure_feed - $1 +# specific_version - $2 +# download link - $3 (optional) +get_specific_product_version() { + # If we find a 'productVersion.txt' at the root of any folder, we'll use its contents + # to resolve the version of what's in the folder, superseding the specified version. + # if 'productVersion.txt' is missing but download link is already available, product version will be taken from download link + eval $invocation + + local azure_feed="$1" + local specific_version="${2//[$'\t\r\n']}" + local package_download_link="" + if [ $# -gt 2 ]; then + local package_download_link="$3" + fi + local specific_product_version=null + + # Try to get the version number, using the productVersion.txt file located next to the installer file. + local download_links=($(get_specific_product_version_url "$azure_feed" "$specific_version" true "$package_download_link") + $(get_specific_product_version_url "$azure_feed" "$specific_version" false "$package_download_link")) + + for download_link in "${download_links[@]}" + do + say_verbose "Checking for the existence of $download_link" + + if machine_has "curl" + then + if ! specific_product_version=$(curl -s --fail "${download_link}${feed_credential}" 2>&1); then + continue + else + echo "${specific_product_version//[$'\t\r\n']}" + return 0 + fi + + elif machine_has "wget" + then + specific_product_version=$(wget -qO- "${download_link}${feed_credential}" 2>&1) + if [ $? = 0 ]; then + echo "${specific_product_version//[$'\t\r\n']}" + return 0 + fi + fi + done + + # Getting the version number with productVersion.txt has failed. Try parsing the download link for a version number. + say_verbose "Failed to get the version using productVersion.txt file. Download link will be parsed instead." + specific_product_version="$(get_product_specific_version_from_download_link "$package_download_link" "$specific_version")" + echo "${specific_product_version//[$'\t\r\n']}" + return 0 +} + +# args: +# azure_feed - $1 +# specific_version - $2 +# is_flattened - $3 +# download link - $4 (optional) +get_specific_product_version_url() { + eval $invocation + + local azure_feed="$1" + local specific_version="$2" + local is_flattened="$3" + local package_download_link="" + if [ $# -gt 3 ]; then + local package_download_link="$4" + fi + + local pvFileName="productVersion.txt" + if [ "$is_flattened" = true ]; then + if [ -z "$runtime" ]; then + pvFileName="sdk-productVersion.txt" + elif [[ "$runtime" == "dotnet" ]]; then + pvFileName="runtime-productVersion.txt" + else + pvFileName="$runtime-productVersion.txt" + fi + fi + + local download_link=null + + if [ -z "$package_download_link" ]; then + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/${pvFileName}" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/${pvFileName}" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/${pvFileName}" + else + return 1 + fi + else + download_link="${package_download_link%/*}/${pvFileName}" + fi + + say_verbose "Constructed productVersion link: $download_link" + echo "$download_link" + return 0 +} + +# args: +# download link - $1 +# specific version - $2 +get_product_specific_version_from_download_link() +{ + eval $invocation + + local download_link="$1" + local specific_version="$2" + local specific_product_version="" + + if [ -z "$download_link" ]; then + echo "$specific_version" + return 0 + fi + + #get filename + filename="${download_link##*/}" + + #product specific version follows the product name + #for filename 'dotnet-sdk-3.1.404-linux-x64.tar.gz': the product version is 3.1.404 + IFS='-' + read -ra filename_elems <<< "$filename" + count=${#filename_elems[@]} + if [[ "$count" -gt 2 ]]; then + specific_product_version="${filename_elems[2]}" + else + specific_product_version=$specific_version + fi + unset IFS; + echo "$specific_product_version" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +construct_legacy_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + + local distro_specific_osname + distro_specific_osname="$(get_legacy_os_name)" || return 1 + + local legacy_download_link=null + if [[ "$runtime" == "dotnet" ]]; then + legacy_download_link="$azure_feed/Runtime/$specific_version/dotnet-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + elif [ -z "$runtime" ]; then + legacy_download_link="$azure_feed/Sdk/$specific_version/dotnet-dev-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + else + return 1 + fi + + echo "$legacy_download_link" + return 0 +} + +get_user_install_path() { + eval $invocation + + if [ ! -z "${DOTNET_INSTALL_DIR:-}" ]; then + echo "$DOTNET_INSTALL_DIR" + else + echo "$HOME/.dotnet" + fi + return 0 +} + +# args: +# install_dir - $1 +resolve_installation_path() { + eval $invocation + + local install_dir=$1 + if [ "$install_dir" = "" ]; then + local user_install_path="$(get_user_install_path)" + say_verbose "resolve_installation_path: user_install_path=$user_install_path" + echo "$user_install_path" + return 0 + fi + + echo "$install_dir" + return 0 +} + +# args: +# relative_or_absolute_path - $1 +get_absolute_path() { + eval $invocation + + local relative_or_absolute_path=$1 + echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" + return 0 +} + +# args: +# input_files - stdin +# root_path - $1 +# out_path - $2 +# override - $3 +copy_files_or_dirs_from_list() { + eval $invocation + + local root_path="$(remove_trailing_slash "$1")" + local out_path="$(remove_trailing_slash "$2")" + local override="$3" + local osname="$(get_current_os_name)" + local override_switch=$( + if [ "$override" = false ]; then + if [ "$osname" = "linux-musl" ]; then + printf -- "-u"; + else + printf -- "-n"; + fi + fi) + + cat | uniq | while read -r file_path; do + local path="$(remove_beginning_slash "${file_path#$root_path}")" + local target="$out_path/$path" + if [ "$override" = true ] || (! ([ -d "$target" ] || [ -e "$target" ])); then + mkdir -p "$out_path/$(dirname "$path")" + if [ -d "$target" ]; then + rm -rf "$target" + fi + cp -R $override_switch "$root_path/$path" "$target" + fi + done +} + +# args: +# zip_uri - $1 +get_remote_file_size() { + local zip_uri="$1" + + if machine_has "curl"; then + file_size=$(curl -sI "$zip_uri" | grep -i content-length | awk '{ num = $2 + 0; print num }') + elif machine_has "wget"; then + file_size=$(wget --spider --server-response -O /dev/null "$zip_uri" 2>&1 | grep -i 'Content-Length:' | awk '{ num = $2 + 0; print num }') + else + say "Neither curl nor wget is available on this system." + return + fi + + if [ -n "$file_size" ]; then + say "Remote file $zip_uri size is $file_size bytes." + echo "$file_size" + else + say_verbose "Content-Length header was not extracted for $zip_uri." + echo "" + fi +} + +# args: +# zip_path - $1 +# out_path - $2 +# remote_file_size - $3 +extract_dotnet_package() { + eval $invocation + + local zip_path="$1" + local out_path="$2" + local remote_file_size="$3" + + local temp_out_path="$(mktemp -d "$temporary_file_template")" + + local failed=false + tar -xzf "$zip_path" -C "$temp_out_path" > /dev/null || failed=true + + local folders_with_version_regex='^.*/[0-9]+\.[0-9]+[^/]+/' + find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false + find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files" + + validate_remote_local_file_sizes "$zip_path" "$remote_file_size" + + rm -rf "$temp_out_path" + if [ -z ${keep_zip+x} ]; then + rm -f "$zip_path" && say_verbose "Temporary zip file $zip_path was removed" + fi + + if [ "$failed" = true ]; then + say_err "Extraction failed" + return 1 + fi + return 0 +} + +# args: +# remote_path - $1 +# disable_feed_credential - $2 +get_http_header() +{ + eval $invocation + local remote_path="$1" + local disable_feed_credential="$2" + + local failed=false + local response + if machine_has "curl"; then + get_http_header_curl $remote_path $disable_feed_credential || failed=true + elif machine_has "wget"; then + get_http_header_wget $remote_path $disable_feed_credential || failed=true + else + failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Failed to get HTTP header: '$remote_path'." + return 1 + fi + return 0 +} + +# args: +# remote_path - $1 +# disable_feed_credential - $2 +get_http_header_curl() { + eval $invocation + local remote_path="$1" + local disable_feed_credential="$2" + + remote_path_with_credential="$remote_path" + if [ "$disable_feed_credential" = false ]; then + remote_path_with_credential+="$feed_credential" + fi + + curl_options="-I -sSL --retry 5 --retry-delay 2 --connect-timeout 15 " + curl $curl_options "$remote_path_with_credential" 2>&1 || return 1 + return 0 +} + +# args: +# remote_path - $1 +# disable_feed_credential - $2 +get_http_header_wget() { + eval $invocation + local remote_path="$1" + local disable_feed_credential="$2" + local wget_options="-q -S --spider --tries 5 " + + local wget_options_extra='' + + # Test for options that aren't supported on all wget implementations. + if [[ $(wget -h 2>&1 | grep -E 'waitretry|connect-timeout') ]]; then + wget_options_extra="--waitretry 2 --connect-timeout 15 " + else + say "wget extra options are unavailable for this environment" + fi + + remote_path_with_credential="$remote_path" + if [ "$disable_feed_credential" = false ]; then + remote_path_with_credential+="$feed_credential" + fi + + wget $wget_options $wget_options_extra "$remote_path_with_credential" 2>&1 + + return $? +} + +# args: +# remote_path - $1 +# [out_path] - $2 - stdout if not provided +download() { + eval $invocation + + local remote_path="$1" + local out_path="${2:-}" + + if [[ "$remote_path" != "http"* ]]; then + cp "$remote_path" "$out_path" + return $? + fi + + local failed=false + local attempts=0 + while [ $attempts -lt 3 ]; do + attempts=$((attempts+1)) + failed=false + if machine_has "curl"; then + downloadcurl "$remote_path" "$out_path" || failed=true + elif machine_has "wget"; then + downloadwget "$remote_path" "$out_path" || failed=true + else + say_err "Missing dependency: neither curl nor wget was found." + exit 1 + fi + + if [ "$failed" = false ] || [ $attempts -ge 3 ] || { [ ! -z $http_code ] && [ $http_code = "404" ]; }; then + break + fi + + say "Download attempt #$attempts has failed: $http_code $download_error_msg" + say "Attempt #$((attempts+1)) will start in $((attempts*10)) seconds." + sleep $((attempts*10)) + done + + if [ "$failed" = true ]; then + say_verbose "Download failed: $remote_path" + return 1 + fi + return 0 +} + +# Updates global variables $http_code and $download_error_msg +downloadcurl() { + eval $invocation + unset http_code + unset download_error_msg + local remote_path="$1" + local out_path="${2:-}" + # Append feed_credential as late as possible before calling curl to avoid logging feed_credential + # Avoid passing URI with credentials to functions: note, most of them echoing parameters of invocation in verbose output. + local remote_path_with_credential="${remote_path}${feed_credential}" + local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " + local curl_exit_code=0; + if [ -z "$out_path" ]; then + curl $curl_options "$remote_path_with_credential" 2>&1 + curl_exit_code=$? + else + curl $curl_options -o "$out_path" "$remote_path_with_credential" 2>&1 + curl_exit_code=$? + fi + + if [ $curl_exit_code -gt 0 ]; then + download_error_msg="Unable to download $remote_path." + # Check for curl timeout codes + if [[ $curl_exit_code == 7 || $curl_exit_code == 28 ]]; then + download_error_msg+=" Failed to reach the server: connection timeout." + else + local disable_feed_credential=false + local response=$(get_http_header_curl $remote_path $disable_feed_credential) + http_code=$( echo "$response" | awk '/^HTTP/{print $2}' | tail -1 ) + if [[ ! -z $http_code && $http_code != 2* ]]; then + download_error_msg+=" Returned HTTP status code: $http_code." + fi + fi + say_verbose "$download_error_msg" + return 1 + fi + return 0 +} + + +# Updates global variables $http_code and $download_error_msg +downloadwget() { + eval $invocation + unset http_code + unset download_error_msg + local remote_path="$1" + local out_path="${2:-}" + # Append feed_credential as late as possible before calling wget to avoid logging feed_credential + local remote_path_with_credential="${remote_path}${feed_credential}" + local wget_options="--tries 20 " + + local wget_options_extra='' + local wget_result='' + + # Test for options that aren't supported on all wget implementations. + if [[ $(wget -h 2>&1 | grep -E 'waitretry|connect-timeout') ]]; then + wget_options_extra="--waitretry 2 --connect-timeout 15 " + else + say "wget extra options are unavailable for this environment" + fi + + if [ -z "$out_path" ]; then + wget -q $wget_options $wget_options_extra -O - "$remote_path_with_credential" 2>&1 + wget_result=$? + else + wget $wget_options $wget_options_extra -O "$out_path" "$remote_path_with_credential" 2>&1 + wget_result=$? + fi + + if [[ $wget_result != 0 ]]; then + local disable_feed_credential=false + local response=$(get_http_header_wget $remote_path $disable_feed_credential) + http_code=$( echo "$response" | awk '/^ HTTP/{print $2}' | tail -1 ) + download_error_msg="Unable to download $remote_path." + if [[ ! -z $http_code && $http_code != 2* ]]; then + download_error_msg+=" Returned HTTP status code: $http_code." + # wget exit code 4 stands for network-issue + elif [[ $wget_result == 4 ]]; then + download_error_msg+=" Failed to reach the server: connection timeout." + fi + say_verbose "$download_error_msg" + return 1 + fi + + return 0 +} + +get_download_link_from_aka_ms() { + eval $invocation + + #quality is not supported for LTS or STS channel + #STS maps to current + if [[ ! -z "$normalized_quality" && ("$normalized_channel" == "LTS" || "$normalized_channel" == "STS") ]]; then + normalized_quality="" + say_warning "Specifying quality for STS or LTS channel is not supported, the quality will be ignored." + fi + + say_verbose "Retrieving primary payload URL from aka.ms for channel: '$normalized_channel', quality: '$normalized_quality', product: '$normalized_product', os: '$normalized_os', architecture: '$normalized_architecture'." + + #construct aka.ms link + aka_ms_link="https://aka.ms/dotnet" + if [ "$internal" = true ]; then + aka_ms_link="$aka_ms_link/internal" + fi + aka_ms_link="$aka_ms_link/$normalized_channel" + if [[ ! -z "$normalized_quality" ]]; then + aka_ms_link="$aka_ms_link/$normalized_quality" + fi + aka_ms_link="$aka_ms_link/$normalized_product-$normalized_os-$normalized_architecture.tar.gz" + say_verbose "Constructed aka.ms link: '$aka_ms_link'." + + #get HTTP response + #do not pass credentials as a part of the $aka_ms_link and do not apply credentials in the get_http_header function + #otherwise the redirect link would have credentials as well + #it would result in applying credentials twice to the resulting link and thus breaking it, and in echoing credentials to the output as a part of redirect link + disable_feed_credential=true + response="$(get_http_header $aka_ms_link $disable_feed_credential)" + + say_verbose "Received response: $response" + # Get results of all the redirects. + http_codes=$( echo "$response" | awk '$1 ~ /^HTTP/ {print $2}' ) + # They all need to be 301, otherwise some links are broken (except for the last, which is not a redirect but 200 or 404). + broken_redirects=$( echo "$http_codes" | sed '$d' | grep -v '301' ) + + # All HTTP codes are 301 (Moved Permanently), the redirect link exists. + if [[ -z "$broken_redirects" ]]; then + aka_ms_download_link=$( echo "$response" | awk '$1 ~ /^Location/{print $2}' | tail -1 | tr -d '\r') + + if [[ -z "$aka_ms_download_link" ]]; then + say_verbose "The aka.ms link '$aka_ms_link' is not valid: failed to get redirect location." + return 1 + fi + + say_verbose "The redirect location retrieved: '$aka_ms_download_link'." + return 0 + else + say_verbose "The aka.ms link '$aka_ms_link' is not valid: received HTTP code: $(echo "$broken_redirects" | paste -sd "," -)." + return 1 + fi +} + +get_feeds_to_use() +{ + feeds=( + "https://dotnetcli.azureedge.net/dotnet" + "https://dotnetbuilds.azureedge.net/public" + ) + + if [[ -n "$azure_feed" ]]; then + feeds=("$azure_feed") + fi + + if [[ "$no_cdn" == "true" ]]; then + feeds=( + "https://dotnetcli.blob.core.windows.net/dotnet" + "https://dotnetbuilds.blob.core.windows.net/public" + ) + + if [[ -n "$uncached_feed" ]]; then + feeds=("$uncached_feed") + fi + fi +} + +# THIS FUNCTION MAY EXIT (if the determined version is already installed). +generate_download_links() { + + download_links=() + specific_versions=() + effective_versions=() + link_types=() + + # If generate_akams_links returns false, no fallback to old links. Just terminate. + # This function may also 'exit' (if the determined version is already installed). + generate_akams_links || return + + # Check other feeds only if we haven't been able to find an aka.ms link. + if [[ "${#download_links[@]}" -lt 1 ]]; then + for feed in ${feeds[@]} + do + # generate_regular_links may also 'exit' (if the determined version is already installed). + generate_regular_links $feed || return + done + fi + + if [[ "${#download_links[@]}" -eq 0 ]]; then + say_err "Failed to resolve the exact version number." + return 1 + fi + + say_verbose "Generated ${#download_links[@]} links." + for link_index in ${!download_links[@]} + do + say_verbose "Link $link_index: ${link_types[$link_index]}, ${effective_versions[$link_index]}, ${download_links[$link_index]}" + done +} + +# THIS FUNCTION MAY EXIT (if the determined version is already installed). +generate_akams_links() { + local valid_aka_ms_link=true; + + normalized_version="$(to_lowercase "$version")" + if [[ "$normalized_version" != "latest" ]] && [ -n "$normalized_quality" ]; then + say_err "Quality and Version options are not allowed to be specified simultaneously. See https://learn.microsoft.com/dotnet/core/tools/dotnet-install-script#options for details." + return 1 + fi + + if [[ -n "$json_file" || "$normalized_version" != "latest" ]]; then + # aka.ms links are not needed when exact version is specified via command or json file + return + fi + + get_download_link_from_aka_ms || valid_aka_ms_link=false + + if [[ "$valid_aka_ms_link" == true ]]; then + say_verbose "Retrieved primary payload URL from aka.ms link: '$aka_ms_download_link'." + say_verbose "Downloading using legacy url will not be attempted." + + download_link=$aka_ms_download_link + + #get version from the path + IFS='/' + read -ra pathElems <<< "$download_link" + count=${#pathElems[@]} + specific_version="${pathElems[count-2]}" + unset IFS; + say_verbose "Version: '$specific_version'." + + #Retrieve effective version + effective_version="$(get_specific_product_version "$azure_feed" "$specific_version" "$download_link")" + + # Add link info to arrays + download_links+=($download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+=("aka.ms") + + # Check if the SDK version is already installed. + if [[ "$dry_run" != true ]] && is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "$asset_name with version '$effective_version' is already installed." + exit 0 + fi + + return 0 + fi + + # if quality is specified - exit with error - there is no fallback approach + if [ ! -z "$normalized_quality" ]; then + say_err "Failed to locate the latest version in the channel '$normalized_channel' with '$normalized_quality' quality for '$normalized_product', os: '$normalized_os', architecture: '$normalized_architecture'." + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support." + return 1 + fi + say_verbose "Falling back to latest.version file approach." +} + +# THIS FUNCTION MAY EXIT (if the determined version is already installed) +# args: +# feed - $1 +generate_regular_links() { + local feed="$1" + local valid_legacy_download_link=true + + specific_version=$(get_specific_version_from_version "$feed" "$channel" "$normalized_architecture" "$version" "$json_file") || specific_version='0' + + if [[ "$specific_version" == '0' ]]; then + say_verbose "Failed to resolve the specific version number using feed '$feed'" + return + fi + + effective_version="$(get_specific_product_version "$feed" "$specific_version")" + say_verbose "specific_version=$specific_version" + + download_link="$(construct_download_link "$feed" "$channel" "$normalized_architecture" "$specific_version" "$normalized_os")" + say_verbose "Constructed primary named payload URL: $download_link" + + # Add link info to arrays + download_links+=($download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+=("primary") + + legacy_download_link="$(construct_legacy_download_link "$feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false + + if [ "$valid_legacy_download_link" = true ]; then + say_verbose "Constructed legacy named payload URL: $legacy_download_link" + + download_links+=($legacy_download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+=("legacy") + else + legacy_download_link="" + say_verbose "Cound not construct a legacy_download_link; omitting..." + fi + + # Check if the SDK version is already installed. + if [[ "$dry_run" != true ]] && is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "$asset_name with version '$effective_version' is already installed." + exit 0 + fi +} + +print_dry_run() { + + say "Payload URLs:" + + for link_index in "${!download_links[@]}" + do + say "URL #$link_index - ${link_types[$link_index]}: ${download_links[$link_index]}" + done + + resolved_version=${specific_versions[0]} + repeatable_command="./$script_name --version "\""$resolved_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\"" --os "\""$normalized_os"\""" + + if [ ! -z "$normalized_quality" ]; then + repeatable_command+=" --quality "\""$normalized_quality"\""" + fi + + if [[ "$runtime" == "dotnet" ]]; then + repeatable_command+=" --runtime "\""dotnet"\""" + elif [[ "$runtime" == "aspnetcore" ]]; then + repeatable_command+=" --runtime "\""aspnetcore"\""" + fi + + repeatable_command+="$non_dynamic_parameters" + + if [ -n "$feed_credential" ]; then + repeatable_command+=" --feed-credential "\"""\""" + fi + + say "Repeatable invocation: $repeatable_command" +} + +calculate_vars() { + eval $invocation + + script_name=$(basename "$0") + normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" + say_verbose "Normalized architecture: '$normalized_architecture'." + normalized_os="$(get_normalized_os "$user_defined_os")" + say_verbose "Normalized OS: '$normalized_os'." + normalized_quality="$(get_normalized_quality "$quality")" + say_verbose "Normalized quality: '$normalized_quality'." + normalized_channel="$(get_normalized_channel "$channel")" + say_verbose "Normalized channel: '$normalized_channel'." + normalized_product="$(get_normalized_product "$runtime")" + say_verbose "Normalized product: '$normalized_product'." + install_root="$(resolve_installation_path "$install_dir")" + say_verbose "InstallRoot: '$install_root'." + + normalized_architecture="$(get_normalized_architecture_for_specific_sdk_version "$version" "$normalized_channel" "$normalized_architecture")" + + if [[ "$runtime" == "dotnet" ]]; then + asset_relative_path="shared/Microsoft.NETCore.App" + asset_name=".NET Core Runtime" + elif [[ "$runtime" == "aspnetcore" ]]; then + asset_relative_path="shared/Microsoft.AspNetCore.App" + asset_name="ASP.NET Core Runtime" + elif [ -z "$runtime" ]; then + asset_relative_path="sdk" + asset_name=".NET Core SDK" + fi + + get_feeds_to_use +} + +install_dotnet() { + eval $invocation + local download_failed=false + local download_completed=false + local remote_file_size=0 + + mkdir -p "$install_root" + zip_path="${zip_path:-$(mktemp "$temporary_file_template")}" + say_verbose "Zip path: $zip_path" + + for link_index in "${!download_links[@]}" + do + download_link="${download_links[$link_index]}" + specific_version="${specific_versions[$link_index]}" + effective_version="${effective_versions[$link_index]}" + link_type="${link_types[$link_index]}" + + say "Attempting to download using $link_type link $download_link" + + # The download function will set variables $http_code and $download_error_msg in case of failure. + download_failed=false + download "$download_link" "$zip_path" 2>&1 || download_failed=true + + if [ "$download_failed" = true ]; then + case $http_code in + 404) + say "The resource at $link_type link '$download_link' is not available." + ;; + *) + say "Failed to download $link_type link '$download_link': $download_error_msg" + ;; + esac + rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" + else + download_completed=true + break + fi + done + + if [[ "$download_completed" == false ]]; then + say_err "Could not find \`$asset_name\` with version = $specific_version" + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" + return 1 + fi + + remote_file_size="$(get_remote_file_size "$download_link")" + + say "Extracting zip from $download_link" + extract_dotnet_package "$zip_path" "$install_root" "$remote_file_size" || return 1 + + # Check if the SDK version is installed; if not, fail the installation. + # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. + if [[ $specific_version == *"rtm"* || $specific_version == *"servicing"* ]]; then + IFS='-' + read -ra verArr <<< "$specific_version" + release_version="${verArr[0]}" + unset IFS; + say_verbose "Checking installation: version = $release_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$release_version"; then + say "Installed version is $effective_version" + return 0 + fi + fi + + # Check if the standard SDK version is installed. + say_verbose "Checking installation: version = $effective_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "Installed version is $effective_version" + return 0 + fi + + # Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm. + say_err "Failed to verify the version of installed \`$asset_name\`.\nInstallation source: $download_link.\nInstallation location: $install_root.\nReport the bug at https://github.com/dotnet/install-scripts/issues." + say_err "\`$asset_name\` with version = $effective_version failed to install with an error." + return 1 +} + +args=("$@") + +local_version_file_relative_path="/.version" +bin_folder_relative_path="" +temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX" + +channel="LTS" +version="Latest" +json_file="" +install_dir="" +architecture="" +dry_run=false +no_path=false +no_cdn=false +azure_feed="" +uncached_feed="" +feed_credential="" +verbose=false +runtime="" +runtime_id="" +quality="" +internal=false +override_non_versioned_files=true +non_dynamic_parameters="" +user_defined_os="" + +while [ $# -ne 0 ] +do + name="$1" + case "$name" in + -c|--channel|-[Cc]hannel) + shift + channel="$1" + ;; + -v|--version|-[Vv]ersion) + shift + version="$1" + ;; + -q|--quality|-[Qq]uality) + shift + quality="$1" + ;; + --internal|-[Ii]nternal) + internal=true + non_dynamic_parameters+=" $name" + ;; + -i|--install-dir|-[Ii]nstall[Dd]ir) + shift + install_dir="$1" + ;; + --arch|--architecture|-[Aa]rch|-[Aa]rchitecture) + shift + architecture="$1" + ;; + --os|-[Oo][SS]) + shift + user_defined_os="$1" + ;; + --shared-runtime|-[Ss]hared[Rr]untime) + say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'." + if [ -z "$runtime" ]; then + runtime="dotnet" + fi + ;; + --runtime|-[Rr]untime) + shift + runtime="$1" + if [[ "$runtime" != "dotnet" ]] && [[ "$runtime" != "aspnetcore" ]]; then + say_err "Unsupported value for --runtime: '$1'. Valid values are 'dotnet' and 'aspnetcore'." + if [[ "$runtime" == "windowsdesktop" ]]; then + say_err "WindowsDesktop archives are manufactured for Windows platforms only." + fi + exit 1 + fi + ;; + --dry-run|-[Dd]ry[Rr]un) + dry_run=true + ;; + --no-path|-[Nn]o[Pp]ath) + no_path=true + non_dynamic_parameters+=" $name" + ;; + --verbose|-[Vv]erbose) + verbose=true + non_dynamic_parameters+=" $name" + ;; + --no-cdn|-[Nn]o[Cc]dn) + no_cdn=true + non_dynamic_parameters+=" $name" + ;; + --azure-feed|-[Aa]zure[Ff]eed) + shift + azure_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --uncached-feed|-[Uu]ncached[Ff]eed) + shift + uncached_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --feed-credential|-[Ff]eed[Cc]redential) + shift + feed_credential="$1" + #feed_credential should start with "?", for it to be added to the end of the link. + #adding "?" at the beginning of the feed_credential if needed. + [[ -z "$(echo $feed_credential)" ]] || [[ $feed_credential == \?* ]] || feed_credential="?$feed_credential" + ;; + --runtime-id|-[Rr]untime[Ii]d) + shift + runtime_id="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + say_warning "Use of --runtime-id is obsolete and should be limited to the versions below 2.1. To override architecture, use --architecture option instead. To override OS, use --os option instead." + ;; + --jsonfile|-[Jj][Ss]on[Ff]ile) + shift + json_file="$1" + ;; + --skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles) + override_non_versioned_files=false + non_dynamic_parameters+=" $name" + ;; + --keep-zip|-[Kk]eep[Zz]ip) + keep_zip=true + non_dynamic_parameters+=" $name" + ;; + --zip-path|-[Zz]ip[Pp]ath) + shift + zip_path="$1" + ;; + -?|--?|-h|--help|-[Hh]elp) + script_name="$(basename "$0")" + echo ".NET Tools Installer" + echo "Usage:" + echo " # Install a .NET SDK of a given Quality from a given Channel" + echo " $script_name [-c|--channel ] [-q|--quality ]" + echo " # Install a .NET SDK of a specific public version" + echo " $script_name [-v|--version ]" + echo " $script_name -h|-?|--help" + echo "" + echo "$script_name is a simple command line interface for obtaining dotnet cli." + echo " Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" + echo " - The SDK needs to be installed without user interaction and without admin rights." + echo " - The SDK installation doesn't need to persist across multiple CI runs." + echo " To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer." + echo "" + echo "Options:" + echo " -c,--channel Download from the channel specified, Defaults to \`$channel\`." + echo " -Channel" + echo " Possible values:" + echo " - STS - the most recent Standard Term Support release" + echo " - LTS - the most recent Long Term Support release" + echo " - 2-part version in a format A.B - represents a specific release" + echo " examples: 2.0; 1.0" + echo " - 3-part version in a format A.B.Cxx - represents a specific SDK release" + echo " examples: 5.0.1xx, 5.0.2xx." + echo " Supported since 5.0 release" + echo " Warning: Value 'Current' is deprecated for the Channel parameter. Use 'STS' instead." + echo " Note: The version parameter overrides the channel parameter when any version other than 'latest' is used." + echo " -v,--version Use specific VERSION, Defaults to \`$version\`." + echo " -Version" + echo " Possible values:" + echo " - latest - the latest build on specific channel" + echo " - 3-part version in a format A.B.C - represents specific version of build" + echo " examples: 2.0.0-preview2-006120; 1.1.0" + echo " -q,--quality Download the latest build of specified quality in the channel." + echo " -Quality" + echo " The possible values are: daily, signed, validated, preview, GA." + echo " Works only in combination with channel. Not applicable for STS and LTS channels and will be ignored if those channels are used." + echo " For SDK use channel in A.B.Cxx format. Using quality for SDK together with channel in A.B format is not supported." + echo " Supported since 5.0 release." + echo " Note: The version parameter overrides the channel parameter when any version other than 'latest' is used, and therefore overrides the quality." + echo " --internal,-Internal Download internal builds. Requires providing credentials via --feed-credential parameter." + echo " --feed-credential Token to access Azure feed. Used as a query string to append to the Azure feed." + echo " -FeedCredential This parameter typically is not specified." + echo " -i,--install-dir Install under specified location (see Install Location below)" + echo " -InstallDir" + echo " --architecture Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`." + echo " --arch,-Architecture,-Arch" + echo " Possible values: x64, arm, arm64, s390x, ppc64le and loongarch64" + echo " --os Specifies operating system to be used when selecting the installer." + echo " Overrides the OS determination approach used by the script. Supported values: osx, linux, linux-musl, freebsd, rhel.6." + echo " In case any other value is provided, the platform will be determined by the script based on machine configuration." + echo " Not supported for legacy links. Use --runtime-id to specify platform for legacy links." + echo " Refer to: https://aka.ms/dotnet-os-lifecycle for more information." + echo " --runtime Installs a shared runtime only, without the SDK." + echo " -Runtime" + echo " Possible values:" + echo " - dotnet - the Microsoft.NETCore.App shared runtime" + echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime" + echo " --dry-run,-DryRun Do not perform installation. Display download link." + echo " --no-path, -NoPath Do not set PATH for the current process." + echo " --verbose,-Verbose Display diagnostics information." + echo " --azure-feed,-AzureFeed For internal use only." + echo " Allows using a different storage to download SDK archives from." + echo " This parameter is only used if --no-cdn is false." + echo " --uncached-feed,-UncachedFeed For internal use only." + echo " Allows using a different storage to download SDK archives from." + echo " This parameter is only used if --no-cdn is true." + echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable." + echo " -SkipNonVersionedFiles" + echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." + echo " --jsonfile Determines the SDK version from a user specified global.json file." + echo " Note: global.json must have a value for 'SDK:Version'" + echo " --keep-zip,-KeepZip If set, downloaded file is kept." + echo " --zip-path, -ZipPath If set, downloaded file is stored at the specified path." + echo " -?,--?,-h,--help,-Help Shows this help message" + echo "" + echo "Install Location:" + echo " Location is chosen in following order:" + echo " - --install-dir option" + echo " - Environmental variable DOTNET_INSTALL_DIR" + echo " - $HOME/.dotnet" + exit 0 + ;; + *) + say_err "Unknown argument \`$name\`" + exit 1 + ;; + esac + + shift +done + +say_verbose "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" +say_verbose "- The SDK needs to be installed without user interaction and without admin rights." +say_verbose "- The SDK installation doesn't need to persist across multiple CI runs." +say_verbose "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.\n" + +if [ "$internal" = true ] && [ -z "$(echo $feed_credential)" ]; then + message="Provide credentials via --feed-credential parameter." + if [ "$dry_run" = true ]; then + say_warning "$message" + else + say_err "$message" + exit 1 + fi +fi + +check_min_reqs +calculate_vars +# generate_regular_links call below will 'exit' if the determined version is already installed. +generate_download_links + +if [[ "$dry_run" = true ]]; then + print_dry_run + exit 0 +fi + +install_dotnet + +bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")" +if [ "$no_path" = false ]; then + say "Adding to current process PATH: \`$bin_path\`. Note: This change will be visible only when sourcing script." + export PATH="$bin_path":"$PATH" +else + say "Binaries of dotnet can be found in $bin_path" +fi + +say "Note that the script does not resolve dependencies during installation." +say "To check the list of dependencies, go to https://learn.microsoft.com/dotnet/core/install, select your operating system and check the \"Dependencies\" section." +say "Installation finished successfully." From 4db22d38c6f04996de99ff7062555e0eb2f66b9c Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 27 Feb 2024 12:49:38 -0800 Subject: [PATCH 047/285] Docker-in-docker: Updates default value for "dockerDashComposeVersion" to "latest (#873) * Set v2 by default ; draft * Fix edge case: compose switch * Update description * validate "compose-switch" installation * fix test --- .../devcontainer-feature.json | 13 +- src/docker-in-docker/install.sh | 113 +++++++++--------- test/docker-in-docker/docker_build.sh | 4 +- test/docker-in-docker/docker_compose_v1.sh | 13 ++ test/docker-in-docker/docker_compose_v2.sh | 16 +++ test/docker-in-docker/scenarios.json | 20 ++++ 6 files changed, 119 insertions(+), 60 deletions(-) create mode 100755 test/docker-in-docker/docker_compose_v1.sh create mode 100755 test/docker-in-docker/docker_compose_v2.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index d74f205a1..ab26a5e11 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.9.2", + "version": "2.10.0", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", @@ -29,11 +29,11 @@ "type": "string", "enum": [ "none", - "v1", + "latest", "v2" ], - "default": "v1", - "description": "Default version of Docker Compose (v1 or v2 or none)" + "default": "latest", + "description": "Default version of Docker Compose (latest, v2 or none)" }, "azureDnsAutoDetection": { "type": "boolean", @@ -50,6 +50,11 @@ "type": "boolean", "default": true, "description": "Install Docker Buildx" + }, + "installDockerComposeSwitch": { + "type": "boolean", + "default": true, + "description": "Install Compose Switch (provided docker compose is available) which is a replacement to the Compose V1 docker-compose (python) executable. It translates the command line into Compose V2 docker compose then runs the latter." } }, "entrypoint": "/usr/local/share/docker-init.sh", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 9f0e18cee..2a27f5c9c 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -11,11 +11,12 @@ DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version USE_MOBY="${MOBY:-"true"}" MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" -DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none +DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"latest"}" #latest, v2 or none AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL:-""}" USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" +INSTALL_DOCKER_COMPOSE_SWITCH="${INSTALLDOCKERCOMPOSESWITCH:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" @@ -247,75 +248,77 @@ fi echo "Finished installing docker / moby!" +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + # If 'docker-compose' command is to be included if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then - # Install Docker Compose if not already installed and is on a supported architecture - if type docker-compose > /dev/null 2>&1; then - echo "Docker Compose v1 already installed." - else - target_compose_arch="${architecture}" - if [ "${target_compose_arch}" = "amd64" ]; then - target_compose_arch="x86_64" - fi - # https://github.com/devcontainers/features/issues/832 - if [ "${target_compose_arch}" != "x86_64" ] && [ "${VERSION_CODENAME}" != "bookworm" ]; then - # Use pip to get a version that runs on this architecture - check_packages python3-minimal python3-pip libffi-dev python3-venv - export PIPX_HOME=/usr/local/pipx - mkdir -p ${PIPX_HOME} - export PIPX_BIN_DIR=/usr/local/bin - export PYTHONUSERBASE=/tmp/pip-tmp - export PIP_CACHE_DIR=/tmp/pip-tmp/cache - pipx_bin=pipx - if ! type pipx > /dev/null 2>&1; then - pip3 install --disable-pip-version-check --no-cache-dir --user pipx - pipx_bin=/tmp/pip-tmp/bin/pipx - fi - - set +e - ${pipx_bin} install --pip-args '--no-cache-dir --force-reinstall' docker-compose - exit_code=$? - set -e - - if [ ${exit_code} -ne 0 ]; then - # Temporary: https://github.com/devcontainers/features/issues/616 - # See https://github.com/yaml/pyyaml/issues/601 - echo "(*) Failed to install docker-compose via pipx. Trying via pip3..." - - export PYTHONUSERBASE=/usr/local - pip3 install --disable-pip-version-check --no-cache-dir --user "Cython<3.0" pyyaml wheel docker-compose --no-build-isolation - fi + case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 + esac - rm -rf /tmp/pip-tmp + docker_compose_path="/usr/local/bin/docker-compose" + if [ "${DOCKER_DASH_COMPOSE_VERSION}" = "v1" ]; then + err "The final Compose V1 release, version 1.29.2, was May 10, 2021. These packages haven't received any security updates since then. Use at your own risk." + INSTALL_DOCKER_COMPOSE_SWITCH="false" + + if [ "${target_compose_arch}" = "x86_64" ]; then + echo "(*) Installing docker compose v1..." + curl -fsSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64" -o ${docker_compose_path} + chmod +x ${docker_compose_path} + + # Download the SHA256 checksum + DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64.sha256" | awk '{print $1}')" + echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum + sha256sum -c docker-compose.sha256sum --ignore-missing + elif [ "${VERSION_CODENAME}" = "bookworm" ]; then + err "Docker compose v1 is unavailable for 'bookworm' on Arm64. Kindly switch to use v2" + exit 1 else - compose_v1_version="1" - find_version_from_git_tags compose_v1_version "https://github.com/docker/compose" "tags/" - echo "(*) Installing docker-compose ${compose_v1_version}..." - curl -fsSL "https://github.com/docker/compose/releases/download/${compose_v1_version}/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose + # Use pip to get a version that runs on this architecture + check_packages python3-minimal python3-pip libffi-dev python3-venv + echo "(*) Installing docker compose v1 via pip..." + export PYTHONUSERBASE=/usr/local + pip3 install --disable-pip-version-check --no-cache-dir --user "Cython<3.0" pyyaml wheel docker-compose --no-build-isolation fi + else + compose_version=${DOCKER_DASH_COMPOSE_VERSION#v} + find_version_from_git_tags compose_version "https://github.com/docker/compose" "tags/v" + echo "(*) Installing docker-compose ${compose_version}..." + curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} + chmod +x ${docker_compose_path} + + # Download the SHA256 checksum + DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}.sha256" | awk '{print $1}')" + echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum + sha256sum -c docker-compose.sha256sum --ignore-missing + + mkdir -p ${cli_plugins_dir} + cp ${docker_compose_path} ${cli_plugins_dir} fi +fi - # Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation - current_v1_compose_path="$(which docker-compose)" - target_v1_compose_path="$(dirname "${current_v1_compose_path}")/docker-compose-v1" - if ! type compose-switch > /dev/null 2>&1; then +# Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation +if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /dev/null 2>&1; then + if type docker-compose > /dev/null 2>&1; then echo "(*) Installing compose-switch..." + current_compose_path="$(which docker-compose)" + target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" compose_switch_version="latest" find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch chmod +x /usr/local/bin/compose-switch # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 - # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) - mv "${current_v1_compose_path}" "${target_v1_compose_path}" - update-alternatives --install /usr/local/bin/docker-compose docker-compose /usr/local/bin/compose-switch 99 - update-alternatives --install /usr/local/bin/docker-compose docker-compose "${target_v1_compose_path}" 1 - fi - if [ "${DOCKER_DASH_COMPOSE_VERSION}" = "v1" ]; then - update-alternatives --set docker-compose "${target_v1_compose_path}" + mv "${current_compose_path}" "${target_compose_path}" + update-alternatives --install ${docker_compose_path} docker-compose /usr/local/bin/compose-switch 99 + update-alternatives --install ${docker_compose_path} docker-compose "${target_compose_path}" 1 else - update-alternatives --set docker-compose /usr/local/bin/compose-switch + err "Skipping installation of compose-switch as docker compose is unavailable..." fi fi diff --git a/test/docker-in-docker/docker_build.sh b/test/docker-in-docker/docker_build.sh index 7e71be8a1..322a819fc 100755 --- a/test/docker-in-docker/docker_build.sh +++ b/test/docker-in-docker/docker_build.sh @@ -9,8 +9,10 @@ source dev-container-features-test-lib check "docker-buildx" docker buildx version check "docker-build" docker build ./ -check "installs docker-compose v1 install" bash -c "type docker-compose" check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" +check "docker compose" bash -c "docker compose version | grep -E '2.[0-9]+.[0-9]+'" +check "docker-compose" bash -c "docker-compose --version | grep -E '2.[0-9]+.[0-9]+'" + check "docker-buildx" bash -c "docker buildx version" check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" diff --git a/test/docker-in-docker/docker_compose_v1.sh b/test/docker-in-docker/docker_compose_v1.sh new file mode 100755 index 000000000..3f7453c83 --- /dev/null +++ b/test/docker-in-docker/docker_compose_v1.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests + +check "docker-compose" bash -c "docker-compose version | grep -E '1.[0-9]+.[0-9]+'" + +# Report result +reportResults diff --git a/test/docker-in-docker/docker_compose_v2.sh b/test/docker-in-docker/docker_compose_v2.sh new file mode 100755 index 000000000..5a512d2c5 --- /dev/null +++ b/test/docker-in-docker/docker_compose_v2.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests + +check "docker compose" bash -c "docker compose version | grep -E '2.[0-9]+.[0-9]+'" +check "docker-compose" bash -c "docker-compose --version | grep -E '2.[0-9]+.[0-9]+'" +check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" +check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index ccf57b188..e209f3453 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -87,6 +87,26 @@ } } }, + "docker_compose_v1": { + "image": "mcr.microsoft.com/devcontainers/base:focal", + "features": { + "docker-in-docker": { + "moby": true, + "installDockerBuildx": true, + "dockerDashComposeVersion": "v1" + } + } + }, + "docker_compose_v2": { + "image": "mcr.microsoft.com/devcontainers/base:focal", + "features": { + "docker-in-docker": { + "moby": true, + "installDockerBuildx": true, + "dockerDashComposeVersion": "v2" + } + } + }, "docker_build_fallback_buildx": { "image": "ubuntu:focal", "features": { From 03806caa7b6747489311e1f4d8f912ee456039e6 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:50:57 -0800 Subject: [PATCH 048/285] Automated documentation update (#886) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/docker-in-docker/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/docker-in-docker/README.md b/src/docker-in-docker/README.md index 2ab700118..b913fa45f 100644 --- a/src/docker-in-docker/README.md +++ b/src/docker-in-docker/README.md @@ -18,10 +18,11 @@ Create child containers *inside* a container, independent from the host's docker | version | Select or enter a Docker/Moby Engine version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | | mobyBuildxVersion | Install a specific version of moby-buildx when using Moby | string | latest | -| dockerDashComposeVersion | Default version of Docker Compose (v1 or v2 or none) | string | v1 | +| dockerDashComposeVersion | Default version of Docker Compose (latest, v2 or none) | string | latest | | azureDnsAutoDetection | Allow automatically setting the dockerd DNS server when the installation script detects it is running in Azure | boolean | true | | dockerDefaultAddressPool | Define default address pools for Docker networks. e.g. base=192.168.0.0/16,size=24 | string | - | | installDockerBuildx | Install Docker Buildx | boolean | true | +| installDockerComposeSwitch | Install Compose Switch (provided docker compose is available) which is a replacement to the Compose V1 docker-compose (python) executable. It translates the command line into Compose V2 docker compose then runs the latter. | boolean | true | ## Customizations From 5ca92d26ce59ba17e9c5c67fb265b3694c4f18c3 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 29 Feb 2024 00:25:54 +0530 Subject: [PATCH 049/285] [kubectl-helm-minikube]: Handle failing build with fallback to prev. version for helm (#875) * [kubectl-helm-minikube]: Handle failing build with fallback to prev. version * changes acc. to comments and added tests * only to add fallback mechanism for helm and removed for kubectl & minikube * bump the patch version * changes for last comments by @samruddhikhandale * changes for suggestions --- .../devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 28 +++- .../install_only_helm_fallback.sh | 124 ++++++++++++++++++ test/kubectl-helm-minikube/scenarios.json | 10 ++ 4 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 test/kubectl-helm-minikube/install_only_helm_fallback.sh diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index fa8ea6138..16472205f 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.6", + "version": "1.1.7", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index 5deb377c3..69d4e0237 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -155,6 +155,21 @@ if [ ${KUBECTL_VERSION} != "none" ]; then fi fi +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' +} + +get_helm() { + HELM_VERSION=$1 + helm_filename="helm-${HELM_VERSION}-linux-${architecture}.tar.gz" + tmp_helm_filename="/tmp/helm/${helm_filename}" + curl -sSL "https://get.helm.sh/${helm_filename}" -o "${tmp_helm_filename}" + curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" +} + if [ ${HELM_VERSION} != "none" ]; then # Install Helm, verify signature and checksum echo "Downloading Helm..." @@ -163,10 +178,15 @@ if [ ${HELM_VERSION} != "none" ]; then HELM_VERSION="v${HELM_VERSION}" fi mkdir -p /tmp/helm - helm_filename="helm-${HELM_VERSION}-linux-${architecture}.tar.gz" - tmp_helm_filename="/tmp/helm/${helm_filename}" - curl -sSL "https://get.helm.sh/${helm_filename}" -o "${tmp_helm_filename}" - curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" + get_helm "${HELM_VERSION}" + if grep -q "BlobNotFound" "${tmp_helm_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." + repo_url=https://api.github.com/repos/helm/helm/releases + requested_version=$(get_previous_version "${repo_url}") + echo -e "\nAttempting to install ${requested_version}" + HELM_VERSION=${requested_version} + get_helm "${HELM_VERSION}" + fi export GNUPGHOME="/tmp/helm/gnupg" mkdir -p "${GNUPGHOME}" chmod 700 ${GNUPGHOME} diff --git a/test/kubectl-helm-minikube/install_only_helm_fallback.sh b/test/kubectl-helm-minikube/install_only_helm_fallback.sh new file mode 100644 index 000000000..58ed137de --- /dev/null +++ b/test/kubectl-helm-minikube/install_only_helm_fallback.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -e + +# Optional: Import test library +source dev-container-features-test-lib +HL="\033[1;33m" +N="\033[0;37m" +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by kubectl-helm-minikube feature${N}:" + +set +e + check "helm version" helm version +set -e + +# Function to handle errors +handle_error() { + local exit_code=$? + local line_number=$1 + local command=$2 + echo "Error occurred at line $line_number with exit code $exit_code in command $command" + exit $exit_code +} +trap 'handle_error $LINENO ${BASH_COMMAND%% *}' ERR +echo "This is line $LINENO" + +## Check for fallback version installation instead of latest ( when artifact not found ) +architecture="$(uname -m)" +case $architecture in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture $architecture unsupported"; exit 1 ;; +esac +HELM_SHA256="${HELM_SHA256:-"automatic"}" +HELM_GPG_KEYS_URI="https://raw.githubusercontent.com/helm/helm/main/KEYS" + +repo_url=https://api.github.com/repos/helm/helm/releases + +# Function to fetch the latest version of the plugin +get_latest_version() { + curl -s "$repo_url/latest" | jq -r '.tag_name' +} + +# Function to change the patch number in a semver version +change_patch_number() { + local version="$1" # Input version + local new_patch="$2" # New patch number + # Extract major, minor, and current patch numbers + local major=$(echo "$version" | cut -d. -f1) + local minor=$(echo "$version" | cut -d. -f2) + local current_patch=$(echo "$version" | cut -d. -f3) + # Construct the new version with the updated patch number + local new_version="$major.$minor.$new_patch" + echo "$new_version" +} + +# Function to fetch the previous version of the plugin +get_previous_version() { + # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' +} + +get_helm() { + HELM_VERSION=$1 + helm_filename="helm-${HELM_VERSION}-linux-${architecture}.tar.gz" + tmp_helm_filename="/tmp/helm/${helm_filename}" + sudo curl -sSL "https://get.helm.sh/${helm_filename}" -o "${tmp_helm_filename}" + sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" +} + +latest_version=$(get_latest_version) +NON_EXISTING_PATCH_VERSION="xyz" +HELM_VERSION="$(change_patch_number ${latest_version} ${NON_EXISTING_PATCH_VERSION})" +echo -e "\n๐Ÿ‘‰${HL} Trying to install HELM_VERSION = ${HELM_VERSION}${N}"; +sudo mkdir -p /tmp/helm +get_helm "${HELM_VERSION}" +if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." + requested_version=$(get_previous_version) + echo -e "\nAttempting to install ${requested_version}" + HELM_VERSION=${requested_version} + get_helm "${HELM_VERSION}" +fi +export GNUPGHOME="/tmp/helm/gnupg" +sudo mkdir -p "${GNUPGHOME}" +sudo chmod 700 ${GNUPGHOME} +sudo curl -sSL "${HELM_GPG_KEYS_URI}" -o /tmp/helm/KEYS +sudo echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | sudo tee ${GNUPGHOME}/dirmngr.conf >/dev/null +sudo gpg -q --import "/tmp/helm/KEYS" +if ! sudo gpg --verify "${tmp_helm_filename}.asc" | sudo tee ${GNUPGHOME}/verify.log 2>&1; then + echo "Verification failed!" + sudo cat /tmp/helm/gnupg/verify.log + exit 1 +fi + +if [ "${HELM_SHA256}" = "automatic" ]; then + sudo curl -sSL "https://get.helm.sh/${helm_filename}.sha256" -o "${tmp_helm_filename}.sha256" + sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.sha256.asc" -o "${tmp_helm_filename}.sha256.asc" + if ! sudo gpg --verify "${tmp_helm_filename}.sha256.asc" | sudo tee /tmp/helm/gnupg/verify.log 2>&1; then + echo "Verification failed!" + sudo cat /tmp/helm/gnupg/verify.log + exit 1 + fi + HELM_SHA256="$(sudo cat "${tmp_helm_filename}.sha256")" +fi + +([ "${HELM_SHA256}" = "dev-mode" ] || (sudo echo "${HELM_SHA256} *${tmp_helm_filename}" | sha256sum -c -)) +sudo tar xf "${tmp_helm_filename}" -C /tmp/helm +sudo mv -f "/tmp/helm/linux-${architecture}/helm" /usr/local/bin/ +sudo chmod 0755 /usr/local/bin/helm +sudo rm -rf /tmp/helm +if ! type helm > /dev/null 2>&1; then + echo '(!) Helm installation failed!' + exit 1 +fi + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}:" + +set +e + check "helm version" helm version +set -e + +# Report result +reportResults diff --git a/test/kubectl-helm-minikube/scenarios.json b/test/kubectl-helm-minikube/scenarios.json index 08ff8cc79..b82a9966e 100644 --- a/test/kubectl-helm-minikube/scenarios.json +++ b/test/kubectl-helm-minikube/scenarios.json @@ -8,5 +8,15 @@ "minikube": "none" } } + }, + "install_only_helm_fallback": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "kubectl-helm-minikube": { + "version": "none", + "helm": "latest", + "minikube": "none" + } + } } } From 025cae18394696e5684251fbf72710e523c4bb65 Mon Sep 17 00:00:00 2001 From: Nebula <40148908+nebula-it@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:13:15 -0800 Subject: [PATCH 050/285] Fix profile not downloading (#876) * Fix profile not downloading Since the powershell folder is owned by root, the profile download fails without becoming sudo first. ``` ls -l /opt/microsoft/powershell/ total 0 drwxr-xr-x. 1 root root 22 Feb 23 08:37 7 ``` * Update install.sh * Update install.sh * Fix: Add missing quote * Update install.sh * Update src/powershell/install.sh Co-authored-by: Samruddhi Khandale * Update install.sh * Update patch version in devcontainer-feature.json --------- Co-authored-by: Samruddhi Khandale --- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 5a831f7e3..289bd33f0 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.1", + "version": "1.3.2", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 2c052d7cd..27e5d8a27 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -12,6 +12,8 @@ set -e # Clean up rm -rf /var/lib/apt/lists/* +USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" + POWERSHELL_VERSION=${VERSION:-"latest"} POWERSHELL_MODULES="${MODULES:-""}" POWERSHELL_PROFILE_URL="${PROFILE_URL}" @@ -163,10 +165,26 @@ if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then done fi + # Determine the appropriate non-root user + if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi + elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root + fi # If URL for powershell profile is provided, download it to '/opt/microsoft/powershell/7/profile.ps1' if [ -n "$POWERSHELL_PROFILE_URL" ]; then echo "Downloading PowerShell Profile from: $POWERSHELL_PROFILE_URL" - curl -sSL -o "/opt/microsoft/powershell/7/profile.ps1" "$POWERSHELL_PROFILE_URL" + su ${USERNAME} -c "curl -sSL -o '/opt/microsoft/powershell/7/profile.ps1' '$POWERSHELL_PROFILE_URL'" fi # Clean up From 084df810aed51ae4a3de9b352a57a9890d5e967d Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 4 Mar 2024 11:36:03 -0800 Subject: [PATCH 051/285] Oryx: Clean up additional installation of .NET (#892) --- src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 14 +++++++++----- test/oryx/install_dotnet_and_oryx.sh | 3 +++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index fe13e2fcb..a075096bf 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.0", + "version": "1.3.1", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index f4c547fb5..407406141 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -86,7 +86,6 @@ install_dotnet_with_script() DOTNET_BINARY="dotnet" export PATH="${PATH}:/usr/share/dotnet" - DOTNET_BINARY_INSTALLATION="/usr/share/dotnet/sdk/${version}" } install_dotnet_using_apt() { @@ -146,7 +145,6 @@ usermod -a -G oryx "${USERNAME}" # Required to decide if we want to clean up dotnet later. DOTNET_INSTALLATION_PACKAGE="" -DOTNET_BINARY_INSTALLATION="" DOTNET_BINARY="" if dotnet --version > /dev/null ; then @@ -156,6 +154,7 @@ fi MAJOR_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 1) PATCH_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 3) +PINNED_SDK_VERSION="" # Oryx needs to be built with .NET 8 if [[ "${DOTNET_BINARY}" = "" ]] || [[ $MAJOR_VERSION_ID != "8" ]] || [[ $MAJOR_VERSION_ID = "8" && ${PATCH_VERSION_ID} -ge "101" ]] ; then echo "'dotnet 8' was not detected. Attempting to install .NET 8 to build oryx." @@ -182,7 +181,7 @@ mkdir -p ${ORYX} git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX -if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then +if [[ "${PINNED_SDK_VERSION}" != "" ]]; then cd $GIT_ORYX dotnet new globaljson --sdk-version ${PINNED_SDK_VERSION} fi @@ -234,9 +233,14 @@ if [[ "${DOTNET_INSTALLATION_PACKAGE}" != "" ]]; then apt purge -yq $DOTNET_INSTALLATION_PACKAGE fi -if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then +if [[ "${PINNED_SDK_VERSION}" != "" ]]; then rm -f ${GIT_ORYX}/global.json - rm -rf ${DOTNET_BINARY_INSTALLATION} + rm -rf /usr/share/dotnet/sdk/$PINNED_SDK_VERSION + + # Extract the major, minor version and the first digit of the patch version + MAJOR_MINOR_PATCH1_VERSION=${PINNED_SDK_VERSION%??} + rm -rf /usr/share/dotnet/shared/Microsoft.NETCore.App/$MAJOR_MINOR_PATCH1_VERSION + rm -rf /usr/share/dotnet/shared/Microsoft.AspNetCore.App/$MAJOR_MINOR_PATCH1_VERSION fi diff --git a/test/oryx/install_dotnet_and_oryx.sh b/test/oryx/install_dotnet_and_oryx.sh index da0a9162d..670d7565a 100644 --- a/test/oryx/install_dotnet_and_oryx.sh +++ b/test/oryx/install_dotnet_and_oryx.sh @@ -5,6 +5,9 @@ set -e # Optional: Import test library source dev-container-features-test-lib +# Runtimes are listed twice due to 'Microsoft.NETCore.App' and 'Microsoft.AspNetCore.App' +check "two versions of dotnet runtimes are present" bash -c "[ $(dotnet --list-runtimes | wc -l) -eq 4 ]" + check "Oryx version" oryx --version check "Dotnet is not removed if it is not installed by the Oryx Feature" dotnet --version From 4d2e62e4ed458469fd81b71ad3f4d8e1c5596b6f Mon Sep 17 00:00:00 2001 From: Nebula <40148908+nebula-it@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:07:37 -0800 Subject: [PATCH 052/285] Fix PowerShell profile not loading (#889) * Update devcontainer-feature.json * Fix profile loading * Update devcontainer-feature.json * Update devcontainer-feature.json * Add -E to sudo so it inherits environment --- src/powershell/devcontainer-feature.json | 4 ++-- src/powershell/install.sh | 25 +++++------------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 289bd33f0..86314be64 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.2", + "version": "1.3.3", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", @@ -20,7 +20,7 @@ "default": "", "description": "Optional comma separated list of PowerShell modules to install." }, - "powershellProfileURL ": { + "powershellProfileURL": { "type": "string", "default": "", "description": "Optional (publicly accessible) URL to download PowerShell profile." diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 27e5d8a27..0ce63c95d 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -12,11 +12,9 @@ set -e # Clean up rm -rf /var/lib/apt/lists/* -USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" - POWERSHELL_VERSION=${VERSION:-"latest"} POWERSHELL_MODULES="${MODULES:-""}" -POWERSHELL_PROFILE_URL="${PROFILE_URL}" +POWERSHELL_PROFILE_URL="${POWERSHELLPROFILEURL}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" POWERSHELL_ARCHIVE_ARCHITECTURES="amd64" @@ -165,26 +163,13 @@ if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then done fi - # Determine the appropriate non-root user - if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then - USERNAME="" - POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") - for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do - if id -u ${CURRENT_USER} > /dev/null 2>&1; then - USERNAME=${CURRENT_USER} - break - fi - done - if [ "${USERNAME}" = "" ]; then - USERNAME=root - fi - elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then - USERNAME=root - fi + # If URL for powershell profile is provided, download it to '/opt/microsoft/powershell/7/profile.ps1' if [ -n "$POWERSHELL_PROFILE_URL" ]; then echo "Downloading PowerShell Profile from: $POWERSHELL_PROFILE_URL" - su ${USERNAME} -c "curl -sSL -o '/opt/microsoft/powershell/7/profile.ps1' '$POWERSHELL_PROFILE_URL'" + # Get profile path from currently installed pwsh + profilePath=$(pwsh -noni -c '$PROFILE.AllUsersAllHosts') + sudo -E curl -sSL -o "$profilePath" "$POWERSHELL_PROFILE_URL" fi # Clean up From d9e12f33552f81f52218e1701f69f00e86467bd5 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:18:49 -0800 Subject: [PATCH 053/285] Automated documentation update (#893) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/powershell/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powershell/README.md b/src/powershell/README.md index a44614604..f018778c0 100644 --- a/src/powershell/README.md +++ b/src/powershell/README.md @@ -17,7 +17,7 @@ Installs PowerShell along with needed dependencies. Useful for base Dockerfiles |-----|-----|-----|-----| | version | Select or enter a version of PowerShell. | string | latest | | modules | Optional comma separated list of PowerShell modules to install. | string | - | -| powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | +| powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | ## Customizations From 58ab1a1a6d3bad632c147447b1a9107183c8d43a Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 5 Mar 2024 11:25:53 -0800 Subject: [PATCH 054/285] [Git-Lfs] - Install previous version if current tags are missing artifacts (#894) * [Git-Lfs] - Install previous version if current tags are missing artifacts * fix test * fix debian:23 test --- src/git-lfs/devcontainer-feature.json | 7 ++++- src/git-lfs/install.sh | 37 +++++++++++++++++++++------ test/git-lfs/scenarios.json | 10 ++++++++ test/git-lfs/use_github.sh | 10 ++++++++ 4 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 test/git-lfs/use_github.sh diff --git a/src/git-lfs/devcontainer-feature.json b/src/git-lfs/devcontainer-feature.json index 998b0b50e..ccdd99950 100644 --- a/src/git-lfs/devcontainer-feature.json +++ b/src/git-lfs/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "git-lfs", - "version": "1.1.1", + "version": "1.2.0", "name": "Git Large File Support (LFS)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/git-lfs", "description": "Installs Git Large File Support (Git LFS) along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like git and curl.", @@ -18,6 +18,11 @@ "type": "boolean", "default": true, "description": "Automatically pull LFS files when creating the container. When false, running 'git lfs pull' in the container will have the same effect." + }, + "installDirectlyFromGitHubRelease": { + "type": "boolean", + "default": false, + "description": "Installs 'git-lfs' from GitHub releases instead of package manager feeds" } }, "postCreateCommand": "/usr/local/share/pull-git-lfs-artifacts.sh", diff --git a/src/git-lfs/install.sh b/src/git-lfs/install.sh index 3f9aa2fff..2259cef59 100755 --- a/src/git-lfs/install.sh +++ b/src/git-lfs/install.sh @@ -9,6 +9,7 @@ GIT_LFS_VERSION=${VERSION:-"latest"} AUTO_PULL=${AUTOPULL:="true"} +INSTALL_WITH_GITHUB=${INSTALLDIRECTLYFROMGITHUBRELEASE:="false"} GIT_LFS_ARCHIVE_GPG_KEY_URI="https://packagecloud.io/github/git-lfs/gpgkey" GIT_LFS_ARCHIVE_ARCHITECTURES="amd64 arm64" @@ -130,14 +131,34 @@ install_using_apt() { git-lfs install --skip-repo } +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_from_release() { + git_lfs_filename="git-lfs-linux-${architecture}-v${GIT_LFS_VERSION}.tar.gz" + echo "Looking for release artfact: ${git_lfs_filename}" + curl -sSL -o "${git_lfs_filename}" "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/${git_lfs_filename}" +} + install_using_github() { echo "(*) No apt package for ${VERSION_CODENAME} ${architecture}. Installing manually." mkdir -p /tmp/git-lfs cd /tmp/git-lfs find_version_from_git_tags GIT_LFS_VERSION "https://github.com/git-lfs/git-lfs" - git_lfs_filename="git-lfs-linux-${architecture}-v${GIT_LFS_VERSION}.tar.gz" - echo "Looking for release artfact: ${git_lfs_filename}" - curl -sSL -o "${git_lfs_filename}" "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/${git_lfs_filename}" + install_from_release + + if grep -q "Not Found" "${git_lfs_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for Git lfs v${GIT_LFS_VERSION}..." + repo_url=https://api.github.com/repos/git-lfs/git-lfs/releases + requested_version=$(get_previous_version "${repo_url}") + echo -e "\nAttempting to install ${requested_version}" + GIT_LFS_VERSION=${requested_version#v} + install_from_release + fi + # Verify file curl -sSL -o "sha256sums.asc" "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/sha256sums.asc" receive_gpg_keys GIT_LFS_CHECKSUM_GPG_KEYS @@ -165,7 +186,7 @@ export DEBIAN_FRONTEND=noninteractive # Install git, curl, gpg, dirmngr and debian-archive-keyring if missing . /etc/os-release -check_packages curl ca-certificates gnupg2 dirmngr apt-transport-https +check_packages curl ca-certificates gnupg2 dirmngr apt-transport-https jq if ! type git > /dev/null 2>&1; then check_packages git fi @@ -176,14 +197,14 @@ fi # Install Git LFS echo "Installing Git LFS..." architecture="$(dpkg --print-architecture)" -if [[ "${GIT_LFS_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${GIT_LFS_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then - install_using_apt || use_github="true" +if [[ "${GIT_LFS_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${GIT_LFS_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]] && [[ "${INSTALL_WITH_GITHUB}" = "false" ]]; then + install_using_apt || INSTALL_WITH_GITHUB="true" else - use_github="true" + INSTALL_WITH_GITHUB="true" fi # If no archive exists or apt install fails, try direct from github -if [ "${use_github}" = "true" ]; then +if [ "${INSTALL_WITH_GITHUB}" = "true" ]; then install_using_github fi diff --git a/test/git-lfs/scenarios.json b/test/git-lfs/scenarios.json index 6cc3c24a5..009dfbb64 100644 --- a/test/git-lfs/scenarios.json +++ b/test/git-lfs/scenarios.json @@ -18,5 +18,15 @@ "autoPull": false } } + }, + "use_github": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "remoteUser": "vscode", + "features": { + "git-lfs": { + "version": "latest", + "installDirectlyFromGitHubRelease": true + } + } } } \ No newline at end of file diff --git a/test/git-lfs/use_github.sh b/test/git-lfs/use_github.sh new file mode 100644 index 000000000..458066f77 --- /dev/null +++ b/test/git-lfs/use_github.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "git-lfs" bash -c "git-lfs --version" + +reportResults \ No newline at end of file From 3c3a270a3703bb2b24857399f5e1d65120cb9152 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:33:50 -0800 Subject: [PATCH 055/285] Automated documentation update (#895) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/git-lfs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/git-lfs/README.md b/src/git-lfs/README.md index 9a9a06659..07f671755 100644 --- a/src/git-lfs/README.md +++ b/src/git-lfs/README.md @@ -17,6 +17,7 @@ Installs Git Large File Support (Git LFS) along with needed dependencies. Useful |-----|-----|-----|-----| | version | Select version of Git LFS to install | string | latest | | autoPull | Automatically pull LFS files when creating the container. When false, running 'git lfs pull' in the container will have the same effect. | boolean | true | +| installDirectlyFromGitHubRelease | Installs 'git-lfs' from GitHub releases instead of package manager feeds | boolean | false | From 965e12010f31baea61c7a9389082b19cfe498207 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 5 Mar 2024 17:20:49 -0800 Subject: [PATCH 056/285] [docker] - Install previous version if current tags are missing artifacts (#897) [docker-in-docker] - Install previous version if current tags are missing artifacts --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 31 ++++++++++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index ab26a5e11..671f4e2ed 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.0", + "version": "2.10.1", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 2a27f5c9c..dbe34e2ea 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -109,6 +109,20 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_compose_switch_fallback() { + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") + echo -e "\nAttempting to install ${previous_version}" + compose_switch_version=${previous_version#v} + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +} + ########################################### # Start docker-in-docker installation ########################################### @@ -290,6 +304,15 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then find_version_from_git_tags compose_version "https://github.com/docker/compose" "tags/v" echo "(*) Installing docker-compose ${compose_version}..." curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} + + if grep -q "Not Found" "${docker_compose_path}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose/releases") + echo -e "\nAttempting to install ${previous_version}" + compose_version=${previous_version#v} + curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} + fi + chmod +x ${docker_compose_path} # Download the SHA256 checksum @@ -310,7 +333,7 @@ if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /d target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" compose_switch_version="latest" find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || install_compose_switch_fallback chmod +x /usr/local/bin/compose-switch # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) @@ -337,12 +360,6 @@ fi usermod -aG docker ${USERNAME} -# Function to fetch the version released prior to the latest version -get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects -} - install_previous_version_artifacts() { wget_exit_code=$? if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. From 8af4ce3dea3cdc2ea96062a47b3363b59aa9f1a2 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:44:02 +0530 Subject: [PATCH 057/285] [docker-outside-of-docker] compose-switch can fallback to previous version (#901) * compose-switch fallback previous version * changes required --- .../devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 18 +++++++++- .../docker_build_compose_fallback.sh | 36 +++++++++++++++++++ test/docker-outside-of-docker/scenarios.json | 10 ++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 test/docker-outside-of-docker/docker_build_compose_fallback.sh diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index d4c1447ba..54fca5ec1 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.2", + "version": "1.4.3", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 65424740e..dd39e1f68 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -99,6 +99,22 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Function to fetch the previous version of the plugin +get_previous_version() { + repo_url=$1 + # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + + +install_compose_switch_fallback() { + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") + echo -e "\nAttempting to install ${previous_version}" + compose_switch_version=${previous_version#v} + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose +} + # Ensure apt is in non-interactive to avoid prompts export DEBIAN_FRONTEND=noninteractive @@ -255,7 +271,7 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then echo "(*) Installing compose-switch as docker-compose..." compose_switch_version="latest" find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback chmod +x /usr/local/bin/docker-compose # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 fi diff --git a/test/docker-outside-of-docker/docker_build_compose_fallback.sh b/test/docker-outside-of-docker/docker_build_compose_fallback.sh new file mode 100644 index 000000000..e1c0885e6 --- /dev/null +++ b/test/docker-outside-of-docker/docker_build_compose_fallback.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Optional: Import test library +source dev-container-features-test-lib + +check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" + +# Fetch host/container arch. +architecture="$(dpkg --print-architecture)" + +repo_url="https://api.github.com/repos/docker/compose-switch/releases" + +# Function to fetch the previous version of the plugin +get_previous_version() { + sudo curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects +} + +install_compose_switch_fallback() { + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch ${test_compose_switch_version}..." + previous_version=$(get_previous_version) + echo -e "\nAttempting to install ${previous_version}" + compose_switch_version=${previous_version} + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose +} + +install_compose-switch_as_docker-compose() { + echo "(*) Installing compose-switch as docker-compose..." + test_compose_switch_version="1.2.xyz" + echo -e "\nTesting with $test_compose_switch_version..." + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${test_compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + sudo chmod +x /usr/local/bin/docker-compose +} + +install_compose-switch_as_docker-compose + +check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file diff --git a/test/docker-outside-of-docker/scenarios.json b/test/docker-outside-of-docker/scenarios.json index 61f1ab402..6239a1ada 100644 --- a/test/docker-outside-of-docker/scenarios.json +++ b/test/docker-outside-of-docker/scenarios.json @@ -1,4 +1,14 @@ { + "docker_build_compose_fallback": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu-20.04", + "features": { + "docker-outside-of-docker": { + "moby": false, + "dockerDashComposeVersion": "latest" + } + }, + "containerUser": "vscode" + }, "docker_init_moby": { "image": "mcr.microsoft.com/devcontainers/base:ubuntu-20.04", "features": { From 460684dd971cb341b65d46fa2bd445f352bf75a2 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 13 Mar 2024 02:34:13 +0530 Subject: [PATCH 058/285] [Node] - nvm - fallback to previous version - code fix (#904) * [node] - nvm - added fallback to prev. version - if latest version tag released but binary not found * bumped patch version --- src/node/devcontainer-feature.json | 2 +- src/node/install.sh | 7 +++++-- test/node/nvm_test_fallback.sh | 27 +++++++++++++++++++++++++++ test/node/scenarios.json | 8 ++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 test/node/nvm_test_fallback.sh diff --git a/src/node/devcontainer-feature.json b/src/node/devcontainer-feature.json index a93f75ace..f15ae4a4d 100644 --- a/src/node/devcontainer-feature.json +++ b/src/node/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "node", - "version": "1.4.0", + "version": "1.4.1", "name": "Node.js (via nvm), yarn and pnpm", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/node", "description": "Installs Node.js, nvm, yarn, pnpm, and needed dependencies.", diff --git a/src/node/install.sh b/src/node/install.sh index 493166d84..11a123534 100755 --- a/src/node/install.sh +++ b/src/node/install.sh @@ -293,8 +293,11 @@ set -e umask 0002 # Do not update profile - we'll do this manually export PROFILE=/dev/null -curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh" | bash - +curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh" | bash || { + PREV_NVM_VERSION=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/') + curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/\${PREV_NVM_VERSION}/install.sh" | bash + NVM_VERSION="\${PREV_NVM_VERSION}" +} source "${NVM_DIR}/nvm.sh" if [ "${NODE_VERSION}" != "" ]; then nvm alias default "${NODE_VERSION}" diff --git a/test/node/nvm_test_fallback.sh b/test/node/nvm_test_fallback.sh new file mode 100644 index 000000000..68e4071ac --- /dev/null +++ b/test/node/nvm_test_fallback.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +set -e + +trap 'echo "Error occurred at line $LINENO"; exit 1' ERR +source /usr/local/share/nvm/nvm.sh +#check nvm version +echo -e "\nโœ… nvm version as installed by feature = v$(nvm --version)"; +NVM_DIR="/usr/local/share/nvm" +NODE_VERSION="lts" +FAKE_NVM_VERSION="1.2.XYZ" +curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/v${FAKE_NVM_VERSION}/install.sh" | bash || { + PREV_NVM_VERSION=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/') + curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/${PREV_NVM_VERSION}/install.sh" | bash + NVM_VERSION="${PREV_NVM_VERSION}" +} + +#check nvm version +echo -e "\nโœ… nvm version as installed by test = v$(nvm --version)"; + +# Report result +reportResults diff --git a/test/node/scenarios.json b/test/node/scenarios.json index 262fba8ea..954c2c2e9 100644 --- a/test/node/scenarios.json +++ b/test/node/scenarios.json @@ -1,4 +1,12 @@ { + "nvm_test_fallback": { + "image": "debian:11", + "features": { + "node": { + "version": "lts" + } + } + }, "install_additional_node": { "image": "debian:11", "features": { From a674273c8602df49e03d3f74d7a0ccd5eea07cea Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 13 Mar 2024 02:36:34 +0530 Subject: [PATCH 059/285] [Powershell] - Fallback to prev. version - fix (#906) * [Powershell] - Fallback to prev. version - fix * patch version updated --- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 33 ++++++- .../install_powershell_fallback_test.sh | 97 +++++++++++++++++++ test/powershell/scenarios.json | 9 ++ 4 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 test/powershell/install_powershell_fallback_test.sh diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 86314be64..dd7a96b90 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.3", + "version": "1.3.4", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 0ce63c95d..4ba8bcf5c 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -106,6 +106,29 @@ install_using_apt() { apt-get install -yq powershell${version_suffix} || return 1 } +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_prev_pwsh() { + echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." + previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") + echo -e "\nAttempting to install ${previous_version}" + POWERSHELL_VERSION="${previous_version#v}" + install_pwsh "${POWERSHELL_VERSION}" +} + +install_pwsh() { + POWERSHELL_VERSION=$1 + powershell_filename="powershell-${POWERSHELL_VERSION}-linux-${architecture}.tar.gz" + powershell_target_path="/opt/microsoft/powershell/$(echo ${POWERSHELL_VERSION} | grep -oE '[^\.]+' | head -n 1)" + mkdir -p /tmp/pwsh "${powershell_target_path}" + cd /tmp/pwsh + curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" +} + install_using_github() { # Fall back on direct download if no apt package exists in microsoft pool check_packages curl ca-certificates gnupg2 dirmngr libc6 libgcc1 libgssapi-krb5-2 libstdc++6 libunwind8 libuuid1 zlib1g libicu[0-9][0-9] @@ -116,11 +139,11 @@ install_using_github() { architecture="x64" fi find_version_from_git_tags POWERSHELL_VERSION https://github.com/PowerShell/PowerShell - powershell_filename="powershell-${POWERSHELL_VERSION}-linux-${architecture}.tar.gz" - powershell_target_path="/opt/microsoft/powershell/$(echo ${POWERSHELL_VERSION} | grep -oE '[^\.]+' | head -n 1)" - mkdir -p /tmp/pwsh "${powershell_target_path}" - cd /tmp/pwsh - curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" + install_pwsh "${POWERSHELL_VERSION}" + if grep -q "Not Found" "${powershell_filename}"; then + install_prev_pwsh + fi + # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. curl -sSL -o "release.html" "https://github.com/PowerShell/PowerShell/releases/tag/v${POWERSHELL_VERSION}" powershell_archive_sha256="$(cat release.html | tr '\n' ' ' | sed 's|<[^>]*>||g' | grep -oP "${powershell_filename}\s+\K[0-9a-fA-F]{64}" || echo '')" diff --git a/test/powershell/install_powershell_fallback_test.sh b/test/powershell/install_powershell_fallback_test.sh new file mode 100644 index 000000000..852d7d103 --- /dev/null +++ b/test/powershell/install_powershell_fallback_test.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Extension-specific tests +check "az.resources" pwsh -Command "(Get-Module -ListAvailable -Name Az.Resources).Version.ToString()" +check "az.storage" pwsh -Command "(Get-Module -ListAvailable -Name Az.Storage).Version.ToString()" +check "profile" pwsh -Command "(Get-Variable $env:ProfileLoaded).Value" + +check "Powershell version as installed by feature" bash -c "pwsh --version" + +. /etc/os-release +architecture="$(dpkg --print-architecture)" + +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_prev_pwsh() { + echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." + previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") + echo -e "\nAttempting to install ${previous_version}" + POWERSHELL_VERSION="${previous_version#v}" + install_pwsh "${POWERSHELL_VERSION}" +} + +install_pwsh() { + POWERSHELL_VERSION=$1 + powershell_filename="powershell-${POWERSHELL_VERSION}-linux-${architecture}.tar.gz" + powershell_target_path="/opt/microsoft/powershell/$(echo ${POWERSHELL_VERSION} | grep -oE '[^\.]+' | head -n 1)" + sudo mkdir -p /tmp/pwsh "${powershell_target_path}" + cd /tmp/pwsh + sudo curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + sudo apt-get update -y + fi +} + +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + sudo chmod +x /var/lib/apt/lists/ + sudo mkdir -p /var/lib/apt/lists/partial + sudo chmod +rx /var/lib/dpkg/lock-frontend + apt_get_update + sudo apt-get -y install --no-install-recommends "$@" + fi +} + +install_using_github() { + # Fall back on direct download if no apt package exists in microsoft pool + check_packages curl ca-certificates gnupg2 dirmngr libc6 libgcc1 libgssapi-krb5-2 libstdc++6 libunwind8 libuuid1 zlib1g libicu[0-9][0-9] + if ! type git > /dev/null 2>&1; then + check_packages git + fi + if [ "${architecture}" = "amd64" ]; then + architecture="x64" + fi + + echo -e "\nTrying to install a non-existing version for Powershell..." + + POWERSHELL_VERSION="1.2.XYZ" + install_pwsh "${POWERSHELL_VERSION}" + + if grep -q "Not Found" "${powershell_filename}"; then + install_prev_pwsh + fi + + echo -e "\n" $POWERSHELL_VERSION "=powershell_version\n"; + # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. + sudo curl -sSL -o "release.html" "https://github.com/PowerShell/PowerShell/releases/tag/v${POWERSHELL_VERSION}" + powershell_archive_sha256="$(cat release.html | tr '\n' ' ' | sed 's|<[^>]*>||g' | grep -oP "${powershell_filename}\s+\K[0-9a-fA-F]{64}" || echo '')" + if [ -z "${powershell_archive_sha256}" ]; then + echo "(!) WARNING: Failed to retrieve SHA256 for archive. Skipping validaiton." + else + echo "SHA256: ${powershell_archive_sha256}" + echo "${powershell_archive_sha256} *${powershell_filename}" | sha256sum -c - + fi + sudo tar xf "${powershell_filename}" -C "${powershell_target_path}" + sudo ln -s "${powershell_target_path}/pwsh" /usr/local/bin/pwsh + sudo rm -rf /tmp/pwsh +} + +install_using_github + +check "Powershell version as installed by test" bash -c "pwsh --version" + +# Report result +reportResults diff --git a/test/powershell/scenarios.json b/test/powershell/scenarios.json index b2659d7ad..6a810b0c9 100644 --- a/test/powershell/scenarios.json +++ b/test/powershell/scenarios.json @@ -7,5 +7,14 @@ "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" } } + }, + "install_powershell_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "powershell": { + "modules": "az.resources, az.storage", + "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" + } + } } } From 5ad3f6fbbf63096917622ac3da72266ad377c984 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 14 Mar 2024 03:59:50 +0530 Subject: [PATCH 060/285] [python] - cpython - prev. version fallback - fix (#907) * [python] - cpython - prev. version fallback - fix * according to comments --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 94 ++++- ...tall_cpython_fallback_prev_version_test.sh | 355 ++++++++++++++++++ test/python/scenarios.json | 10 + 4 files changed, 441 insertions(+), 20 deletions(-) create mode 100644 test/python/install_cpython_fallback_prev_version_test.sh diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index c14dd7b97..9f211244f 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.4.1", + "version": "1.4.2", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index fcec73b0c..e8a9d24e1 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -248,6 +248,48 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + + # Use Oryx to install something using a partial version match oryx_install() { local platform=$1 @@ -368,6 +410,29 @@ install_openssl3() { rm -rf /tmp/openssl3 } +install_prev_vers_cpython() { + VERSION=$1 + echo -e "\n(!) Failed to fetch the latest artifacts for cpython ${VERSION}..." + find_prev_version_from_git_tags VERSION https://github.com/python/cpython + echo -e "\nAttempting to install ${VERSION}" + install_cpython "${VERSION}" +} + +install_cpython() { + VERSION=$1 + INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" + if [ -d "${INSTALL_PATH}" ]; then + echo "(!) Python version ${VERSION} already exists." + exit 1 + fi + mkdir -p /tmp/python-src ${INSTALL_PATH} + cd /tmp/python-src + cpython_tgz_filename="Python-${VERSION}.tgz" + cpython_tgz_url="https://www.python.org/ftp/python/${VERSION}/${cpython_tgz_filename}" + echo "Downloading ${cpython_tgz_filename}..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}" "${cpython_tgz_url}" +} + install_from_source() { VERSION=$1 echo "(*) Building Python ${VERSION} from source..." @@ -378,13 +443,6 @@ install_from_source() { # Find version using soft match find_version_from_git_tags VERSION "https://github.com/python/cpython" - INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" - - if [ -d "${INSTALL_PATH}" ]; then - echo "(!) Python version ${VERSION} already exists." - exit 1 - fi - # Some platforms/os versions need modern versions of openssl installed # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand @@ -396,23 +454,21 @@ install_from_source() { ;; esac - # Download tgz of source - mkdir -p /tmp/python-src ${INSTALL_PATH} - cd /tmp/python-src - local tgz_filename="Python-${VERSION}.tgz" - local tgz_url="https://www.python.org/ftp/python/${VERSION}/${tgz_filename}" - echo "Downloading ${tgz_filename}..." - curl -sSL -o "/tmp/python-src/${tgz_filename}" "${tgz_url}" - + install_cpython "${VERSION}" + if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then + if grep -q "404 Not Found" "/tmp/python-src/${cpython_tgz_filename}"; then + install_prev_vers_cpython "${VERSION}" + fi + fi; # Verify signature if [[ ${VERSION_CODENAME} = "centos7" ]] || [[ ${VERSION_CODENAME} = "rhel7" ]]; then receive_gpg_keys_centos7 PYTHON_SOURCE_GPG_KEYS else receive_gpg_keys PYTHON_SOURCE_GPG_KEYS fi - echo "Downloading ${tgz_filename}.asc..." - curl -sSL -o "/tmp/python-src/${tgz_filename}.asc" "${tgz_url}.asc" - gpg --verify "${tgz_filename}.asc" + echo "Downloading ${cpython_tgz_filename}.asc..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}.asc" "${cpython_tgz_url}.asc" + gpg --verify "${cpython_tgz_filename}.asc" # Update min protocol for testing only - https://bugs.python.org/issue41561 if [ -f /etc/pki/tls/openssl.cnf ]; then @@ -424,7 +480,7 @@ install_from_source() { export OPENSSL_CONF=/tmp/python-src/openssl.cnf # Untar and build - tar -xzf "/tmp/python-src/${tgz_filename}" -C "/tmp/python-src" --strip-components=1 + tar -xzf "/tmp/python-src/${cpython_tgz_filename}" -C "/tmp/python-src" --strip-components=1 local config_args="" if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then config_args="${config_args} --enable-optimizations" diff --git a/test/python/install_cpython_fallback_prev_version_test.sh b/test/python/install_cpython_fallback_prev_version_test.sh new file mode 100644 index 000000000..7056dba0c --- /dev/null +++ b/test/python/install_cpython_fallback_prev_version_test.sh @@ -0,0 +1,355 @@ +#!/bin/bash + +# Optional: Import test library +source dev-container-features-test-lib + +check "Python version as installed by Feature" bash -c "python3 -V" + +PYTHON_INSTALL_PATH="/usr/local/python" +OPTIMIZE_BUILD_FROM_SOURCE="false" +OVERRIDE_DEFAULT_VERSION="true" +ENABLESHARED="false" +if [ "$(id -u)" -ne 0 ]; then + echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' + exit 1 +fi +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +MAJOR_VERSION_ID=$(echo ${VERSION_ID} | cut -d . -f 1) +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + if [[ "${ID}" = "rhel" ]] || [[ "${ID}" = *"alma"* ]] || [[ "${ID}" = *"rocky"* ]]; then + VERSION_CODENAME="rhel${MAJOR_VERSION_ID}" + else + VERSION_CODENAME="${ID}${MAJOR_VERSION_ID}" + fi +else + echo "Linux distro ${ID} not supported." + exit 1 +fi + +# Setup INSTALL_CMD & PKG_MGR_CMD +if type apt-get > /dev/null 2>&1; then + PKG_MGR_CMD=apt-get + INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" +elif type microdnf > /dev/null 2>&1; then + PKG_MGR_CMD=microdnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +else + PKG_MGR_CMD=yum + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --noplugins --setopt=install_weak_deps=0" +fi + +pkg_mgr_update() { + case $ADJUSTED_ID in + debian) + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${PKG_MGR_CMD} update -y + fi + ;; + rhel) + if [ ${PKG_MGR_CMD} = "microdnf" ]; then + if [ "$(ls /var/cache/yum/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} makecache ..." + ${PKG_MGR_CMD} makecache + fi + else + if [ "$(ls /var/cache/${PKG_MGR_CMD}/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} check-update ..." + set +e + ${PKG_MGR_CMD} check-update + rc=$? + if [ $rc != 0 ] && [ $rc != 100 ]; then + exit 1 + fi + set -e + fi + fi + ;; + esac +} + +check_packages() { + case ${ADJUSTED_ID} in + debian) + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + rhel) + if ! rpm -q "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + esac +} + +# Import the specified key in a variable name passed in as +receive_gpg_keys() { + local keys=${!1} + local keyring_args="" + local gpg_cmd="gpg" + if [ ! -z "$2" ]; then + mkdir -p "$(dirname \"$2\")" + keyring_args="--no-default-keyring --keyring $2" + fi + if [ ! -z "${KEYSERVER_PROXY}" ]; then + keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + set +e + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} +# RHEL7/CentOS7 has an older gpg that does not have dirmngr +# Iterate through keyservers until we have all the keys downloaded +receive_gpg_keys_centos7() { + local keys=${!1} + local keyring_args="" + local gpg_cmd="gpg" + if [ ! -z "$2" ]; then + mkdir -p "$(dirname \"$2\")" + keyring_args="--no-default-keyring --keyring $2" + fi + if [ ! -z "${KEYSERVER_PROXY}" ]; then + keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + num_keys=$(echo ${keys} | wc -w) + set +e + echo "(*) Downloading GPG keys..." + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do + for keyserver in $(echo "${GPG_KEY_SERVERS}" | sed 's/keyserver //'); do + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver=${keyserver} ) 2>&1 + downloaded_keys=$(gpg --list-keys | grep ^pub | wc -l) + if [[ ${num_keys} = ${downloaded_keys} ]]; then + gpg_ok="true" + break + fi + done + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} + + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +add_symlink() { + CURRENT_PATH="${PYTHON_INSTALL_PATH}/current" + if [[ ! -d "${CURRENT_PATH}" ]]; then + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + fi + + if [ "${OVERRIDE_DEFAULT_VERSION}" = "true" ]; then + if [[ $(ls -l ${CURRENT_PATH}) != *"-> ${INSTALL_PATH}"* ]] ; then + rm "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + fi + fi +} + +install_prev_vers_cpython() { + VERSION=$1 + echo -e "\n(!) Failed to fetch the latest artifacts for cpython ${VERSION}..." + find_prev_version_from_git_tags VERSION https://github.com/python/cpython + echo -e "\nAttempting to install ${VERSION}" + install_cpython "${VERSION}" +} + +install_cpython() { + VERSION=$1 + INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" + mkdir -p /tmp/python-src ${INSTALL_PATH} + cd /tmp/python-src + cpython_tgz_filename="Python-${VERSION}.tgz" + cpython_tgz_url="https://www.python.org/ftp/python/${VERSION}/${cpython_tgz_filename}" + echo "Downloading ${cpython_tgz_filename}..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}" "${cpython_tgz_url}" +} + +install_from_source() { + VERSION=$1 + echo "(*) Building Python ${VERSION} from source..." + echo "(*) Building Python ${VERSION} from source..." + if ! type git > /dev/null 2>&1; then + check_packages git + fi + + echo -e "\nTrying to Install a fake version whose source binary wouldn't exist" + VERSION="3.12.xyz" + + install_cpython "${VERSION}" + if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then + if grep -q "404 Not Found" "/tmp/python-src/${cpython_tgz_filename}"; then + # Use grep to search for "404 Not Found" in the file + echo "\"404 Not Found\" found in /tmp/python-src/${cpython_tgz_filename}. Not able to create source binary" + install_prev_vers_cpython "${VERSION}" + fi + fi; + + # Verify signature + if [[ ${VERSION_CODENAME} = "centos7" ]] || [[ ${VERSION_CODENAME} = "rhel7" ]]; then + receive_gpg_keys_centos7 PYTHON_SOURCE_GPG_KEYS + else + receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + fi + + echo "Downloading ${cpython_tgz_filename}.asc..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}.asc" "${cpython_tgz_url}.asc" + + # Untar and build + tar -xzf "/tmp/python-src/${cpython_tgz_filename}" -C "/tmp/python-src" --strip-components=1 + local config_args="" + + if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then + config_args="${config_args} --enable-optimizations" + fi + if [ "${ENABLESHARED}" = "true" ]; then + config_args=" ${config_args} --enable-shared" + # need double-$: LDFLAGS ends up in Makefile $$ becomes $ when evaluated. + # backslash needed for shell that Make calls escape the $. + export LDFLAGS="${LDFLAGS} -Wl,-rpath="'\$$ORIGIN'"/../lib" + fi + if [ -n "${ADDL_CONFIG_ARGS}" ]; then + config_args="${config_args} ${ADDL_CONFIG_ARGS}" + fi + + ./configure --prefix="${INSTALL_PATH}" --with-ensurepip=install ${config_args} + make -j 8 + make install + + cd /tmp + rm -rf /tmp/python-src ${GNUPGHOME} /tmp/vscdc-settings.env + + ln -s "${INSTALL_PATH}/bin/python3" "${INSTALL_PATH}/bin/python" + ln -s "${INSTALL_PATH}/bin/pip3" "${INSTALL_PATH}/bin/pip" + ln -s "${INSTALL_PATH}/bin/idle3" "${INSTALL_PATH}/bin/idle" + ln -s "${INSTALL_PATH}/bin/pydoc3" "${INSTALL_PATH}/bin/pydoc" + ln -s "${INSTALL_PATH}/bin/python3-config" "${INSTALL_PATH}/bin/python-config" + + add_symlink + +} + +install_from_source + +check "Python version as installed by Fallback Test" bash -c "python3 -V" \ No newline at end of file diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 920e4f541..be37869df 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -1,4 +1,14 @@ { + "install_cpython_fallback_prev_version_test": { + "image": "python:3.12", + "features": { + "python": { + "version": "3.12", + "installTools": false, + "skipVulnerabilityPatching": true + } + } + }, "install_python310_skipVulnerabilityPatching_true": { "image": "python:3.10", "features": { From 2495879eade0b514aafc2ef253db93755883f552 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 15 Mar 2024 23:11:12 +0530 Subject: [PATCH 061/285] Reverting get_previous_version logic to get the first tag_name's value from json (#902) * changes as requested * changes acc. to comments --- src/kubectl-helm-minikube/devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 4 ++-- test/docker-in-docker/docker_build_fallback_buildx.sh | 3 ++- test/kubectl-helm-minikube/install_only_helm_fallback.sh | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index 16472205f..41a8b8b68 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.7", + "version": "1.1.8", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index 69d4e0237..d7de3984f 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -158,8 +158,8 @@ fi # Function to fetch the version released prior to the latest version get_previous_version() { repo_url=$1 - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' + # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' } get_helm() { diff --git a/test/docker-in-docker/docker_build_fallback_buildx.sh b/test/docker-in-docker/docker_build_fallback_buildx.sh index 8d30d2a46..707e0a68a 100644 --- a/test/docker-in-docker/docker_build_fallback_buildx.sh +++ b/test/docker-in-docker/docker_build_fallback_buildx.sh @@ -26,7 +26,8 @@ get_latest_version() { # Function to fetch the previous version of the plugin get_previous_version() { - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' } # Function to change the patch number in a semver version diff --git a/test/kubectl-helm-minikube/install_only_helm_fallback.sh b/test/kubectl-helm-minikube/install_only_helm_fallback.sh index 58ed137de..b8071a0b5 100644 --- a/test/kubectl-helm-minikube/install_only_helm_fallback.sh +++ b/test/kubectl-helm-minikube/install_only_helm_fallback.sh @@ -57,7 +57,7 @@ change_patch_number() { # Function to fetch the previous version of the plugin get_previous_version() { # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' } get_helm() { From 5baf166f67683d6801c327d1c084d15f737947cd Mon Sep 17 00:00:00 2001 From: Leobaldo Alcantara Neto Date: Tue, 26 Mar 2024 14:05:35 -0300 Subject: [PATCH 062/285] Update DOCKER_DASH_COMPOSE_VERSION default to v2 (#920) * Update DOCKER_DASH_COMPOSE_VERSION default to v2 Changed it because devcontainer-feature.json documentation indicates that it's the default version. * Update devcontainer-feature.json version to 1.4.4 --- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 54fca5ec1..2a55fdbca 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.3", + "version": "1.4.4", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index dd39e1f68..6b63469f9 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -10,7 +10,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" -DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none +DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v2"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" SOURCE_SOCKET="${SOURCE_SOCKET:-"/var/run/docker-host.sock"}" From 0100f66138e2082cd76b92c30826cbb4544755a3 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 27 Mar 2024 05:25:37 +0530 Subject: [PATCH 063/285] [docker-in-docker]-docker_compose-fallback-github api (#914) * [docker-in-docker]-docker_compose-fallback-github api * changes misc. * misc. change * changes as required in review comments * changes miscellaneous * suggested changes * changes as requested by review comments on pr * Minor changes as suggested... --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 142 +++++++++--- .../docker_build_fallback_buildx.sh | 206 +++++++++++++----- .../docker_build_fallback_compose.sh | 174 +++++++++++++++ test/docker-in-docker/scenarios.json | 13 +- 5 files changed, 442 insertions(+), 95 deletions(-) create mode 100644 test/docker-in-docker/docker_build_fallback_compose.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 671f4e2ed..812db444c 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.1", + "version": "2.10.2", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index dbe34e2ea..0dc9e52d1 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -109,18 +109,73 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } -install_compose_switch_fallback() { - echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version#v} - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } ########################################### @@ -265,6 +320,16 @@ echo "Finished installing docker / moby!" docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" +# fallback for docker-compose +fallback_compose(){ + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + get_previous_version "${url}" "${repo_url}" compose_version + echo -e "\nAttempting to install v${compose_version}" + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} +} + # If 'docker-compose' command is to be included if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then case "${architecture}" in @@ -301,17 +366,16 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then fi else compose_version=${DOCKER_DASH_COMPOSE_VERSION#v} - find_version_from_git_tags compose_version "https://github.com/docker/compose" "tags/v" + docker_compose_url="https://github.com/docker/compose" + find_version_from_git_tags compose_version "$docker_compose_url" "tags/v" echo "(*) Installing docker-compose ${compose_version}..." - curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} - - if grep -q "Not Found" "${docker_compose_path}"; then - echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_version=${previous_version#v} - curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} - fi + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} || { + if [[ $DOCKER_DASH_COMPOSE_VERSION == "latest" ]]; then + fallback_compose "$docker_compose_url" + else + echo -e "Error: Failed to install docker-compose v${compose_version}" + fi + } chmod +x ${docker_compose_path} @@ -325,6 +389,16 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then fi fi +# fallback method for compose-switch +fallback_compose-switch() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + get_previous_version "$url" "$repo_url" compose_switch_version + echo -e "\nAttempting to install v${compose_switch_version}" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +} + # Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /dev/null 2>&1; then if type docker-compose > /dev/null 2>&1; then @@ -332,8 +406,9 @@ if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /d current_compose_path="$(which docker-compose)" target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" compose_switch_version="latest" - find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || install_compose_switch_fallback + compose_switch_url="https://github.com/docker/compose-switch" + find_version_from_git_tags compose_switch_version "$compose_switch_url" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || fallback_compose-switch "$compose_switch_url" chmod +x /usr/local/bin/compose-switch # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) @@ -360,29 +435,26 @@ fi usermod -aG docker ${USERNAME} -install_previous_version_artifacts() { - wget_exit_code=$? - if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. - echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." - repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL - previous_version=$(get_previous_version "${repo_url}") - buildx_file_name="buildx-${previous_version}.linux-${architecture}" - echo -e "\nAttempting to install ${previous_version}" - wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} - else - echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" - exit 1 - fi +# fallback for docker/buildx +fallback_buildx() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + get_previous_version "$url" "$repo_url" buildx_version + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + echo -e "\nAttempting to install v${buildx_version}" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} } if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then buildx_version="latest" - find_version_from_git_tags buildx_version "https://github.com/docker/buildx" "refs/tags/v" + docker_buildx_url="https://github.com/docker/buildx" + find_version_from_git_tags buildx_version "$docker_buildx_url" "refs/tags/v" echo "(*) Installing buildx ${buildx_version}..." buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" cd /tmp - wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || fallback_buildx "$docker_buildx_url" docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" diff --git a/test/docker-in-docker/docker_build_fallback_buildx.sh b/test/docker-in-docker/docker_build_fallback_buildx.sh index 707e0a68a..e139613a0 100644 --- a/test/docker-in-docker/docker_build_fallback_buildx.sh +++ b/test/docker-in-docker/docker_build_fallback_buildx.sh @@ -14,83 +14,177 @@ check "docker-build" docker build ./ check "docker-buildx" bash -c "docker buildx version" check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" -echo -e "\n๐Ÿ‘‰${HL} Creating a scenario for fallback${N}\n" # Code to test the made up scenario when latest version of docker/buildx fails on wget command for fetching the artifacts -repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL architecture="$(dpkg --print-architecture)" +case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 +esac -# Function to fetch the latest version of the plugin -get_latest_version() { - curl -s "$repo_url/latest" | jq -r '.tag_name' +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + err "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" } -# Function to fetch the previous version of the plugin -get_previous_version() { - # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e } -# Function to change the patch number in a semver version -change_patch_number() { - local version="$1" # Input version - local new_patch="$2" # New patch number - # Extract major, minor, and current patch numbers - local major=$(echo "$version" | cut -d. -f1) - local minor=$(echo "$version" | cut -d. -f2) - local current_patch=$(echo "$version" | cut -d. -f3) - # Construct the new version with the updated patch number - local new_version="$major.$minor.$new_patch" - echo "$new_version" +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + echo -e "\nAttempting to find latest version using Github Api." + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $mode != "install_from_github_api_valid" ]]; then + message="API rate limit exceeded" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAttempting to find latest version using Github Api Failed. Exceeded API Rate Limit." + echo -e "\nAttempting to find latest version using Github Tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using Github Api Succeeded." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } -change_version_to_fail() { - latest_version=$1 - new_patch_number="xyz" # for testing a tag not found scenario for docker/buildx plugin - latest_version=$(get_latest_version) # can take latest_version from fn get_latest_version - buildx_version_fallback_test=$(change_patch_number "$latest_version" "$new_patch_number") # for testing a tag not found scenario for docker/buildx plugin - echo "${buildx_version_fallback_test}" +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -install_previous_version_artifacts() { - wget_exit_code=$? - if [ $wget_exit_code -ne 0 ]; then # means wget command to fetch latest version failed - if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. - echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx ${buildx_version}..." - previous_version=$(get_previous_version) - echo -e "\nAttempting to install ${previous_version}" - buildx_file_name="buildx-${previous_version}.linux-${architecture}" - wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} - else - echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" - exit 1 - fi - fi +install_using_get_previous_version() { + local url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + get_previous_version "${url}" "${repo_url}" buildx_version "${mode}" + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + echo -e "\nAttempting to install v${buildx_version}" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} } -test_version=$(change_version_to_fail "$(get_latest_version)") -buildx_file_name="buildx-${test_version}.linux-${architecture}" -buildx_version=$test_version +install_docker_buildx() { + mode=$1 + echo -e "\n${HL} Creating a scenario for fallback${N}\n" -# This wget command will fail as the wrong version won't fetch artifact -wget https://github.com/docker/buildx/releases/download/${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + buildx_version="0.13.xyz" + echo "(*) Installing buildx ${buildx_version}..." + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + cd /tmp -docker_home="/usr/libexec/docker" -cli_plugins_dir="${docker_home}/cli-plugins" + docker_buildx_url="https://github.com/docker/buildx" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_using_get_previous_version "${docker_buildx_url}" "${mode}" + + docker_home="/usr/libexec/docker" + cli_plugins_dir="${docker_home}/cli-plugins" -mkdir -p ${cli_plugins_dir} -mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx -chmod +x ${cli_plugins_dir}/docker-buildx + mkdir -p ${cli_plugins_dir} + mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx + chmod +x ${cli_plugins_dir}/docker-buildx -chown -R "${USERNAME}:docker" "${docker_home}" -chmod -R g+r+w "${docker_home}" -find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s + chown -R "${USERNAME}:docker" "${docker_home}" + chmod -R g+r+w "${docker_home}" + find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s +} + +echo -e "\n๐Ÿ‘‰${HL} docker-buildx version as installed by docker-in-docker test ( installing by github api ) ${N}" +install_docker_buildx "install_from_github_api_valid" + +# Definition specific tests after test for fallback +check "docker-buildx" docker buildx version +check "docker-buildx" bash -c "docker buildx version" + +echo -e "\n๐Ÿ‘‰${HL} docker-buildx version as installed by docker-in-docker test ( installing by find_prev_version_from_git_tags ) ${N}" +install_docker_buildx # Definition specific tests after test for fallback -echo -e "\n๐Ÿ‘‰${HL} docker/buildx version as installed by test for fallback${N}" check "docker-buildx" docker buildx version -check "docker-build" docker build ./ check "docker-buildx" bash -c "docker buildx version" -check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" # Report result reportResults diff --git a/test/docker-in-docker/docker_build_fallback_compose.sh b/test/docker-in-docker/docker_build_fallback_compose.sh new file mode 100644 index 000000000..4a18640f0 --- /dev/null +++ b/test/docker-in-docker/docker_build_fallback_compose.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# Optional: Import test library +source dev-container-features-test-lib + +# Setup STDERR. +err() { + echo "(!) $*" >&2 +} + +HL="\033[1;33m" +N="\033[0;37m" +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker feature${N}" +check "docker-compose" bash -c "docker-compose version" + +architecture="$(dpkg --print-architecture)" +case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 +esac + +docker_compose_path="/usr/local/bin/docker-compose" +cli_plugins_dir="${docker_home}/cli-plugins" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + err "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + echo -e "\nAttempting to find latest version using Github Api." + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $mode != "install_from_github_api_valid" ]]; then + message="API rate limit exceeded" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAttempting to find latest version using Github Api Failed. Exceeded API Rate Limit." + echo -e "\nAttempting to find latest version using Github Tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using Github Api Succeeded." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_using_get_previous_version() { + local url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + get_previous_version "$url" "$repo_url" compose_version "$mode" + echo -e "\nAttempting to install v${compose_version}" + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} +} + +install_docker_compose() { + mode=$1 + compose_version="2.25.xyz" + docker_compose_url="https://github.com/docker/compose" + echo "(*) Installing docker-compose ${compose_version}..." + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} || install_using_get_previous_version "$docker_compose_url" "$mode" +} + +chmod +x ${docker_compose_path} + +# Download the SHA256 checksum +DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}.sha256" | awk '{print $1}')" +echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum +sha256sum -c docker-compose.sha256sum --ignore-missing + +mkdir -p ${cli_plugins_dir} +cp ${docker_compose_path} ${cli_plugins_dir} + +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker test ( installing by github api ) ${N}" +install_docker_compose "install_from_github_api_valid" + +check "docker-compose" bash -c "docker-compose version" + +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker test ( installing by find_prev_version_from_git_tags ) ${N}" +install_docker_compose + +check "docker-compose" bash -c "docker-compose version" diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index e209f3453..33333583d 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -1,4 +1,13 @@ { + "docker_build_fallback_compose": { + "image": "ubuntu:focal", + "features": { + "docker-in-docker": { + "version": "latest", + "dockerDashComposeVersion": "latest" + } + } + }, "dockerDefaultAddressPool": { "image": "mcr.microsoft.com/vscode/devcontainers/javascript-node:0-18", "remoteUser": "node", @@ -112,9 +121,7 @@ "features": { "docker-in-docker": { "version": "latest", - "installDockerBuildx": true, - "moby": "false", - "dockerDashComposeVersion": "v2" + "installDockerBuildx": true } } }, From 0a4fa18e1db577a60f569f7684dde62430a3179d Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 03:30:20 +0530 Subject: [PATCH 064/285] [azure-cli] - python3.12 - feature broken due to distutils module removed - code fix (#909) * [azure-cli] - python3.12 - feature broken due to distutils module removed - fix * update default for installUsingPython * removed the installUsingPython=true flag from the building scenario * bump update the patch version --------- Co-authored-by: Samruddhi Khandale --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 8 +++---- .../install_with_python_3_12_bookworm.sh | 21 +++++++++++++++++++ test/azure-cli/scenarios.json | 9 ++++++++ 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 test/azure-cli/install_with_python_3_12_bookworm.sh diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index 136b03fdc..f25e5f9a2 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.2", + "version": "1.2.3", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index 6d3426699..7f4c4ff1c 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -15,10 +15,10 @@ rm -rf /var/lib/apt/lists/* AZ_VERSION=${VERSION:-"latest"} AZ_EXTENSIONS=${EXTENSIONS} AZ_INSTALLBICEP=${INSTALLBICEP:-false} -INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-true} +INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -AZCLI_ARCHIVE_ARCHITECTURES="amd64" -AZCLI_ARCHIVE_VERSION_CODENAMES="stretch buster bullseye bionic focal jammy" +AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" +AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' @@ -183,7 +183,7 @@ install_with_complete_python_installation() { export DEBIAN_FRONTEND=noninteractive -# See if we're on x86_64 and if so, install via apt-get, otherwise use pip3 +# See if we're on x86_64 or AARCH64 and if so, install via apt-get, otherwise use pip3 echo "(*) Installing Azure CLI..." . /etc/os-release architecture="$(dpkg --print-architecture)" diff --git a/test/azure-cli/install_with_python_3_12_bookworm.sh b/test/azure-cli/install_with_python_3_12_bookworm.sh new file mode 100644 index 000000000..074876148 --- /dev/null +++ b/test/azure-cli/install_with_python_3_12_bookworm.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version + +echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"'; then + echo -e "\n\nโœ… Passed 'O.S is Linux 12 (bookworm)'!" +else + echo -e "\n\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!" +fi + + +# Report result +reportResults \ No newline at end of file diff --git a/test/azure-cli/scenarios.json b/test/azure-cli/scenarios.json index b7a732395..041f50731 100644 --- a/test/azure-cli/scenarios.json +++ b/test/azure-cli/scenarios.json @@ -38,5 +38,14 @@ "installUsingPython": true } } + }, + "install_with_python_3_12_bookworm": { + "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bookworm", + "user": "vscode", + "features": { + "azure-cli": { + "version": "latest" + } + } } } \ No newline at end of file From 988cdd2bb9eb9a96cf16ed30e2bfba32c68ad81c Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:41:51 +0530 Subject: [PATCH 065/285] [Terraform] feature fallback fix (#912) * implementation of fallback logic for cosign, terraform * tflint, terragrunt - wrote fallback logic for these * misc change * [Terraform] - fallback - apply * changes as requested by review comments * misc change * misc change * 2-step fallback implemented in install.sh, test cases yet to be updated * install jq for ubuntu:focal, jammy & debian:11, 12 * test for terraform_docs updated * tfsec test updated * terraform fallback test updated * misc change * added tflint fallback test file * terragrunt fallback test file added * minor change in tfsec test file * few more changes to cosign, terragrunt installations.. * changes to test files for terragrunt & cosign * Changes acc. to review comments for pr * changes requested * changes as requested in review comments ! --- src/terraform/devcontainer-feature.json | 2 +- src/terraform/install.sh | 192 ++++++++++-- test/terraform/scenarios.json | 40 +++ .../terraform/terraform_docs_fallback_test.sh | 220 ++++++++++++++ test/terraform/terraform_fallback_test.sh | 206 +++++++++++++ test/terraform/terragrunt_fallback_test.sh | 219 +++++++++++++ test/terraform/tflint_fallback_test.sh | 287 ++++++++++++++++++ test/terraform/tfsec_fallback_test.sh | 224 ++++++++++++++ 8 files changed, 1372 insertions(+), 18 deletions(-) create mode 100644 test/terraform/terraform_docs_fallback_test.sh create mode 100644 test/terraform/terraform_fallback_test.sh create mode 100644 test/terraform/terragrunt_fallback_test.sh create mode 100644 test/terraform/tflint_fallback_test.sh create mode 100644 test/terraform/tfsec_fallback_test.sh diff --git a/src/terraform/devcontainer-feature.json b/src/terraform/devcontainer-feature.json index 51fb9a27a..5c9c042cc 100644 --- a/src/terraform/devcontainer-feature.json +++ b/src/terraform/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "terraform", - "version": "1.3.5", + "version": "1.3.6", "name": "Terraform, tflint, and TFGrunt", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/terraform", "description": "Installs the Terraform CLI and optionally TFLint and Terragrunt. Auto-detects latest version and installs needed dependencies.", diff --git a/src/terraform/install.sh b/src/terraform/install.sh index c1a382597..dc7aef8c3 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -137,6 +137,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + find_sentinel_version_from_url() { local variable_name=$1 local requested_version=${!variable_name} @@ -177,6 +218,73 @@ check_packages() { fi } +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + INSTALLER_FN=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_cosign() { + COSIGN_VERSION=$1 + local URL=$2 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." +} + # Install 'cosign' for validating signatures # https://docs.sigstore.dev/cosign/overview/ ensure_cosign() { @@ -184,12 +292,10 @@ ensure_cosign() { if ! type cosign > /dev/null 2>&1; then echo "Installing cosign..." - LATEST_COSIGN_VERSION="latest" - find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' - curl -L "https://github.com/sigstore/cosign/releases/latest/download/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb" -o /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - - dpkg -i /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - rm /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb + COSIGN_VERSION="latest" + cosign_url='https://github.com/sigstore/cosign' + find_version_from_git_tags COSIGN_VERSION "${cosign_url}" + install_cosign "${COSIGN_VERSION}" "${cosign_url}" fi if ! type cosign > /dev/null 2>&1; then echo "(!) Failed to install cosign." @@ -207,18 +313,30 @@ if ! type git > /dev/null 2>&1; then check_packages git fi +terraform_url='https://github.com/hashicorp/terraform' +tflint_url='https://github.com/terraform-linters/tflint' +terragrunt_url='https://github.com/gruntwork-io/terragrunt' # Verify requested version is available, convert latest -find_version_from_git_tags TERRAFORM_VERSION 'https://github.com/hashicorp/terraform' -find_version_from_git_tags TFLINT_VERSION 'https://github.com/terraform-linters/tflint' -find_version_from_git_tags TERRAGRUNT_VERSION 'https://github.com/gruntwork-io/terragrunt' +find_version_from_git_tags TERRAFORM_VERSION "$terraform_url" +find_version_from_git_tags TFLINT_VERSION "$tflint_url" +find_version_from_git_tags TERRAGRUNT_VERSION "$terragrunt_url" + +install_terraform() { + local TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +} mkdir -p /tmp/tf-downloads cd /tmp/tf-downloads - # Install Terraform, tflint, Terragrunt echo "Downloading terraform..." terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" -curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +install_terraform "$TERRAFORM_VERSION" +if grep -q "The specified key does not exist." "${terraform_filename}"; then + install_previous_version TERRAFORM_VERSION $terraform_url "install_terraform" + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" +fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_SHA256}" = "automatic" ]; then receive_gpg_keys TERRAFORM_GPG_KEY @@ -233,10 +351,18 @@ fi unzip ${terraform_filename} mv -f terraform /usr/local/bin/ +install_tflint() { + TFLINT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} +} + if [ "${TFLINT_VERSION}" != "none" ]; then echo "Downloading tflint..." TFLINT_FILENAME="tflint_linux_${architecture}.zip" - curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} + install_tflint "$TFLINT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + install_previous_version TFLINT_VERSION "$tflint_url" "install_tflint" + fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then if [ "${TFLINT_SHA256}" != "automatic" ]; then @@ -277,10 +403,20 @@ if [ "${TFLINT_VERSION}" != "none" ]; then unzip /tmp/tf-downloads/${TFLINT_FILENAME} mv -f tflint /usr/local/bin/ fi + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} +} + if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." terragrunt_filename="terragrunt_linux_${architecture}" - curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} + install_terragrunt "$TERRAGRUNT_VERSION" + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then + install_previous_version TERRAGRUNT_VERSION $terragrunt_url "install_terragrunt" + fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS @@ -318,12 +454,23 @@ if [ "${INSTALL_SENTINEL}" = "true" ]; then mv -f /tmp/tf-downloads/sentinel /usr/local/bin/sentinel fi +install_tfsec() { + local TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + if [ "${INSTALL_TFSEC}" = "true" ]; then TFSEC_VERSION="latest" - find_version_from_git_tags TFSEC_VERSION 'https://github.com/aquasecurity/tfsec' + tfsec_url='https://github.com/aquasecurity/tfsec' + find_version_from_git_tags TFSEC_VERSION $tfsec_url tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" echo "(*) Downloading TFSec... ${tfsec_filename}" - curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + install_previous_version TFSEC_VERSION $tfsec_url "install_tfsec" + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then if [ "${TFSEC_SHA256}" = "automatic" ]; then curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt @@ -338,12 +485,23 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec fi +install_terraform_docs() { + local TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + if [ "${INSTALL_TERRAFORM_DOCS}" = "true" ]; then TERRAFORM_DOCS_VERSION="latest" - find_version_from_git_tags TERRAFORM_DOCS_VERSION 'https://github.com/terraform-docs/terraform-docs' + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' + find_version_from_git_tags TERRAFORM_DOCS_VERSION $terraform_docs_url tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" echo "(*) Downloading Terraform docs... ${tfdocs_filename}" - curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} + install_terraform_docs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url "install_terraform_docs" + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index 0a4d8956c..04693666c 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -15,6 +15,14 @@ } } }, + "tfsec_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTFsec": true + } + } + }, "install_terraform_docs": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { @@ -23,6 +31,38 @@ } } }, + "terraform_docs_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTerraformDocs": true + } + } + }, + "terraform_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "version": "latest" + } + } + }, + "terragrunt_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "terragrunt": "latest" + } + } + }, + "tflint_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "tflint": "latest" + } + } + }, "older_tflint": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh new file mode 100644 index 000000000..256ef2f85 --- /dev/null +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +# Terraform Docs specific tests +check "terraform-docs version as installed by feature" terraform-docs --version + +TERRAFORM_DOCS_SHA256="automatic" + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_terraform_docs() { + local TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + + +try_install_terraform_docs_dummy_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TERRAFORM_DOCS_VERSION="0.17.xyz" + echo -e "\nInstalling TERRAFORM_DOCS dummy version.." v${TERRAFORM_DOCS_VERSION} + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + echo "(*) Downloading Terraform docs... ${tfdocs_filename}" + install_terraform_docs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url $mode "install_terraform_docs" + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi + if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then + if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then + curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum + else + echo "${TERRAFORM_DOCS_SHA256} *${tfsec_filename}" > tfdocs_SHA256SUMS + fi + sha256sum --ignore-missing -c tfdocs_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfdocs + tar -xzf /tmp/tf-downloads/${tfdocs_filename} -C /tmp/tf-downloads/tfdocs + sudo chmod a+x /tmp/tf-downloads/tfdocs/terraform-docs + sudo mv -f /tmp/tf-downloads/tfdocs/terraform-docs /usr/local/bin/terraform-docs +} + +try_install_terraform_docs_dummy_version "mode1" + +check "terraform-docs version as installed by test (mode 1: install using find_prev_version_from_git_tags)" terraform-docs --version + +try_install_terraform_docs_dummy_version "mode2" + +check "terraform-docs version as installed by test (mode 2: install using GitHub Api)" terraform-docs --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh new file mode 100644 index 000000000..a5193daa4 --- /dev/null +++ b/test/terraform/terraform_fallback_test.sh @@ -0,0 +1,206 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +check "terraform version as installed by feature" terraform --version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_terraform() { + local TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +} + +try_install_dummy_terraform_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terraform_url='https://github.com/hashicorp/terraform' + TERRAFORM_VERSION="1.7.xyz" + echo -e "\nAttempting to install dummy version for Terraform v${TERRAFORM_VERSION}..." + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + install_terraform "$TERRAFORM_VERSION" + if grep -q "The specified key does not exist." "${terraform_filename}"; then + install_previous_version TERRAFORM_VERSION $terraform_url $mode "install_terraform" + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + fi + unzip ${terraform_filename} + sudo mv -f terraform /usr/local/bin/ +} + +try_install_dummy_terraform_version "mode1" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terraform --version + +try_install_dummy_terraform_version "mode2" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terraform --version + +# Report result +reportResults + diff --git a/test/terraform/terragrunt_fallback_test.sh b/test/terraform/terragrunt_fallback_test.sh new file mode 100644 index 000000000..83289380f --- /dev/null +++ b/test/terraform/terragrunt_fallback_test.sh @@ -0,0 +1,219 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +check "terragrunt version as installed by feature" terragrunt --version + +TERRAGRUNT_SHA256="automatic" + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} +} + + +try_install_dummy_terragrunt_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terragrunt_url='https://github.com/gruntwork-io/terragrunt' + TERRAGRUNT_VERSION="0.55.xyz" + echo -e "\nAttempting to install terragrunt dummy v${TERRAGRUNT_VERSION}" + echo "Downloading Terragrunt... v${TERRAGRUNT_VERSION}" + terragrunt_filename="terragrunt_linux_${architecture}" + install_terragrunt "$TERRAGRUNT_VERSION" + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then + install_previous_version TERRAGRUNT_VERSION $terragrunt_url $mode "install_terragrunt" + fi + if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then + if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then + curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS + else + echo "${TERRAGRUNT_SHA256} *${terragrunt_filename}" > terragrunt_SHA256SUMS + fi + sha256sum --ignore-missing -c terragrunt_SHA256SUMS + fi + sudo chmod a+x /tmp/tf-downloads/${terragrunt_filename} + sudo mv -f /tmp/tf-downloads/${terragrunt_filename} /usr/local/bin/terragrunt +} + +try_install_dummy_terragrunt_version "mode1" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terragrunt --version + +try_install_dummy_terragrunt_version "mode2" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terragrunt --version + +# Report result +reportResults + diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh new file mode 100644 index 000000000..5619ff4fb --- /dev/null +++ b/test/terraform/tflint_fallback_test.sh @@ -0,0 +1,287 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +TFLINT_SHA256="automatic" + +GPG_KEY_SERVERS="keyserver hkps://keyserver.ubuntu.com +keyserver hkps://keys.openpgp.org +keyserver hkps://keyserver.pgp.com" + +check "tflint version as installed by feature" tflint --version +check "cosign version as installed by feature" cosign version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_cosign() { + COSIGN_VERSION=$1 + local URL=$2 + local mode=$3 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION $mode + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." +} + +# Install 'cosign' for validating signatures +# https://docs.sigstore.dev/cosign/overview/ +ensure_cosign() { + mode=$1 + if ! type cosign > /dev/null 2>&1; then + echo -e "\nAttempting to install dummy cosign version..." + COSIGN_VERSION="2.2.xyz" + echo "Installing cosign... v${COSIGN_VERSION}" + cosign_url='https://github.com/sigstore/cosign' + install_cosign "${COSIGN_VERSION}" "${cosign_url}" $mode + fi + if ! type cosign > /dev/null 2>&1; then + echo "(!) Failed to install cosign." + exit 1 + fi + cosign version +} + +install_tflint() { + TFLINT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} +} + + +try_install_dummy_tflint_cosign_version() { + mode=$1 + tflint_url='https://github.com/terraform-linters/tflint' + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + echo -e "\nTrying to install dummy tflint version..." + TFLINT_VERSION="0.50.XYZ" + echo "Downloading tflint...v${TFLINT_VERSION}" + TFLINT_FILENAME="tflint_linux_${architecture}.zip" + install_tflint "$TFLINT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + install_previous_version TFLINT_VERSION "$tflint_url" $mode "install_tflint" + fi + if [ "${TFLINT_SHA256}" != "dev-mode" ]; then + + if [ "${TFLINT_SHA256}" != "automatic" ]; then + echo "${TFLINT_SHA256} *${TFLINT_FILENAME}" > tflint_checksums.txt + sha256sum --ignore-missing -c tflint_checksums.txt + else + curl -sSL -o tflint_checksums.txt https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt + + set +e + curl -sSL -o checksums.txt.keyless.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.keyless.sig + set -e + + # Check that checksums.txt.keyless.sig exists and is not empty + if [ -s checksums.txt.keyless.sig ]; then + # Validate checksums with cosign + curl -sSL -o checksums.txt.pem https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.pem + ensure_cosign $mode + cosign verify-blob \ + --certificate=/tmp/tf-downloads/checksums.txt.pem \ + --signature=/tmp/tf-downloads/checksums.txt.keyless.sig \ + --certificate-identity-regexp="^https://github.com/terraform-linters/tflint" \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + /tmp/tf-downloads/tflint_checksums.txt + # Ensure that checksums.txt has $TFLINT_FILENAME + grep ${TFLINT_FILENAME} /tmp/tf-downloads/tflint_checksums.txt + # Validate downloaded file + sha256sum --ignore-missing -c tflint_checksums.txt + else + # Fallback to older, GPG-based verification (pre-0.47.0 of tflint) + curl -sSL -o tflint_checksums.txt.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.sig + curl -sSL -o tflint_key "${TFLINT_GPG_KEY_URI}" + gpg -q --import tflint_key + gpg --verify tflint_checksums.txt.sig tflint_checksums.txt + fi + fi + fi + + unzip /tmp/tf-downloads/${TFLINT_FILENAME} + sudo mv -f tflint /usr/local/bin/ +} + +try_install_dummy_tflint_cosign_version "mode1" + +check "tflint version as installed when mode=1" tflint --version +check "cosign version as installed when mode=1" cosign version + +try_install_dummy_tflint_cosign_version "mode2" + +check "tflint version as installed when mode=2" tflint --version +check "cosign version as installed when mode=2" cosign version \ No newline at end of file diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh new file mode 100644 index 000000000..5c87d821e --- /dev/null +++ b/test/terraform/tfsec_fallback_test.sh @@ -0,0 +1,224 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +# Trap errors and call the error handling function +trap 'handle_error' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +TFSEC_SHA256="automatic" + +# TFSec specific tests +check "tfsec version as installed by feature" tfsec --version + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_tfsec() { + local TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + +try_install_tfsec_dummy_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TFSEC_VERSION="1.28.XYZ" + echo -e "\nInstalling TFSEC dummy version.." v${TFSEC_VERSION} + tfsec_url='https://github.com/aquasecurity/tfsec' + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + echo "(*) Downloading TFSec... ${tfsec_filename}" + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + install_previous_version TFSEC_VERSION $tfsec_url $mode "install_tfsec" + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi + if [ "${TFSEC_SHA256}" != "dev-mode" ]; then + if [ "${TFSEC_SHA256}" = "automatic" ]; then + curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt + else + echo "${TFSEC_SHA256} *${tfsec_filename}" > tfsec_SHA256SUMS + fi + sha256sum --ignore-missing -c tfsec_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfsec + tar -xzf /tmp/tf-downloads/${tfsec_filename} -C /tmp/tf-downloads/tfsec + chmod a+x /tmp/tf-downloads/tfsec/tfsec + sudo mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec +} + +try_install_tfsec_dummy_version "mode1" + +check "tfsec version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" tfsec --version + +try_install_tfsec_dummy_version "mode2" + +check "tfsec version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" tfsec --version + +# Report result +reportResults \ No newline at end of file From cf03551a529f39f4a50b37e39fac0964c7a579ca Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:58:53 +0530 Subject: [PATCH 066/285] [kubectl-helm-minikube] - helm - alternative fallback method implemented (#923) * [kubectl-helm-minikube] - helm - alternative fallback method implemented * bump patch version --- .../devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 80 +++++++- .../install_only_helm_fallback.sh | 192 ++++++++++++------ 3 files changed, 200 insertions(+), 74 deletions(-) diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index 41a8b8b68..8c5dfddc0 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.8", + "version": "1.1.9", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index d7de3984f..871a77ca1 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -88,6 +88,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + apt_get_update() { if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then @@ -157,9 +198,32 @@ fi # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name#v} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="v${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } get_helm() { @@ -173,7 +237,8 @@ get_helm() { if [ ${HELM_VERSION} != "none" ]; then # Install Helm, verify signature and checksum echo "Downloading Helm..." - find_version_from_git_tags HELM_VERSION "https://github.com/helm/helm" + helm_url="https://github.com/helm/helm" + find_version_from_git_tags HELM_VERSION "${helm_url}" if [ "${HELM_VERSION::1}" != 'v' ]; then HELM_VERSION="v${HELM_VERSION}" fi @@ -181,10 +246,9 @@ if [ ${HELM_VERSION} != "none" ]; then get_helm "${HELM_VERSION}" if grep -q "BlobNotFound" "${tmp_helm_filename}"; then echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." - repo_url=https://api.github.com/repos/helm/helm/releases - requested_version=$(get_previous_version "${repo_url}") - echo -e "\nAttempting to install ${requested_version}" - HELM_VERSION=${requested_version} + repo_url=$(get_github_api_repo_url "${helm_url}") + get_previous_version "${helm_url}" "${repo_url}" HELM_VERSION + echo -e "\nAttempting to install ${HELM_VERSION}" get_helm "${HELM_VERSION}" fi export GNUPGHOME="/tmp/helm/gnupg" diff --git a/test/kubectl-helm-minikube/install_only_helm_fallback.sh b/test/kubectl-helm-minikube/install_only_helm_fallback.sh index b8071a0b5..9f6a8e095 100644 --- a/test/kubectl-helm-minikube/install_only_helm_fallback.sh +++ b/test/kubectl-helm-minikube/install_only_helm_fallback.sh @@ -31,33 +31,118 @@ case $architecture in i?86) architecture="386";; *) echo "(!) Architecture $architecture unsupported"; exit 1 ;; esac -HELM_SHA256="${HELM_SHA256:-"automatic"}" -HELM_GPG_KEYS_URI="https://raw.githubusercontent.com/helm/helm/main/KEYS" -repo_url=https://api.github.com/repos/helm/helm/releases +helm_url="https://github.com/helm/helm" -# Function to fetch the latest version of the plugin -get_latest_version() { - curl -s "$repo_url/latest" | jq -r '.tag_name' +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" } -# Function to change the patch number in a semver version -change_patch_number() { - local version="$1" # Input version - local new_patch="$2" # New patch number - # Extract major, minor, and current patch numbers - local major=$(echo "$version" | cut -d. -f1) - local minor=$(echo "$version" | cut -d. -f2) - local current_patch=$(echo "$version" | cut -d. -f3) - # Construct the new version with the updated patch number - local new_version="$major.$minor.$new_patch" - echo "$new_version" +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e } -# Function to fetch the previous version of the plugin +# Function to fetch the version released prior to the latest version get_previous_version() { - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name#v} + + output=$(curl -s "$repo_url"); + + message=$(echo "$output" | jq -r '.message') + + if [ $mode == "mode1" ]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="v${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } get_helm() { @@ -68,53 +153,30 @@ get_helm() { sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" } -latest_version=$(get_latest_version) -NON_EXISTING_PATCH_VERSION="xyz" -HELM_VERSION="$(change_patch_number ${latest_version} ${NON_EXISTING_PATCH_VERSION})" -echo -e "\n๐Ÿ‘‰${HL} Trying to install HELM_VERSION = ${HELM_VERSION}${N}"; -sudo mkdir -p /tmp/helm -get_helm "${HELM_VERSION}" -if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then - echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." - requested_version=$(get_previous_version) - echo -e "\nAttempting to install ${requested_version}" - HELM_VERSION=${requested_version} +install_helm() { + mode=$1 + HELM_VERSION="v3.14.xyz" + echo -e "\n๐Ÿ‘‰Trying to install HELM_VERSION = ${HELM_VERSION}"; + sudo mkdir -p /tmp/helm get_helm "${HELM_VERSION}" -fi -export GNUPGHOME="/tmp/helm/gnupg" -sudo mkdir -p "${GNUPGHOME}" -sudo chmod 700 ${GNUPGHOME} -sudo curl -sSL "${HELM_GPG_KEYS_URI}" -o /tmp/helm/KEYS -sudo echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | sudo tee ${GNUPGHOME}/dirmngr.conf >/dev/null -sudo gpg -q --import "/tmp/helm/KEYS" -if ! sudo gpg --verify "${tmp_helm_filename}.asc" | sudo tee ${GNUPGHOME}/verify.log 2>&1; then - echo "Verification failed!" - sudo cat /tmp/helm/gnupg/verify.log - exit 1 -fi - -if [ "${HELM_SHA256}" = "automatic" ]; then - sudo curl -sSL "https://get.helm.sh/${helm_filename}.sha256" -o "${tmp_helm_filename}.sha256" - sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.sha256.asc" -o "${tmp_helm_filename}.sha256.asc" - if ! sudo gpg --verify "${tmp_helm_filename}.sha256.asc" | sudo tee /tmp/helm/gnupg/verify.log 2>&1; then - echo "Verification failed!" - sudo cat /tmp/helm/gnupg/verify.log - exit 1 + if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." + repo_url=$(get_github_api_repo_url "${helm_url}") + get_previous_version "${helm_url}" "${repo_url}" HELM_VERSION $mode + echo -e "\nAttempting to install ${HELM_VERSION}" + get_helm "${HELM_VERSION}" fi - HELM_SHA256="$(sudo cat "${tmp_helm_filename}.sha256")" -fi - -([ "${HELM_SHA256}" = "dev-mode" ] || (sudo echo "${HELM_SHA256} *${tmp_helm_filename}" | sha256sum -c -)) -sudo tar xf "${tmp_helm_filename}" -C /tmp/helm -sudo mv -f "/tmp/helm/linux-${architecture}/helm" /usr/local/bin/ -sudo chmod 0755 /usr/local/bin/helm -sudo rm -rf /tmp/helm -if ! type helm > /dev/null 2>&1; then - echo '(!) Helm installation failed!' - exit 1 -fi - -echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}:" +} + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}: (mode1: installation using find_prev_version_using_git_tags() fn)" +install_helm "mode1" + +set +e + check "helm version" helm version +set -e + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}: (mode2: installation using GitHub api)" +install_helm "mode2" set +e check "helm version" helm version From 32d2b5baf8071f00297fa0cb9fcfd79b9f238387 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 23:00:49 +0530 Subject: [PATCH 067/285] [docker-outside-of-docker] - alternative fallback implementation (#924) --- .../devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 84 ++++++++- .../docker_build_compose_fallback.sh | 161 ++++++++++++++++-- 3 files changed, 224 insertions(+), 23 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 2a55fdbca..2506031ae 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.4", + "version": "1.4.5", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 6b63469f9..16ad15f60 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -99,19 +99,84 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } -# Function to fetch the previous version of the plugin +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} install_compose_switch_fallback() { + compose_switch_url=$1 + repo_url=$(get_github_api_repo_url "${compose_switch_url}") echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version#v} + get_previous_version "${compose_switch_url}" "${repo_url}" compose_switch_version + echo -e "\nAttempting to install v${compose_switch_version}" curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose } @@ -270,8 +335,9 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then else echo "(*) Installing compose-switch as docker-compose..." compose_switch_version="latest" - find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + compose_switch_url="https://github.com/docker/compose-switch" + find_version_from_git_tags compose_switch_version "${compose_switch_url}" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback "${compose_switch_url}" chmod +x /usr/local/bin/docker-compose # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 fi diff --git a/test/docker-outside-of-docker/docker_build_compose_fallback.sh b/test/docker-outside-of-docker/docker_build_compose_fallback.sh index e1c0885e6..193d79d28 100644 --- a/test/docker-outside-of-docker/docker_build_compose_fallback.sh +++ b/test/docker-outside-of-docker/docker_build_compose_fallback.sh @@ -3,34 +3,169 @@ # Optional: Import test library source dev-container-features-test-lib +echo -e "\n๐Ÿ‘‰ Checking version of compose-switch installed as docker-compose as installed by feature"; check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" +trap 'echo "Last executed command failed at line ${LINENO}"' ERR + # Fetch host/container arch. architecture="$(dpkg --print-architecture)" -repo_url="https://api.github.com/repos/docker/compose-switch/releases" +sudo mkdir -p /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" -# Function to fetch the previous version of the plugin + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - sudo curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $mode == 'mode1' ]]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } install_compose_switch_fallback() { - echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch ${test_compose_switch_version}..." - previous_version=$(get_previous_version) - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version} - sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose + compose_switch_url=$1 + mode=$2 + repo_url=$(get_github_api_repo_url "${compose_switch_url}") + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + get_previous_version "${compose_switch_url}" "${repo_url}" compose_switch_version $mode + echo -e "\nAttempting to install v${compose_switch_version}" + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose } install_compose-switch_as_docker-compose() { + mode=$1 echo "(*) Installing compose-switch as docker-compose..." - test_compose_switch_version="1.2.xyz" - echo -e "\nTesting with $test_compose_switch_version..." - sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${test_compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + compose_switch_version="1.0.6" + compose_switch_url="https://github.com/docker/compose-switch" + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback "${compose_switch_url}" $mode sudo chmod +x /usr/local/bin/docker-compose } -install_compose-switch_as_docker-compose +echo -e "\n๐Ÿ‘‰ Trying to install compose-switch as docker-compose using mode 1 ( find_prev_version_from_git_tags method )"; +install_compose-switch_as_docker-compose "mode1" +check "installs compose-switch as docker-compose mode 1" bash -c "[[ -f /usr/local/bin/docker-compose ]]" -check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file +echo -e "\n๐Ÿ‘‰ Trying to install compose-switch as docker-compose using mode 2 ( GitHub Api )"; +install_compose-switch_as_docker-compose "mode2" +check "installs compose-switch as docker-compose mode 2" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file From 55dbe138fc159c462a6c4554b4619db3ff3b04ab Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Tue, 2 Apr 2024 06:05:32 +0530 Subject: [PATCH 068/285] [PowerShell]- Fallback method with find_prev_vers_frm_git_tags() implemented (#926) * [PowerShell]- Fallback method with find_prev_vers_frm_git_tags() done * bumped the feature version --- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 81 ++++++++- .../install_powershell_fallback_test.sh | 164 ++++++++++++++---- 3 files changed, 201 insertions(+), 46 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index dd7a96b90..82ef39a30 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.4", + "version": "1.3.5", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 4ba8bcf5c..533da6f37 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -106,17 +106,83 @@ install_using_apt() { apt-get install -yq powershell${version_suffix} || return 1 } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + check_packages jq + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + + install_prev_pwsh() { + pwsh_url=$1 + repo_url=$(get_github_api_repo_url $pwsh_url) echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." - previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") - echo -e "\nAttempting to install ${previous_version}" - POWERSHELL_VERSION="${previous_version#v}" + get_previous_version $pwsh_url $repo_url POWERSHELL_VERSION + echo -e "\nAttempting to install v${POWERSHELL_VERSION}" install_pwsh "${POWERSHELL_VERSION}" } @@ -138,10 +204,11 @@ install_using_github() { if [ "${architecture}" = "amd64" ]; then architecture="x64" fi - find_version_from_git_tags POWERSHELL_VERSION https://github.com/PowerShell/PowerShell + pwsh_url="https://github.com/PowerShell/PowerShell" + find_version_from_git_tags POWERSHELL_VERSION $pwsh_url install_pwsh "${POWERSHELL_VERSION}" if grep -q "Not Found" "${powershell_filename}"; then - install_prev_pwsh + install_prev_pwsh $pwsh_url fi # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. diff --git a/test/powershell/install_powershell_fallback_test.sh b/test/powershell/install_powershell_fallback_test.sh index 852d7d103..39c8bc49e 100644 --- a/test/powershell/install_powershell_fallback_test.sh +++ b/test/powershell/install_powershell_fallback_test.sh @@ -15,16 +15,125 @@ check "Powershell version as installed by feature" bash -c "pwsh --version" . /etc/os-release architecture="$(dpkg --print-architecture)" +sudo mkdir -p /var/lib/apt/lists/ + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [ $mode == "mode1" ]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } install_prev_pwsh() { + local pwsh_url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url $pwsh_url) echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." - previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") - echo -e "\nAttempting to install ${previous_version}" - POWERSHELL_VERSION="${previous_version#v}" + get_previous_version $pwsh_url $repo_url POWERSHELL_VERSION $mode + echo -e "\nAttempting to install v${POWERSHELL_VERSION}" install_pwsh "${POWERSHELL_VERSION}" } @@ -37,44 +146,18 @@ install_pwsh() { sudo curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - sudo apt-get update -y - fi -} - -check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - sudo chmod +x /var/lib/apt/lists/ - sudo mkdir -p /var/lib/apt/lists/partial - sudo chmod +rx /var/lib/dpkg/lock-frontend - apt_get_update - sudo apt-get -y install --no-install-recommends "$@" - fi -} - install_using_github() { - # Fall back on direct download if no apt package exists in microsoft pool - check_packages curl ca-certificates gnupg2 dirmngr libc6 libgcc1 libgssapi-krb5-2 libstdc++6 libunwind8 libuuid1 zlib1g libicu[0-9][0-9] - if ! type git > /dev/null 2>&1; then - check_packages git - fi + mode=$1 if [ "${architecture}" = "amd64" ]; then architecture="x64" fi - - echo -e "\nTrying to install a non-existing version for Powershell..." - - POWERSHELL_VERSION="1.2.XYZ" + pwsh_url="https://github.com/PowerShell/PowerShell" + POWERSHELL_VERSION="7.4.xyz" install_pwsh "${POWERSHELL_VERSION}" - if grep -q "Not Found" "${powershell_filename}"; then - install_prev_pwsh + install_prev_pwsh $pwsh_url $mode fi - echo -e "\n" $POWERSHELL_VERSION "=powershell_version\n"; # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. sudo curl -sSL -o "release.html" "https://github.com/PowerShell/PowerShell/releases/tag/v${POWERSHELL_VERSION}" powershell_archive_sha256="$(cat release.html | tr '\n' ' ' | sed 's|<[^>]*>||g' | grep -oP "${powershell_filename}\s+\K[0-9a-fA-F]{64}" || echo '')" @@ -86,12 +169,17 @@ install_using_github() { fi sudo tar xf "${powershell_filename}" -C "${powershell_target_path}" sudo ln -s "${powershell_target_path}/pwsh" /usr/local/bin/pwsh - sudo rm -rf /tmp/pwsh + sudo rm -rf /tmp/pwsh /usr/local/bin/pwsh + } -install_using_github +echo -e "\nInstalling Powershell with find_prev_version_from_git_tags() fn ๐Ÿ‘ˆ๐Ÿป" +install_using_github "mode1" +check "Powershell version as installed by test (find_prev_version_from_git_tags() fn)" bash -c "pwsh --version" -check "Powershell version as installed by test" bash -c "pwsh --version" +echo -e "\nInstalling Powershell with GitHub Api ๐Ÿ‘ˆ๐Ÿป" +install_using_github "mode2" +check "Powershell version as installed by test (GitHub Api)" bash -c "pwsh --version" # Report result reportResults From a7c8c9a83b3ce0f7f0c546537d814009f9e48590 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 3 Apr 2024 03:06:19 +0530 Subject: [PATCH 069/285] [Php] - fallback to previous version - code fix (#908) * [php]- php fallback to prev. version - fix * update patch version for php feature * few changes.. * small change * test completes successfully pointing at the new fallbacked php version * changes acc. to review comments.. --- src/php/devcontainer-feature.json | 2 +- src/php/install.sh | 69 +++++++++- test/php/scenarios.json | 8 ++ test/php/test_php_fallback.sh | 201 ++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 6 deletions(-) create mode 100644 test/php/test_php_fallback.sh diff --git a/src/php/devcontainer-feature.json b/src/php/devcontainer-feature.json index 77fcf9812..a4edcca95 100644 --- a/src/php/devcontainer-feature.json +++ b/src/php/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "php", - "version": "1.1.2", + "version": "1.1.3", "name": "PHP", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/php", "options": { diff --git a/src/php/install.sh b/src/php/install.sh index 48140e1b3..357395e88 100755 --- a/src/php/install.sh +++ b/src/php/install.sh @@ -121,6 +121,46 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Install PHP Composer addcomposer() { "${PHP_SRC}" -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" @@ -130,8 +170,7 @@ addcomposer() { "${PHP_SRC}" -r "unlink('composer-setup.php');" } -install_php() { - PHP_VERSION="$1" +init_php_install() { PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" if [ -d "${PHP_INSTALL_DIR}" ]; then echo "(!) PHP version ${PHP_VERSION} already exists." @@ -142,7 +181,6 @@ install_php() { groupadd -r php fi usermod -a -G php "${USERNAME}" - PHP_URL="https://www.php.net/distributions/php-${PHP_VERSION}.tar.gz" PHP_INI_DIR="${PHP_INSTALL_DIR}/ini" @@ -155,7 +193,26 @@ install_php() { PHP_SRC_DIR="/usr/src/php" mkdir -p $PHP_SRC_DIR cd $PHP_SRC_DIR - wget -O php.tar.xz "$PHP_URL" +} + +install_previous_version() { + PHP_VERSION=$1 + if [[ "$ORIGINAL_PHP_VERSION" == "latest" ]]; then + find_prev_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" + echo -e "\nAttempting to install previous version v${PHP_VERSION}" + init_php_install + wget -O php.tar.xz "$PHP_URL" + else + echo -e "\nFailed to install v$PHP_VERSION" + fi +} + +install_php() { + PHP_VERSION="$1" + + init_php_install + + wget -O php.tar.xz "$PHP_URL" || install_previous_version "$PHP_VERSION" tar -xf $PHP_SRC_DIR/php.tar.xz -C "$PHP_SRC_DIR" --strip-components=1 cd $PHP_SRC_DIR; @@ -195,7 +252,7 @@ install_php() { if [ "${PHP_VERSION}" != "none" ]; then # Persistent / runtime dependencies - RUNTIME_DEPS="wget ca-certificates git build-essential xz-utils" + RUNTIME_DEPS="wget ca-certificates git build-essential xz-utils curl" # PHP dependencies PHP_DEPS="libssl-dev libcurl4-openssl-dev libedit-dev libsqlite3-dev libxml2-dev zlib1g-dev libsodium-dev libonig-dev" @@ -214,6 +271,8 @@ if [ "${PHP_VERSION}" != "none" ]; then # Install dependencies check_packages $RUNTIME_DEPS $PHP_DEPS $PHPIZE_DEPS + # storing value of PHP_VERSION before it changes + ORIGINAL_PHP_VERSION=$PHP_VERSION find_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" install_php "${PHP_VERSION}" diff --git a/test/php/scenarios.json b/test/php/scenarios.json index 1e4df063d..e53bb67ce 100644 --- a/test/php/scenarios.json +++ b/test/php/scenarios.json @@ -32,5 +32,13 @@ "installComposer": true } } + }, + "test_php_fallback": { + "image": "ubuntu:focal", + "features": { + "php": { + "version": "latest" + } + } } } diff --git a/test/php/test_php_fallback.sh b/test/php/test_php_fallback.sh new file mode 100644 index 000000000..84ed83d77 --- /dev/null +++ b/test/php/test_php_fallback.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +echo -e "\nInstalled PHP Version by Feature: ๐Ÿ‘‡ "; php -v; + +USERNAME="root" +PHP_DIR="/usr/local/php" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + echo "${!variable_name}" + echo "$(echo "${requested_version}" | grep -o "." | wc -l)" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + echo "${!variable_name}" + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +init_php_install() { + PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" + if [ -d "${PHP_INSTALL_DIR}" ]; then + echo "(!) PHP version ${PHP_VERSION} already exists." + exit 1 + fi + + if ! cat /etc/group | grep -e "^php:" > /dev/null 2>&1; then + groupadd -r php + fi + usermod -a -G php "${USERNAME}" + + PHP_URL="https://www.php.net/distributions/php-${PHP_VERSION}.tar.gz" + + PHP_INI_DIR="${PHP_INSTALL_DIR}/ini" + CONF_DIR="${PHP_INI_DIR}/conf.d" + mkdir -p "${CONF_DIR}"; + + PHP_EXT_DIR="${PHP_INSTALL_DIR}/extensions" + mkdir -p "${PHP_EXT_DIR}" + + PHP_SRC_DIR="/usr/src/php" + mkdir -p $PHP_SRC_DIR + cd $PHP_SRC_DIR +} + +install_previous_version() { + echo -e "\nInstalling Previous Version..." + find_prev_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" + echo -e "\nNow installing this version as a fallback previous version: ${PHP_VERSION} ๐Ÿคž๐Ÿป" + init_php_install + wget -O php.tar.xz "$PHP_URL" +} + +install_php() { + # trying to install with a possible new tag not having a released source binary yet + PHP_VERSION="8.3.xyz" + + init_php_install + + wget -O php.tar.xz "$PHP_URL" || install_previous_version + + tar -xf $PHP_SRC_DIR/php.tar.xz -C "$PHP_SRC_DIR" --strip-components=1 + cd $PHP_SRC_DIR; + + # PHP 7.4+, the pecl/pear installers are officially deprecated and are removed in PHP 8+ + # Thus, requiring an explicit "--with-pear" + IFS="." + read -a versions <<< "${PHP_VERSION}" + PHP_MAJOR_VERSION=${versions[0]} + PHP_MINOR_VERSION=${versions[1]} + + VERSION_CONFIG="" + if (( $(($PHP_MAJOR_VERSION)) >= 8 )) || (( $(($PHP_MAJOR_VERSION)) == 7 && $(($PHP_MINOR_VERSION)) >= 4 )); then + VERSION_CONFIG="--with-pear" + fi + + ./configure --prefix="${PHP_INSTALL_DIR}" --with-config-file-path="$PHP_INI_DIR" --with-config-file-scan-dir="$CONF_DIR" --enable-option-checking=fatal --with-curl --with-libedit --enable-mbstring --with-openssl --with-zlib --with-password-argon2 --with-sodium=shared "$VERSION_CONFIG" EXTENSION_DIR="$PHP_EXT_DIR"; + + make -j "$(nproc)" + find -type f -name '*.a' -delete + make install + find "${PHP_INSTALL_DIR}" -type f -executable -exec strip --strip-all '{}' + || true + make clean + + cp -v $PHP_SRC_DIR/php.ini-* "$PHP_INI_DIR/"; + cp "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" + + # Install xdebug + "${PHP_INSTALL_DIR}/bin/pecl" install xdebug + XDEBUG_INI="${CONF_DIR}/xdebug.ini" + + echo "zend_extension=${PHP_EXT_DIR}/xdebug.so" > "${XDEBUG_INI}" + echo "xdebug.mode = debug" >> "${XDEBUG_INI}" + echo "xdebug.start_with_request = yes" >> "${XDEBUG_INI}" + echo "xdebug.client_port = 9003" >> "${XDEBUG_INI}" +} + +apt-get purge php.* +PHP_DIR="/usr/local/php" +PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" +PHP_SRC_DIR="/usr/src/php" + +install_php +PHP_SRC="${PHP_INSTALL_DIR}/bin/php" + +updaterc() { + echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." + if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/bash.bashrc + fi + if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/zsh/zshrc + fi +} + +if [ "${PHP_VERSION}" != "none" ]; then + CURRENT_DIR="${PHP_DIR}/current" + if [[ ! -d "${CURRENT_DIR}" ]]; then + ln -s -r "${PHP_INSTALL_DIR}" ${CURRENT_DIR} + fi + + if [[ $(ls -l ${CURRENT_DIR}) != *"-> ${PHP_INSTALL_DIR}"* ]] ; then + rm "${CURRENT_DIR}" + ln -s -r "${PHP_INSTALL_DIR}" "${CURRENT_DIR}" + fi + + rm -rf "${PHP_SRC_DIR}" + updaterc "if [[ \"\${PATH}\" != *\"${CURRENT_DIR}\"* ]]; then export PATH=\"${CURRENT_DIR}/bin:\${PATH}\"; fi" + + chown -R "${USERNAME}:php" "${PHP_DIR}" + chmod -R g+r+w "${PHP_DIR}" + find "${PHP_DIR}" -type d -print0 | xargs -n 1 -0 chmod g+s +fi + +echo -e "\nInstalled PHP Version by Test: ๐Ÿ‘‡ "; php -v; + From 9ccc19e1378ba3b569784236bbb32499dcea138e Mon Sep 17 00:00:00 2001 From: WarrenS Date: Tue, 2 Apr 2024 17:42:58 -0400 Subject: [PATCH 070/285] Added versions 1.71-1.76 to Rust feature version (#929) * Update devcontainer-feature.json * Updated version --- src/rust/devcontainer-feature.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rust/devcontainer-feature.json b/src/rust/devcontainer-feature.json index 7ee455f3f..c4a7dd5b9 100644 --- a/src/rust/devcontainer-feature.json +++ b/src/rust/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "rust", - "version": "1.1.1", + "version": "1.1.2", "name": "Rust", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/rust", "description": "Installs Rust, common Rust utilities, and their required dependencies", @@ -10,6 +10,12 @@ "proposals": [ "latest", "none", + "1.76", + "1.75", + "1.74", + "1.73", + "1.72", + "1.71", "1.70", "1.69", "1.68", From 203dc3f5bde1a8ca25525234757ac54e9b8da64c Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Fri, 5 Apr 2024 04:23:20 +0530 Subject: [PATCH 071/285] oryx dotnet 8.0.1 cleanup (#927) --- src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 1 + test/oryx/test_python_project.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index a075096bf..41bd1f759 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.1", + "version": "1.3.2", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index 407406141..6c193f863 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -241,6 +241,7 @@ if [[ "${PINNED_SDK_VERSION}" != "" ]]; then MAJOR_MINOR_PATCH1_VERSION=${PINNED_SDK_VERSION%??} rm -rf /usr/share/dotnet/shared/Microsoft.NETCore.App/$MAJOR_MINOR_PATCH1_VERSION rm -rf /usr/share/dotnet/shared/Microsoft.AspNetCore.App/$MAJOR_MINOR_PATCH1_VERSION + rm -rf /usr/share/dotnet/templates/$MAJOR_MINOR_PATCH1_VERSION fi diff --git a/test/oryx/test_python_project.sh b/test/oryx/test_python_project.sh index a0d5eef08..610a9687e 100644 --- a/test/oryx/test_python_project.sh +++ b/test/oryx/test_python_project.sh @@ -28,5 +28,7 @@ check "oryx-build-python" oryx build --property python_version="${pythonVersion} check "oryx-build-python-installed" python3 -m pip list | grep mpmath check "oryx-build-python-result" python3 ./src/solve.py +check "templates/8.0.1-does-not-exist" test ! -d "/usr/share/dotnet/templates/8.0.1" + # Report result reportResults From 760f2bf10b30340e55a2cdd9862a994262f13d81 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Louazel Date: Thu, 11 Apr 2024 18:40:05 +0200 Subject: [PATCH 072/285] Set `DEBIAN_FRONTEND=noninteractive` for nvidia-cuda feature (#933) * Set DEBIAN_FRONTEND to noninteractive Signed-off-by: Jean-Baptiste Louazel * Bump nvidia-cuda to 1.1.1 Signed-off-by: Jean-Baptiste Louazel * Install `liburcu6` Signed-off-by: Jean-Baptiste Louazel * Revert "Install `liburcu6`" This reverts commit b7b2931f8bed57826f2019ea884cbe2440e6c9e8. --------- Signed-off-by: Jean-Baptiste Louazel --- src/nvidia-cuda/devcontainer-feature.json | 2 +- src/nvidia-cuda/install.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nvidia-cuda/devcontainer-feature.json b/src/nvidia-cuda/devcontainer-feature.json index 78ad10cd9..bb63ae1c7 100644 --- a/src/nvidia-cuda/devcontainer-feature.json +++ b/src/nvidia-cuda/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "nvidia-cuda", - "version": "1.1.0", + "version": "1.1.1", "name": "NVIDIA CUDA", "description": "Installs shared libraries for NVIDIA CUDA.", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/nvidia-cuda", diff --git a/src/nvidia-cuda/install.sh b/src/nvidia-cuda/install.sh index cb66d3955..d8658964e 100644 --- a/src/nvidia-cuda/install.sh +++ b/src/nvidia-cuda/install.sh @@ -33,6 +33,8 @@ check_packages() { fi } +export DEBIAN_FRONTEND=noninteractive + check_packages wget ca-certificates # Add NVIDIA's package repository to apt so that we can download packages From e7dd9fafd9aeede11d0a59a0ace819e3b774de2a Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Fri, 12 Apr 2024 23:33:24 +0530 Subject: [PATCH 073/285] cp command to follow symlink for systemctl (#937) --- src/common-utils/devcontainer-feature.json | 2 +- src/common-utils/main.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index a058864af..308e256ef 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.4.2", + "version": "2.4.3", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index 8b74830cc..5d7592cfc 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -564,7 +564,7 @@ chmod +rx /usr/local/bin/code # systemctl shim for Debian/Ubuntu - tells people to use 'service' if systemd is not running if [ "${ADJUSTED_ID}" = "debian" ]; then - cp -f "${FEATURE_DIR}/bin/systemctl" /usr/local/bin/systemctl + cp -fL "${FEATURE_DIR}/bin/systemctl" /usr/local/bin/systemctl chmod +rx /usr/local/bin/systemctl fi From b98f5a164be78317af27118e9491d38c17eb16a4 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 17 Apr 2024 04:28:57 +0530 Subject: [PATCH 074/285] [az-cli] - To separate the two methods of installation - using apt, using python (#922) * [az-cli] - To separate the two methods of installation - apt & python * bump patch version * changes as requested by review comments * no need to keep * added test script missing * changes for review comment --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 10 +++++---- ..._using_python_with_python_3_11_bullseye.sh | 21 +++++++++++++++++++ .../install_with_python_3_12_bookworm.sh | 12 +++++------ test/azure-cli/scenarios.json | 10 +++++++++ 5 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 test/azure-cli/install_using_python_with_python_3_11_bullseye.sh diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index f25e5f9a2..6b26ef6fc 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.3", + "version": "1.2.4", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index 7f4c4ff1c..a1b254779 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -15,7 +15,7 @@ rm -rf /var/lib/apt/lists/* AZ_VERSION=${VERSION:-"latest"} AZ_EXTENSIONS=${EXTENSIONS} AZ_INSTALLBICEP=${INSTALLBICEP:-false} -INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-false} +INSTALL_USING_PYTHON=${INSTALLUSINGPYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" @@ -188,13 +188,15 @@ echo "(*) Installing Azure CLI..." . /etc/os-release architecture="$(dpkg --print-architecture)" CACHED_AZURE_VERSION="${AZ_VERSION}" # In case we need to fallback to pip and the apt path has modified the AZ_VERSION variable. -if [[ "${AZCLI_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${AZCLI_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then - install_using_apt || use_pip="true" +if [ "${INSTALL_USING_PYTHON}" != "true" ]; then + if [[ "${AZCLI_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${AZCLI_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then + install_using_apt || use_pip="true" + fi else use_pip="true" fi -if [ "${use_pip}" = "true" ]; then +if [ "${use_pip}" = "true" ]; then AZ_VERSION=${CACHED_AZURE_VERSION} install_using_pip_strategy diff --git a/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh b/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh new file mode 100644 index 000000000..b9957843e --- /dev/null +++ b/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version + +echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"'; then + echo -e "\n\nโœ… Passed 'O.S is Linux 11 (bullseye)'!" +else + echo -e "\n\nโŒ Failed 'O.S is other than Linux 11 (bullseye)'!" +fi + + +# Report result +reportResults \ No newline at end of file diff --git a/test/azure-cli/install_with_python_3_12_bookworm.sh b/test/azure-cli/install_with_python_3_12_bookworm.sh index 074876148..2c8e1fd72 100644 --- a/test/azure-cli/install_with_python_3_12_bookworm.sh +++ b/test/azure-cli/install_with_python_3_12_bookworm.sh @@ -5,17 +5,17 @@ set -e # Import test library for `check` command source dev-container-features-test-lib -# Check to make sure the user is vscode -check "user is vscode" whoami | grep vscode -check "version" az --version -echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +echo -e "\n๐Ÿ”„ Testing 'O.S'" if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"'; then - echo -e "\n\nโœ… Passed 'O.S is Linux 12 (bookworm)'!" + echo -e "\nโœ… Passed 'O.S is Linux 12 (bookworm)'!\n" else - echo -e "\n\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!" + echo -e "\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!\n" fi +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version # Report result reportResults \ No newline at end of file diff --git a/test/azure-cli/scenarios.json b/test/azure-cli/scenarios.json index 041f50731..3ec910399 100644 --- a/test/azure-cli/scenarios.json +++ b/test/azure-cli/scenarios.json @@ -47,5 +47,15 @@ "version": "latest" } } + }, + "install_using_python_with_python_3_11_bullseye": { + "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye", + "user": "vscode", + "features": { + "azure-cli": { + "version": "latest", + "installUsingPython": "true" + } + } } } \ No newline at end of file From 6f4e59866169405c7b7a8ff65e3f2ac3ced6a26e Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 17 Apr 2024 04:35:23 +0530 Subject: [PATCH 075/285] [Ruby]- rvm - fallback code fix (#931) * [Ruby] - Install using fallback - draft * [Ruby] - Rvm - fallback logic implementation * misc changes * changes for review comments.. --- src/ruby/devcontainer-feature.json | 2 +- src/ruby/install.sh | 129 ++++++++++++-- test/ruby/ruby_fallback_test.sh | 277 +++++++++++++++++++++++++++++ test/ruby/scenarios.json | 8 + 4 files changed, 396 insertions(+), 20 deletions(-) create mode 100644 test/ruby/ruby_fallback_test.sh diff --git a/src/ruby/devcontainer-feature.json b/src/ruby/devcontainer-feature.json index 73bcbcced..3722cab06 100644 --- a/src/ruby/devcontainer-feature.json +++ b/src/ruby/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "ruby", - "version": "1.2.0", + "version": "1.2.1", "name": "Ruby (via rvm)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/ruby", "description": "Installs Ruby, rvm, rbenv, common Ruby utilities, and needed dependencies.", diff --git a/src/ruby/install.sh b/src/ruby/install.sh index 7e4514bba..8f95829da 100755 --- a/src/ruby/install.sh +++ b/src/ruby/install.sh @@ -140,6 +140,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + apt_get_update() { if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then @@ -173,25 +214,46 @@ if ! type git > /dev/null 2>&1; then check_packages git fi +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + #install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" "_" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name' | tr '_' '.') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + # Figure out correct version of a three part version number is not passed -find_version_from_git_tags RUBY_VERSION "https://github.com/ruby/ruby" "tags/v" "_" +RUBY_URL="https://github.com/ruby/ruby" +ORIGINAL_RUBY_VERSION=$RUBY_VERSION +find_version_from_git_tags RUBY_VERSION $RUBY_URL "tags/v" "_" -# Just install Ruby if RVM already installed -if rvm --version > /dev/null; then - echo "Ruby Version Manager already exists." - if [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then - echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." - elif [ "${RUBY_VERSION}" != "none" ]; then - echo "Installing specified Ruby version." - su ${USERNAME} -c "rvm install ruby ${RUBY_VERSION}" - fi - SKIP_GEM_INSTALL="false" - SKIP_RBENV_RBUILD="true" -else - # Install RVM - receive_gpg_keys RVM_GPG_KEYS - # Determine appropriate settings for rvm installer +set_rvm_install_args() { + RUBY_VERSION=$1 if [ "${RUBY_VERSION}" = "none" ]; then RVM_INSTALL_ARGS="" elif [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then @@ -210,19 +272,48 @@ else DEFAULT_GEMS="" fi fi +} + +install_previous_version() { + if [[ $ORIGINAL_RUBY_VERSION == "latest" ]]; then + repo_url=$(get_github_api_repo_url "$RUBY_URL") + get_previous_version "${RUBY_URL}" "${repo_url}" RUBY_VERSION + set_rvm_install_args $RUBY_VERSION + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 + else + echo "Failed to install Ruby version $ORIGINAL_RUBY_VERSION. Exiting..." + fi +} + +# Just install Ruby if RVM already installed +if rvm --version > /dev/null; then + echo "Ruby Version Manager already exists." + if [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then + echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." + elif [ "${RUBY_VERSION}" != "none" ]; then + echo "Installing specified Ruby version." + su ${USERNAME} -c "rvm install ruby ${RUBY_VERSION}" + fi + SKIP_GEM_INSTALL="false" + SKIP_RBENV_RBUILD="true" +else + # Install RVM + receive_gpg_keys RVM_GPG_KEYS + # Determine appropriate settings for rvm installer + set_rvm_install_args $RUBY_VERSION # Create rvm group as a system group to reduce the odds of conflict with local user UIDs if ! cat /etc/group | grep -e "^rvm:" > /dev/null 2>&1; then groupadd -r rvm fi # Install rvm - curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 || install_previous_version usermod -aG rvm ${USERNAME} source /usr/local/rvm/scripts/rvm rvm fix-permissions system rm -rf ${GNUPGHOME} fi -if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then +if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then # Non-root user may not have "gem" in path when script is run and no ruby version # is installed by rvm, so handle this by using root's default gem in this case ROOT_GEM="$(which gem || echo "")" @@ -239,7 +330,7 @@ if [ ! -z "${ADDITIONAL_VERSIONS}" ]; then read -a additional_versions <<< "$ADDITIONAL_VERSIONS" for version in "${additional_versions[@]}"; do # Figure out correct version of a three part version number is not passed - find_version_from_git_tags version "https://github.com/ruby/ruby" "tags/v" "_" + find_version_from_git_tags version $RUBY_URL "tags/v" "_" source /usr/local/rvm/scripts/rvm rvm install ruby ${version} done diff --git a/test/ruby/ruby_fallback_test.sh b/test/ruby/ruby_fallback_test.sh new file mode 100644 index 000000000..fe1a9c77b --- /dev/null +++ b/test/ruby/ruby_fallback_test.sh @@ -0,0 +1,277 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +USERNAME="automatic" +echo -e "\nRVM version installed previously by ruby feature ..." +check "rvm" rvm --version +check "ruby" ruby -v + +trap 'echo "Last executed command failed at line ${LINENO}"' ERR + +RVM_GPG_KEYS="409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB" +GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com +keyserver hkp://keyserver.ubuntu.com:80 +keyserver hkps://keys.openpgp.org +keyserver hkp://keyserver.pgp.com" + +# Clean up +rm -rf /var/lib/apt/lists/* + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +architecture="$(uname -m)" +if [ "${architecture}" != "amd64" ] && [ "${architecture}" != "x86_64" ] && [ "${architecture}" != "arm64" ] && [ "${architecture}" != "aarch64" ]; then + echo "(!) Architecture $architecture unsupported" + exit 1 +fi + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Import the specified key in a variable name passed in as +receive_gpg_keys() { + local keys=${!1} + local keyring_args="" + if [ ! -z "$2" ]; then + keyring_args="--no-default-keyring --keyring \"$2\"" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | tee ${GNUPGHOME}/dirmngr.conf > /dev/null + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + set +e + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + #install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $mode == "mode1" ]]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" "_" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name' | tr '_' '.') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + + +# Figure out correct version of a three part version number is not passed +ruby_url="https://github.com/ruby/ruby" + +RUBY_VERSION="3.1.xyz" + +set_rvm_install_args() { + RUBY_VERSION=$1 + if [ "${RUBY_VERSION}" = "none" ]; then + RVM_INSTALL_ARGS="" + elif [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then + echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." + RVM_INSTALL_ARGS="" + else + if [ "${RUBY_VERSION}" = "latest" ] || [ "${RUBY_VERSION}" = "current" ] || [ "${RUBY_VERSION}" = "lts" ]; then + RVM_INSTALL_ARGS="--ruby" + RUBY_VERSION="" + else + RVM_INSTALL_ARGS="--ruby=${RUBY_VERSION}" + fi + if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then + SKIP_GEM_INSTALL="true" + else + DEFAULT_GEMS="" + fi + fi +} + +install_previous_version() { + mode=$1 + repo_url=$(get_github_api_repo_url "$ruby_url") + get_previous_version "${ruby_url}" "${repo_url}" RUBY_VERSION $mode + set_rvm_install_args $RUBY_VERSION + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 +} + +install_rvm() { + mode=$1 + # Install RVM + receive_gpg_keys RVM_GPG_KEYS + # Determine appropriate settings for rvm installer + set_rvm_install_args $RUBY_VERSION + # Create rvm group as a system group to reduce the odds of conflict with local user UIDs + if ! cat /etc/group | grep -e "^rvm:" > /dev/null 2>&1; then + groupadd -r rvm + fi + # Install rvm + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 || install_previous_version "$mode" + sudo usermod -aG rvm ${USERNAME} + source /usr/local/rvm/scripts/rvm + rvm fix-permissions system + rm -rf ${GNUPGHOME} +} + +install_rvm "mode1" +echo -e "\n๐Ÿ‘‰๐Ÿป๐Ÿ‘‰๐ŸปRVM version installed by test file ... (mode: 1 - install using find_prev_version_from_git_tags):" +check "rvm" rvm --version + +install_rvm "mode2" +echo -e "\n๐Ÿ‘‰๐Ÿป๐Ÿ‘‰๐ŸปRVM version installed by test file ... (mode: 1 - install using GitHub Api):" +check "rvm" rvm --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/ruby/scenarios.json b/test/ruby/scenarios.json index 04cac73f5..cfa2d8554 100644 --- a/test/ruby/scenarios.json +++ b/test/ruby/scenarios.json @@ -13,5 +13,13 @@ "features": { "ruby": {} } + }, + "ruby_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:bullseye", + "features": { + "ruby": { + "version": "latest" + } + } } } \ No newline at end of file From bb7b7ea29f84ea31262c1290a2e14e9984286295 Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:50:07 +0530 Subject: [PATCH 076/285] upgrade cuda version to 11.7 and cudnn to 8.5.0 (#942) upgrade cuda version 11.7 and cudnn 8.5.0 --- src/nvidia-cuda/devcontainer-feature.json | 2 +- src/nvidia-cuda/install.sh | 7 +++++++ test/nvidia-cuda/install_cudnn_nvxt_version.sh | 8 ++++---- test/nvidia-cuda/scenarios.json | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/nvidia-cuda/devcontainer-feature.json b/src/nvidia-cuda/devcontainer-feature.json index bb63ae1c7..4a0fb0834 100644 --- a/src/nvidia-cuda/devcontainer-feature.json +++ b/src/nvidia-cuda/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "nvidia-cuda", - "version": "1.1.1", + "version": "1.1.2", "name": "NVIDIA CUDA", "description": "Installs shared libraries for NVIDIA CUDA.", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/nvidia-cuda", diff --git a/src/nvidia-cuda/install.sh b/src/nvidia-cuda/install.sh index d8658964e..79a7a9a20 100644 --- a/src/nvidia-cuda/install.sh +++ b/src/nvidia-cuda/install.sh @@ -12,6 +12,8 @@ INSTALL_TOOLKIT=${INSTALLTOOLKIT} CUDA_VERSION=${CUDAVERSION} CUDNN_VERSION=${CUDNNVERSION} +. /etc/os-release + if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 @@ -33,6 +35,11 @@ check_packages() { fi } +if [ $VERSION_CODENAME = "bookworm" ] || [ $VERSION_CODENAME = "jammy" ] && [ $CUDA_VERSION \< 11.7 ]; then + echo "(!) Unsupported distribution version '${VERSION_CODENAME}' for CUDA < 11.7" + exit 1 +fi + export DEBIAN_FRONTEND=noninteractive check_packages wget ca-certificates diff --git a/test/nvidia-cuda/install_cudnn_nvxt_version.sh b/test/nvidia-cuda/install_cudnn_nvxt_version.sh index a7f46bdd6..4817ae479 100644 --- a/test/nvidia-cuda/install_cudnn_nvxt_version.sh +++ b/test/nvidia-cuda/install_cudnn_nvxt_version.sh @@ -5,11 +5,11 @@ set -e # Optional: Import test library source dev-container-features-test-lib -# Check installation of libcudnn8 (8.3.2) -check "libcudnn.so.8.3.2" test 1 -eq "$(find /usr -name 'libcudnn.so.8.3.2' | wc -l)" +# Check installation of libcudnn8 (8.5.0) +check "libcudnn.so.8.5.0" test 1 -eq "$(find /usr -name 'libcudnn.so.8.5.0' | wc -l)" -# Check installation of cuda-nvtx-11-5 (11.5) -check "cuda-11-5+nvtx" test -e '/usr/local/cuda-11.5/targets/x86_64-linux/include/nvtx3' +# Check installation of cuda-nvtx-11-7 (11.7) +check "cuda-11-7+nvtx" test -e '/usr/local/cuda-11.7/targets/x86_64-linux/include/nvtx3' # Report result reportResults diff --git a/test/nvidia-cuda/scenarios.json b/test/nvidia-cuda/scenarios.json index 3018330e2..bd73263b3 100644 --- a/test/nvidia-cuda/scenarios.json +++ b/test/nvidia-cuda/scenarios.json @@ -16,8 +16,8 @@ "nvidia-cuda": { "installCudnn": true, "installNvtx": true, - "cudaVersion": "11.5", - "cudnnVersion": "8.3.2.44" + "cudaVersion": "11.7", + "cudnnVersion": "8.5.0.96" } } } From d8e9d335952d0b8d488cc3445ad076003bc3c22c Mon Sep 17 00:00:00 2001 From: Barrett Strausser Date: Tue, 7 May 2024 11:10:39 -0400 Subject: [PATCH 077/285] Fix rustup-init sha256sum check (#962) * Fix rustup-init sha256sum check * Semver update --------- Co-authored-by: bstrausser --- src/rust/devcontainer-feature.json | 2 +- src/rust/install.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rust/devcontainer-feature.json b/src/rust/devcontainer-feature.json index c4a7dd5b9..70013442f 100644 --- a/src/rust/devcontainer-feature.json +++ b/src/rust/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "rust", - "version": "1.1.2", + "version": "1.1.3", "name": "Rust", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/rust", "description": "Installs Rust, common Rust utilities, and their required dependencies", diff --git a/src/rust/install.sh b/src/rust/install.sh index 00c0a6e72..4db9edc1e 100755 --- a/src/rust/install.sh +++ b/src/rust/install.sh @@ -186,6 +186,7 @@ else curl -sSL --proto '=https' --tlsv1.2 "https://static.rust-lang.org/rustup/dist/${download_architecture}-unknown-linux-gnu/rustup-init" -o /tmp/rustup/target/${download_architecture}-unknown-linux-gnu/release/rustup-init curl -sSL --proto '=https' --tlsv1.2 "https://static.rust-lang.org/rustup/dist/${download_architecture}-unknown-linux-gnu/rustup-init.sha256" -o /tmp/rustup/rustup-init.sha256 cd /tmp/rustup + cp /tmp/rustup/target/${download_architecture}-unknown-linux-gnu/release/rustup-init /tmp/rustup/rustup-init sha256sum -c rustup-init.sha256 chmod +x target/${download_architecture}-unknown-linux-gnu/release/rustup-init target/${download_architecture}-unknown-linux-gnu/release/rustup-init -y --no-modify-path --profile ${RUSTUP_PROFILE} ${default_toolchain_arg} From 67c10a660868260c284b02f2878dcfdcfd91279f Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 13 May 2024 17:37:11 +0100 Subject: [PATCH 078/285] =?UTF-8?q?=E2=9C=A8=20Add=20Ubuntu=2024=20Noble?= =?UTF-8?q?=20to=20`docker-in-docker`=20(#971)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Ubuntu Noble Signed-off-by: GitHub * Changes Signed-off-by: GitHub --------- Signed-off-by: GitHub --- .github/workflows/test-all.yaml | 1 + .github/workflows/test-pr.yaml | 1 + src/docker-in-docker/devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index 40f5efa75..7a9554314 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -48,6 +48,7 @@ jobs: "debian:12", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", + "mcr.microsoft.com/devcontainers/base:noble" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 3aae841e2..776a9731f 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -55,6 +55,7 @@ jobs: "debian:12", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", + "mcr.microsoft.com/devcontainers/base:noble" ] steps: - uses: actions/checkout@v3 diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 812db444c..4897ebf3e 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.2", + "version": "2.11.0", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 0dc9e52d1..ee9cb6ee6 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -18,8 +18,8 @@ USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" INSTALL_DOCKER_COMPOSE_SWITCH="${INSTALLDOCKERCOMPOSESWITCH:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" -DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy noble" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy noble" # Default: Exit on any failure. set -e From 4d2dabec57722e23ffe547038923a799e332e291 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 17 May 2024 00:12:03 +0530 Subject: [PATCH 079/285] [Desktop-lite]- libasound2 not installing in noble - issue (#973) * [Desktop-lite]- libasound2 not installing in noble - issue * bump to patch version in Desktop-lite feature * misc change * Changes for comments ( review comments ) * changes based on review comments.. --- src/desktop-lite/devcontainer-feature.json | 2 +- src/desktop-lite/install.sh | 11 +++++++- test/desktop-lite/test.sh | 30 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/desktop-lite/devcontainer-feature.json b/src/desktop-lite/devcontainer-feature.json index 5387138b5..5386eb31d 100644 --- a/src/desktop-lite/devcontainer-feature.json +++ b/src/desktop-lite/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "desktop-lite", - "version": "1.0.8", + "version": "1.1.0", "name": "Light-weight Desktop", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/desktop-lite", "description": "Adds a lightweight Fluxbox based desktop to the container that can be accessed using a VNC viewer or the web. GUI-based commands executed from the built-in VS code terminal will open on the desktop automatically.", diff --git a/src/desktop-lite/install.sh b/src/desktop-lite/install.sh index df4390eca..13a524ada 100755 --- a/src/desktop-lite/install.sh +++ b/src/desktop-lite/install.sh @@ -41,7 +41,6 @@ package_list=" libnotify4 \ libnss3 \ libxss1 \ - libasound2 \ xfonts-base \ xfonts-terminus \ fonts-noto \ @@ -198,6 +197,16 @@ fi # Install X11, fluxbox and VS Code dependencies check_packages ${package_list} +# if Ubuntu-24.04, noble(numbat) found, then will install libasound2-dev instead of libasound2. +# this change is temporary, https://packages.ubuntu.com/noble/libasound2 will switch to libasound2 once it is available for Ubuntu-24.04, noble(numbat) +. /etc/os-release +if [ "${ID}" = "ubuntu" ] && [ "${VERSION_CODENAME}" = "noble" ]; then + echo "Ubuntu 24.04, Noble(Numbat) detected. Installing libasound2-dev package..." + check_packages "libasound2-dev" +else + check_packages "libasound2" +fi + # On newer versions of Ubuntu (22.04), # we need an additional package that isn't provided in earlier versions if ! type vncpasswd > /dev/null 2>&1; then diff --git a/test/desktop-lite/test.sh b/test/desktop-lite/test.sh index 9009aa9c6..5d11dd424 100755 --- a/test/desktop-lite/test.sh +++ b/test/desktop-lite/test.sh @@ -5,9 +5,39 @@ set -e # Optional: Import test library source dev-container-features-test-lib +echoStderr() +{ + echo "$@" 1>&2 +} + +checkOSPackage() { + LABEL=$1 + PACKAGE_NAME=$2 + echo -e "\n๐Ÿงช Testing $LABEL" + # Check if the package exists and retrieve its exact version + if [ "$(dpkg-query -W -f='${Status}' "$PACKAGE_NAME" 2>/dev/null | grep -c "ok installed")" -eq 1 ]; then + echo "โœ… Package '$PACKAGE_NAME' is installed." + exit 0 + else + echo "โŒ Package '$PACKAGE_NAME' is not installed." + exit 1 + fi +} + check "desktop-init-exists" bash -c "ls /usr/local/share/desktop-init.sh" check "log-exists" bash -c "ls /tmp/container-init.log" check "fluxbox-exists" bash -c "ls -la ~/.fluxbox" +. /etc/os-release +if [ "${ID}" = "ubuntu" ]; then + if [ "${VERSION_CODENAME}" = "noble" ]; then + checkOSPackage "if libasound2-dev exists !" "libasound2-dev" + else + checkOSPackage "if libasound2 exists !" "libasound2" + fi +else + checkOSPackage "if libasound2 exists !" "libasound2" +fi + # Report result reportResults \ No newline at end of file From ecbfd50952e513db872d8d3380e069ccf74a70a8 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Fri, 17 May 2024 09:16:22 -0700 Subject: [PATCH 080/285] [Updates] Automated vendor dotnet-install script (#970) * Automated dotnet-install script update * Bump version --------- Co-authored-by: github-actions --- src/dotnet/devcontainer-feature.json | 2 +- src/dotnet/scripts/vendor/dotnet-install.sh | 26 +++++++++++++++++---- src/oryx/devcontainer-feature.json | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 78a061d23..8b8ffaf2a 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.5", + "version": "2.0.6", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", diff --git a/src/dotnet/scripts/vendor/dotnet-install.sh b/src/dotnet/scripts/vendor/dotnet-install.sh index f6b08d1f8..42c201af4 100755 --- a/src/dotnet/scripts/vendor/dotnet-install.sh +++ b/src/dotnet/scripts/vendor/dotnet-install.sh @@ -298,6 +298,10 @@ get_machine_architecture() { if command -v uname > /dev/null; then CPUName=$(uname -m) case $CPUName in + armv1*|armv2*|armv3*|armv4*|armv5*|armv6*) + echo "armv6-or-below" + return 0 + ;; armv*l) echo "arm" return 0 @@ -339,7 +343,13 @@ get_normalized_architecture_from_architecture() { local architecture="$(to_lowercase "$1")" if [[ $architecture == \ ]]; then - echo "$(get_machine_architecture)" + machine_architecture="$(get_machine_architecture)" + if [[ "$machine_architecture" == "armv6-or-below" ]]; then + say_err "Architecture \`$machine_architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" + return 1 + fi + + echo $machine_architecture return 0 fi @@ -1013,7 +1023,7 @@ extract_dotnet_package() { rm -rf "$temp_out_path" if [ -z ${keep_zip+x} ]; then - rm -f "$zip_path" && say_verbose "Temporary zip file $zip_path was removed" + rm -f "$zip_path" && say_verbose "Temporary archive file $zip_path was removed" fi if [ "$failed" = true ]; then @@ -1261,6 +1271,12 @@ get_download_link_from_aka_ms() { http_codes=$( echo "$response" | awk '$1 ~ /^HTTP/ {print $2}' ) # They all need to be 301, otherwise some links are broken (except for the last, which is not a redirect but 200 or 404). broken_redirects=$( echo "$http_codes" | sed '$d' | grep -v '301' ) + # The response may end without final code 2xx/4xx/5xx somehow, e.g. network restrictions on www.bing.com causes redirecting to bing.com fails with connection refused. + # In this case it should not exclude the last. + last_http_code=$( echo "$http_codes" | tail -n 1 ) + if ! [[ $last_http_code =~ ^(2|4|5)[0-9][0-9]$ ]]; then + broken_redirects=$( echo "$http_codes" | grep -v '301' ) + fi # All HTTP codes are 301 (Moved Permanently), the redirect link exists. if [[ -z "$broken_redirects" ]]; then @@ -1512,7 +1528,7 @@ install_dotnet() { mkdir -p "$install_root" zip_path="${zip_path:-$(mktemp "$temporary_file_template")}" - say_verbose "Zip path: $zip_path" + say_verbose "Archive path: $zip_path" for link_index in "${!download_links[@]}" do @@ -1536,7 +1552,7 @@ install_dotnet() { say "Failed to download $link_type link '$download_link': $download_error_msg" ;; esac - rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" + rm -f "$zip_path" 2>&1 && say_verbose "Temporary archive file $zip_path was removed" else download_completed=true break @@ -1551,7 +1567,7 @@ install_dotnet() { remote_file_size="$(get_remote_file_size "$download_link")" - say "Extracting zip from $download_link" + say "Extracting archive from $download_link" extract_dotnet_package "$zip_path" "$install_root" "$remote_file_size" || return 1 # Check if the SDK version is installed; if not, fail the installation. diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index 41bd1f759..9e6e698a1 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.2", + "version": "1.3.3", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", From f5787eed01022f177475a99084327e023a84ddaf Mon Sep 17 00:00:00 2001 From: Andy Li Date: Wed, 22 May 2024 00:57:40 +0100 Subject: [PATCH 081/285] Add Ubuntu 24 Noble to `docker-outside-of-docker` (#978) --- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 2506031ae..19018ea8e 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.5", + "version": "1.5.0", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 16ad15f60..de5212fe6 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -19,8 +19,8 @@ USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" -DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy noble" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy noble" set -e From 02b71cbd6cf972ca29059cd409dc0fe8c3b60e65 Mon Sep 17 00:00:00 2001 From: hellodword <46193371+hellodword@users.noreply.github.com> Date: Tue, 28 May 2024 23:03:23 +0000 Subject: [PATCH 082/285] [python] add default formatter (#903) Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 9f211244f..ef16ee643 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.4.2", + "version": "1.5.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", @@ -73,10 +73,14 @@ "vscode": { "extensions": [ "ms-python.python", - "ms-python.vscode-pylance" + "ms-python.vscode-pylance", + "ms-python.autopep8" ], "settings": { - "python.defaultInterpreterPath": "/usr/local/python/current/bin/python" + "python.defaultInterpreterPath": "/usr/local/python/current/bin/python", + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8" + } } } }, @@ -84,4 +88,4 @@ "ghcr.io/devcontainers/features/common-utils", "ghcr.io/devcontainers/features/oryx" ] -} \ No newline at end of file +} From 10ea0b7dd5a653b08266039527f7e095c02591a8 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 28 May 2024 16:20:00 -0700 Subject: [PATCH 083/285] Automated documentation update (#984) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/python/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/README.md b/src/python/README.md index db6ad1039..90d79aee8 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -31,6 +31,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho - `ms-python.python` - `ms-python.vscode-pylance` +- `ms-python.autopep8` From 476a68d0523b004112498dc161c2b6de1bd9fe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Wed, 29 May 2024 21:43:03 +0200 Subject: [PATCH 084/285] [desktop-lite] Allow password-less VNC connections (#982) * [desktop-lite] Allow password-less VNC connections Closes #611 * Restore readme * Update src/desktop-lite/install.sh Co-authored-by: Samruddhi Khandale * Fix --------- Co-authored-by: Samruddhi Khandale --- src/desktop-lite/devcontainer-feature.json | 13 +++++----- src/desktop-lite/install.sh | 29 +++++++++++++++++++--- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/desktop-lite/devcontainer-feature.json b/src/desktop-lite/devcontainer-feature.json index 5386eb31d..417787c51 100644 --- a/src/desktop-lite/devcontainer-feature.json +++ b/src/desktop-lite/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "desktop-lite", - "version": "1.1.0", + "version": "1.2.0", "name": "Light-weight Desktop", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/desktop-lite", "description": "Adds a lightweight Fluxbox based desktop to the container that can be accessed using a VNC viewer or the web. GUI-based commands executed from the built-in VS code terminal will open on the desktop automatically.", @@ -19,17 +19,18 @@ "1.2.0" ], "default": "1.2.0", - "description": "NoVnc Version" + "description": "The noVNC version to use" }, "password": { "type": "string", "proposals": [ "vscode", "codespaces", - "password" + "password", + "noPassword" ], "default": "vscode", - "description": "Enter a password for desktop connections" + "description": "Enter a password for desktop connections. If \"noPassword\", connections from the local host can be established without entering a password" }, "webPort": { "type": "string", @@ -37,7 +38,7 @@ "6080" ], "default": "6080", - "description": "Enter a port for the VNC web client" + "description": "Enter a port for the VNC web client (noVNC)" }, "vncPort": { "type": "string", @@ -45,7 +46,7 @@ "5901" ], "default": "5901", - "description": "Enter a port for the desktop VNC server" + "description": "Enter a port for the desktop VNC server (TigerVNC)" } }, "init": true, diff --git a/src/desktop-lite/install.sh b/src/desktop-lite/install.sh index 13a524ada..ef8b603c6 100755 --- a/src/desktop-lite/install.sh +++ b/src/desktop-lite/install.sh @@ -9,6 +9,9 @@ NOVNC_VERSION="${NOVNCVERSION:-"1.2.0"}" # TODO: Add in a 'latest' auto-detect and swap name to 'version' VNC_PASSWORD=${PASSWORD:-"vscode"} +if [ "$VNC_PASSWORD" = "noPassword" ]; then + unset VNC_PASSWORD +fi NOVNC_PORT="${WEBPORT:-6080}" VNC_PORT="${VNCPORT:-5901}" @@ -372,7 +375,15 @@ sudoIf chown root:\${group_name} /tmp/.X11-unix if [ "\$(echo "\${VNC_RESOLUTION}" | tr -cd 'x' | wc -c)" = "1" ]; then VNC_RESOLUTION=\${VNC_RESOLUTION}x16; fi screen_geometry="\${VNC_RESOLUTION%*x*}" screen_depth="\${VNC_RESOLUTION##*x}" -startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "tigervncserver \${DISPLAY} -geometry \${screen_geometry} -depth \${screen_depth} -rfbport ${VNC_PORT} -dpi \${VNC_DPI:-96} -localhost -desktop fluxbox -fg -passwd /usr/local/etc/vscode-dev-containers/vnc-passwd" + +# Check if VNC_PASSWORD is set and use the appropriate command +common_options="tigervncserver \${DISPLAY} -geometry \${screen_geometry} -depth \${screen_depth} -rfbport ${VNC_PORT} -dpi \${VNC_DPI:-96} -localhost -desktop fluxbox -fg" + +if [ -n "\${VNC_PASSWORD+x}" ]; then + startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "\${common_options} -passwd /usr/local/etc/vscode-dev-containers/vnc-passwd" +else + startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "\${common_options} -SecurityTypes None" +fi # Spin up noVNC if installed and not running. if [ -d "/usr/local/novnc" ] && [ "\$(ps -ef | grep /usr/local/novnc/noVNC*/utils/launch.sh | grep -v grep)" = "" ]; then @@ -388,7 +399,9 @@ exec "\$@" log "** SCRIPT EXIT **" EOF -echo "${VNC_PASSWORD}" | vncpasswd -f > /usr/local/etc/vscode-dev-containers/vnc-passwd +if [ -n "${VNC_PASSWORD+x}" ]; then + echo "${VNC_PASSWORD}" | vncpasswd -f > /usr/local/etc/vscode-dev-containers/vnc-passwd +fi chmod +x /usr/local/share/desktop-init.sh /usr/local/bin/set-resolution # Set up fluxbox config @@ -401,15 +414,23 @@ fi # Clean up rm -rf /var/lib/apt/lists/* +# Determine the message based on whether VNC_PASSWORD is set +if [ -n "${VNC_PASSWORD+x}" ]; then + PASSWORD_MESSAGE="In both cases, use the password \"${VNC_PASSWORD}\" when connecting" +else + PASSWORD_MESSAGE="In both cases, no password is required." +fi + +# Display the message cat << EOF You now have a working desktop! Connect to in one of the following ways: -- Forward port ${NOVNC_PORT} and use a web browser start the noVNC client (recommended) +- Forward port ${NOVNC_PORT} and use a web browser to start the noVNC client (recommended) - Forward port ${VNC_PORT} using VS Code client and connect using a VNC Viewer -In both cases, use the password "${VNC_PASSWORD}" when connecting +${PASSWORD_MESSAGE} (*) Done! From 32797f4f693a3bf18a18928ad28bc55589a7ada6 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Wed, 29 May 2024 12:54:52 -0700 Subject: [PATCH 085/285] Automated documentation update (#989) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/desktop-lite/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/desktop-lite/README.md b/src/desktop-lite/README.md index 7094a2425..6f2d67ce2 100644 --- a/src/desktop-lite/README.md +++ b/src/desktop-lite/README.md @@ -16,10 +16,10 @@ Adds a lightweight Fluxbox based desktop to the container that can be accessed u | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Currently Unused! | string | latest | -| noVncVersion | NoVnc Version | string | 1.2.0 | -| password | Enter a password for desktop connections | string | vscode | -| webPort | Enter a port for the VNC web client | string | 6080 | -| vncPort | Enter a port for the desktop VNC server | string | 5901 | +| noVncVersion | The noVNC version to use | string | 1.2.0 | +| password | Enter a password for desktop connections. If "noPassword", connections from the local host can be established without entering a password | string | vscode | +| webPort | Enter a port for the VNC web client (noVNC) | string | 6080 | +| vncPort | Enter a port for the desktop VNC server (TigerVNC) | string | 5901 | ## Connecting to the desktop From c1df45b189afae33be72c1ce478452fbfc044abe Mon Sep 17 00:00:00 2001 From: Prabhakar Kumar <64955767+prabhakk-mw@users.noreply.github.com> Date: Thu, 30 May 2024 03:00:18 +0530 Subject: [PATCH 086/285] Adds /home/USER/.local/bin/ to PATH in /etc/sudoers.d/vscode, (#887) * Adds /home/USER/.local/bin/ to PATH in /etc/sudoers.d/vscode, fixes devcontainers/features#870 * Bumping up version to 1.4.2 * Adds to sudoers file if already present * Tests to ensure Default secure_path is not overwritten * Bump to version 1.4.4 * Fix version as 1.4.3 Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Prabhakar Kumar <64955767+prabhakk-mw@users.noreply.github.com> * Update src/python/devcontainer-feature.json --------- Co-authored-by: Samruddhi Khandale Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 57 ++++++++++++------- test/python/install_jupyterlab.sh | 3 + ...nstall_jupyterlab_existing_sudoers_file.sh | 36 ++++++++++++ .../Dockerfile | 5 ++ .../sudoers.test | 2 + test/python/scenarios.json | 13 +++++ 7 files changed, 97 insertions(+), 21 deletions(-) create mode 100755 test/python/install_jupyterlab_existing_sudoers_file.sh create mode 100644 test/python/install_jupyterlab_existing_sudoers_file/Dockerfile create mode 100644 test/python/install_jupyterlab_existing_sudoers_file/sudoers.test diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index ef16ee643..7c2c6200a 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.5.0", + "version": "1.6.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index e8a9d24e1..1aad3f631 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -130,7 +130,7 @@ updaterc() { fi } -# Import the specified key in a variable name passed in as +# Import the specified key in a variable name passed in as receive_gpg_keys() { local keys=${!1} local keyring_args="" @@ -152,7 +152,7 @@ receive_gpg_keys() { local retry_count=0 local gpg_ok="false" set +e - until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do echo "(*) Downloading GPG key..." ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" @@ -222,7 +222,7 @@ find_version_from_git_tags() { local repository=$2 local prefix=${3:-"tags/v"} local separator=${4:-"."} - local last_part_optional=${5:-"false"} + local last_part_optional=${5:-"false"} if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator=${separator//./\\.} local last_part @@ -282,7 +282,7 @@ find_prev_version_from_git_tags() { ((breakfix=breakfix-1)) if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then declare -g ${variable_name}="${major}.${minor}" - else + else declare -g ${variable_name}="${major}.${minor}.${breakfix}" fi fi @@ -378,13 +378,13 @@ check_packages() { add_symlink() { if [[ ! -d "${CURRENT_PATH}" ]]; then - ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" fi if [ "${OVERRIDE_DEFAULT_VERSION}" = "true" ]; then if [[ $(ls -l ${CURRENT_PATH}) != *"-> ${INSTALL_PATH}"* ]] ; then rm "${CURRENT_PATH}" - ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" fi fi } @@ -397,7 +397,7 @@ install_openssl3() { openssl3_version="3.0" # Find version using soft match find_version_from_git_tags openssl3_version "https://github.com/openssl/openssl" "openssl-" - local tgz_filename="openssl-${openssl3_version}.tar.gz" + local tgz_filename="openssl-${openssl3_version}.tar.gz" local tgz_url="https://github.com/openssl/openssl/releases/download/openssl-${openssl3_version}/${tgz_filename}" echo "Downloading ${tgz_filename}..." curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" @@ -434,7 +434,7 @@ install_cpython() { } install_from_source() { - VERSION=$1 + VERSION=$1 echo "(*) Building Python ${VERSION} from source..." if ! type git > /dev/null 2>&1; then check_packages git @@ -444,7 +444,7 @@ install_from_source() { find_version_from_git_tags VERSION "https://github.com/python/cpython" # Some platforms/os versions need modern versions of openssl installed - # via common package repositories, for now rhel-7 family, use case statement to + # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand case ${VERSION_CODENAME} in centos7|rhel7) @@ -455,7 +455,7 @@ install_from_source() { esac install_cpython "${VERSION}" - if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then + if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then if grep -q "404 Not Found" "/tmp/python-src/${cpython_tgz_filename}"; then install_prev_vers_cpython "${VERSION}" fi @@ -512,9 +512,9 @@ install_from_source() { } install_using_oryx() { - VERSION=$1 + VERSION=$1 INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" - + if [ -d "${INSTALL_PATH}" ]; then echo "(!) Python version ${VERSION} already exists." exit 1 @@ -727,7 +727,7 @@ if [ "${PYTHON_VERSION}" != "none" ]; then usermod -a -G python "${USERNAME}" CURRENT_PATH="${PYTHON_INSTALL_PATH}/current" - + install_python ${PYTHON_VERSION} # Additional python versions to be installed but not be set as default. @@ -748,7 +748,7 @@ if [ "${PYTHON_VERSION}" != "none" ]; then updaterc "if [[ \"\${PATH}\" != *\"${CURRENT_PATH}/bin\"* ]]; then export PATH=${CURRENT_PATH}/bin:\${PATH}; fi" PATH="${INSTALL_PATH}/bin:${PATH}" fi - + # Updates the symlinks for os-provided, or the installed python version in other cases chown -R "${USERNAME}:python" "${PYTHON_INSTALL_PATH}" chmod -R g+r+w "${PYTHON_INSTALL_PATH}" @@ -776,7 +776,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then umask 0002 mkdir -p ${PIPX_BIN_DIR} chown -R "${USERNAME}:pipx" ${PIPX_HOME} - chmod -R g+r+w "${PIPX_HOME}" + chmod -R g+r+w "${PIPX_HOME}" find "${PIPX_HOME}" -type d -print0 | xargs -0 -n 1 chmod g+s # Update pip if not using os provided python @@ -805,21 +805,21 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then echo "${util} already installed. Skipping." fi done - + # Temporary: Removes โ€œsetup toolsโ€ metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 - if [[ $SKIP_VULNERABILITY_PATCHING = "false" ]]; then + if [[ $SKIP_VULNERABILITY_PATCHING = "false" ]]; then VULNERABLE_VERSIONS=("3.10" "3.11") RUN_TIME_PY_VER_DETECT=$(${PYTHON_SRC} --version 2>&1) PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info - if [[ -e "/usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" ]]; then + if [[ -e "/usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" ]]; then # remove the vulnerable setuptools-65.5.0-py3-none-any.whl file rm /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl # create and change to the setuptools_downloaded directory mkdir -p /tmp/setuptools_downloaded cd /tmp/setuptools_downloaded - # download the source distribution for setuptools using pip + # download the source distribution for setuptools using pip pip download setuptools==65.5.1 --no-binary :all: # extract the filename of the setuptools-*.tar.gz file filename=$(find . -maxdepth 1 -type f) @@ -833,7 +833,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then python setup.py bdist_wheel # move inside the dist directory in pwd cd dist - # copy this file to the ensurepip/_bundled directory + # copy this file to the ensurepip/_bundled directory cp setuptools-65.5.1-py3-none-any.whl /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/ # replace the version in __init__.py file with the installed version sed -i 's/_SETUPTOOLS_VERSION = \"65\.5\.0\"/_SETUPTOOLS_VERSION = "65.5.1"/g' /usr/local/lib/"python${PY_MAJOR_MINOR_VER}"/ensurepip/__init__.py @@ -865,6 +865,23 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then install_user_package $INSTALL_UNDER_ROOT jupyterlab install_user_package $INSTALL_UNDER_ROOT jupyterlab-git + if [ "$INSTALL_UNDER_ROOT" = false ]; then + # JupyterLab would have installed into /home/${USERNAME}/.local/bin + # Adding it to default path for Codespaces which use non-login shells + SUDOERS_FILE="/etc/sudoers.d/$USERNAME" + SEARCH_STR="Defaults secure_path=" + REPLACE_STR="Defaults secure_path=/home/${USERNAME}/.local/bin" + + if grep -qs ${SEARCH_STR} ${SUDOERS_FILE}; then + # string found and file is present + sed -i "s|${SEARCH_STR}|${REPLACE_STR}:|g" "${SUDOERS_FILE}" + else + # either string is not found, or file is not present + # In either case take same action, note >> places at end of file + echo "${REPLACE_STR}:${PATH}" >> ${SUDOERS_FILE} + fi + fi + # Configure JupyterLab if needed if [ -n "${CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN}" ]; then # Resolve config directory diff --git a/test/python/install_jupyterlab.sh b/test/python/install_jupyterlab.sh index 58c4f7f54..033b44edd 100755 --- a/test/python/install_jupyterlab.sh +++ b/test/python/install_jupyterlab.sh @@ -22,5 +22,8 @@ check "jupyterlab_git" grep jupyterlab_git <<< "$packages" # Check for correct JupyterLab configuration check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py +# Check for PATH modification +check "default path has jupyterlab" sudo grep "/home/${user}/.local/bin" /etc/sudoers.d/$user + # Report result reportResults diff --git a/test/python/install_jupyterlab_existing_sudoers_file.sh b/test/python/install_jupyterlab_existing_sudoers_file.sh new file mode 100755 index 000000000..22bbc2f92 --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Check for correct JupyterLab configuration +check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py + +# Check for PATH modification +check "default path has jupyterlab" grep "Defaults secure_path=/home/${user}/.local/bin" /etc/sudoers.d/$user + +# Check if previous PATH exists +check "existing default path is preserved" grep "Defaults secure_path=.*original_content_of_sudoers_file" /etc/sudoers.d/$user + +# Check if PATH modification includes original and new paths +check "existing path included with jupyterlab" grep "Defaults secure_path.*/home/${user}/.local/bin.*original_content_of_sudoers_file" /etc/sudoers.d/$user + + +# Report result +reportResults diff --git a/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile b/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile new file mode 100644 index 000000000..5acb58a37 --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile @@ -0,0 +1,5 @@ +# Builds an image with a preconfigured SUDOERS file +# Used to test the install script for JupyterLab which modifies this file +FROM mcr.microsoft.com/devcontainers/base:focal + +COPY --chown=root sudoers.test /etc/sudoers.d/vscode diff --git a/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test b/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test new file mode 100644 index 000000000..9a26d777b --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test @@ -0,0 +1,2 @@ +# Sudoers File for testing, after install script runs the Defaults secure_path should be appended to +Defaults secure_path=/original_content_of_sudoers_file \ No newline at end of file diff --git a/test/python/scenarios.json b/test/python/scenarios.json index be37869df..e4fe0f0cd 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -68,6 +68,19 @@ } } }, + "install_jupyterlab_existing_sudoers_file": { + "build": { + "dockerfile": "Dockerfile" + }, + "remoteUser": "vscode", + "features": { + "python": { + "version": "latest", + "installJupyterlab": true, + "configureJupyterlabAllowOrigin": "*" + } + } + }, "install_jupyterlab_rhel_family": { "image": "almalinux:8", "remoteUser": "vscode", From b32aa5f0f207e9dc9c5ad36524ed45c6aaec20dd Mon Sep 17 00:00:00 2001 From: Rambaud Pierrick <12rambau@users.noreply.github.com> Date: Thu, 30 May 2024 19:56:16 +0200 Subject: [PATCH 087/285] fix centos-7 build (#985) * fix centos-7 build * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * bump(python): 1.5.0 -> 1.5.1 --------- Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 7c2c6200a..f57568cc6 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.6.0", + "version": "1.6.1", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index 1aad3f631..94007a9b0 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -390,7 +390,6 @@ add_symlink() { } install_openssl3() { - local _prefix=$1 mkdir /tmp/openssl3 ( cd /tmp/openssl3 @@ -403,7 +402,7 @@ install_openssl3() { curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" tar xzf ${tgz_filename} cd openssl-${openssl3_version} - ./config --prefix=${_prefix} --openssldir=${_prefix} --libdir=lib + ./config --libdir=lib make -j $(nproc) make install_dev ) @@ -446,11 +445,12 @@ install_from_source() { # Some platforms/os versions need modern versions of openssl installed # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand + SSL_INSTALL_PATH="/usr/local" case ${VERSION_CODENAME} in centos7|rhel7) check_packages perl-IPC-Cmd - install_openssl3 ${INSTALL_PATH} - ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH} --with-openssl-rpath=${INSTALL_PATH}/lib" + install_openssl3 + ADDL_CONFIG_ARGS="--with-openssl=${SSL_INSTALL_PATH} --with-openssl-rpath=${SSL_INSTALL_PATH}/lib" ;; esac From dbb135408311512248bfb2b161d52d12936bbaa6 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 31 May 2024 05:36:41 +0530 Subject: [PATCH 088/285] [azure-cli] - add support for noble numbat (#986) * [azure-cli] - add support for noble numbat * changes requested --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index 6b26ef6fc..e73d2295c 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.4", + "version": "1.2.5", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index a1b254779..2b52ca158 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -18,7 +18,7 @@ AZ_INSTALLBICEP=${INSTALLBICEP:-false} INSTALL_USING_PYTHON=${INSTALLUSINGPYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" -AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" +AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy noble" if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' From 1e44a6741d33d65bfaf340b5af6107f0a95441ce Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Mon, 3 Jun 2024 22:15:24 +0530 Subject: [PATCH 089/285] [Java] - Document additionalVersions functionality (#987) * [Java] - Document additionalVersions functionality * changes requested * changes as requested in pr review * bump patch version * changes requested --- src/java/devcontainer-feature.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/java/devcontainer-feature.json b/src/java/devcontainer-feature.json index 9ed72ad59..c43862610 100644 --- a/src/java/devcontainer-feature.json +++ b/src/java/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "java", - "version": "1.4.1", + "version": "1.5.0", "name": "Java (via SDKMAN!)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/java", "description": "Installs Java, SDKMAN! (if not installed), and needed dependencies.", @@ -17,6 +17,11 @@ "default": "latest", "description": "Select or enter a Java version to install" }, + "additionalVersions": { + "type": "string", + "default": "", + "description": "Enter additional Java versions, separated by commas." + }, "jdkDistro": { "type": "string", "proposals": [ From 6a9dd0777c5ac0c9a998728491d0e2e0bca11081 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 3 Jun 2024 09:52:05 -0700 Subject: [PATCH 090/285] Automated documentation update (#992) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/java/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/java/README.md b/src/java/README.md index 938cd0f89..1a2d91851 100644 --- a/src/java/README.md +++ b/src/java/README.md @@ -16,6 +16,7 @@ Installs Java, SDKMAN! (if not installed), and needed dependencies. | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select or enter a Java version to install | string | latest | +| additionalVersions | Enter additional Java versions, separated by commas. | string | - | | jdkDistro | Select or enter a JDK distribution | string | ms | | installGradle | Install Gradle, a build automation tool for multi-language software development | boolean | false | | gradleVersion | Select or enter a Gradle version | string | latest | From 865f69c6a2683603090be0d8c531da1cbf549c9b Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Wed, 12 Jun 2024 01:06:35 +0530 Subject: [PATCH 091/285] #963 specific powershell module version install (#993) * #963 specific powershell module version install * review comments addressed * added test for the version specific module installation and addressed review comments * Update src/powershell/install.sh * Update src/powershell/install.sh * Update src/powershell/install.sh --------- Co-authored-by: Samruddhi Khandale --- src/powershell/devcontainer-feature.json | 4 ++-- src/powershell/install.sh | 16 +++++++++++++--- test/powershell/install_modules_version.sh | 13 +++++++++++++ test/powershell/scenarios.json | 8 ++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 test/powershell/install_modules_version.sh diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 82ef39a30..d51b5c03f 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.5", + "version": "1.4.0", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", @@ -18,7 +18,7 @@ "modules": { "type": "string", "default": "", - "description": "Optional comma separated list of PowerShell modules to install." + "description": "Optional comma separated list of PowerShell modules to install. If you need to install a specific version of a module, use '==' to specify the version (e.g. 'az.resources==2.5.0')" }, "powershellProfileURL": { "type": "string", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 533da6f37..43773b3f6 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -242,14 +242,24 @@ if [ "${use_github}" = "true" ]; then install_using_github fi -# If PowerShell modules are requested, loop through and install +# If PowerShell modules are requested, loop through and install if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then echo "Installing PowerShell Modules: ${POWERSHELL_MODULES}" modules=(`echo ${POWERSHELL_MODULES} | tr ',' ' '`) for i in "${modules[@]}" do - echo "Installing ${i}" - pwsh -Command "Install-Module -Name ${i} -AllowClobber -Force -Scope AllUsers" || continue + module_parts=(`echo ${i} | tr '==' ' '`) + module_name="${module_parts[0]}" + args="-Name ${module_name} -AllowClobber -Force -Scope AllUsers" + if [ "${#module_parts[@]}" -eq 2 ]; then + module_version="${module_parts[1]}" + echo "Installing ${module_name} v${module_version}" + args+=" -RequiredVersion ${module_version}" + else + echo "Installing latest version for ${i} module" + fi + + pwsh -Command "Install-Module $args" || continue done fi diff --git a/test/powershell/install_modules_version.sh b/test/powershell/install_modules_version.sh new file mode 100644 index 000000000..ea1a3a98c --- /dev/null +++ b/test/powershell/install_modules_version.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Extension-specific tests +check "az.resources" pwsh -Command "(Get-Module -ListAvailable -Name Az.Resources).Version.ToString()" | grep 2.5.0 +check "az.storage" pwsh -Command "(Get-Module -ListAvailable -Name Az.Storage).Version.ToString()" | grep 4.3.0 + +# Report result +reportResults diff --git a/test/powershell/scenarios.json b/test/powershell/scenarios.json index 6a810b0c9..757a8cc97 100644 --- a/test/powershell/scenarios.json +++ b/test/powershell/scenarios.json @@ -16,5 +16,13 @@ "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" } } + }, + "install_modules_version": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "powershell": { + "modules": "az.resources==2.5.0, az.storage==4.3.0" + } + } } } From 22ee16e26000d47f6f2ea03a09d68a7487e4603d Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:09:20 -0700 Subject: [PATCH 092/285] Automated documentation update (#998) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/powershell/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powershell/README.md b/src/powershell/README.md index f018778c0..31199a4e1 100644 --- a/src/powershell/README.md +++ b/src/powershell/README.md @@ -16,7 +16,7 @@ Installs PowerShell along with needed dependencies. Useful for base Dockerfiles | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select or enter a version of PowerShell. | string | latest | -| modules | Optional comma separated list of PowerShell modules to install. | string | - | +| modules | Optional comma separated list of PowerShell modules to install. If you need to install a specific version of a module, use '==' to specify the version (e.g. 'az.resources==2.5.0') | string | - | | powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | ## Customizations From 15320f018d0cd72490ba073edb8900968b864ade Mon Sep 17 00:00:00 2001 From: Steven Date: Fri, 14 Jun 2024 02:02:14 +0200 Subject: [PATCH 093/285] dotnet: add ability to install workloads (#997) * dotnet: add ability to install workloads * Bump dotnet feature version * Update NOTES instead of README * Simplify workloads example * Fix temp-dir path oopsie * Improve log message * Fix typo imstalling->installing * Install all workloads at once, fix DOTNET vars not taking effect --- src/dotnet/NOTES.md | 18 +++++++++++------- src/dotnet/devcontainer-feature.json | 7 ++++++- src/dotnet/install.sh | 17 +++++++++++++++++ src/dotnet/scripts/dotnet-helpers.sh | 19 +++++++++++++++---- test/dotnet/dotnet_helpers.sh | 14 +++++++++++--- test/dotnet/install_dotnet_workloads.sh | 24 ++++++++++++++++++++++++ test/dotnet/scenarios.json | 12 +++++++++++- 7 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 test/dotnet/install_dotnet_workloads.sh diff --git a/src/dotnet/NOTES.md b/src/dotnet/NOTES.md index 578aceaf3..584c90cef 100644 --- a/src/dotnet/NOTES.md +++ b/src/dotnet/NOTES.md @@ -2,8 +2,7 @@ Installing only the latest .NET SDK version (the default). -``` json -{ +``` jsonc "features": { "ghcr.io/devcontainers/features/dotnet:2": "latest" // or "" or {} } @@ -12,7 +11,6 @@ Installing only the latest .NET SDK version (the default). Installing an additional SDK version. Multiple versions can be specified as comma-separated values. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "additionalVersions": "lts" @@ -23,7 +21,6 @@ Installing an additional SDK version. Multiple versions can be specified as comm Installing specific SDK versions. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0", @@ -35,7 +32,6 @@ Installing specific SDK versions. Installing a specific SDK feature band. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0.4xx", @@ -46,7 +42,6 @@ Installing a specific SDK feature band. Installing a specific SDK patch version. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0.412", @@ -57,7 +52,6 @@ Installing a specific SDK patch version. Installing only the .NET Runtime or the ASP.NET Core Runtime. (The SDK includes all runtimes so this configuration is only useful if you need to run .NET apps without building them from source.) ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "none", @@ -67,6 +61,16 @@ Installing only the .NET Runtime or the ASP.NET Core Runtime. (The SDK includes } ``` +Installing .NET workloads. Multiple workloads can be specified as comma-separated values. + +``` json +"features": { + "ghcr.io/devcontainers/features/dotnet:2": { + "workloads": "aspire, wasm-tools" + } +} +``` + ## OS Support This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 8b8ffaf2a..fa80799a5 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.6", + "version": "2.1.0", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", @@ -32,6 +32,11 @@ "type": "string", "default": "", "description": "Enter additional ASP.NET Core runtime versions, separated by commas. Use 'latest' for the latest version, 'lts' for the latest LTS version, 'X.Y' or 'X.Y.Z' for a specific version." + }, + "workloads": { + "type": "string", + "default": "", + "description": "Enter additional .NET SDK workloads, separated by commas. Use 'dotnet workload search' to learn what workloads are available to install." } }, "containerEnv": { diff --git a/src/dotnet/install.sh b/src/dotnet/install.sh index 237a8a0be..d289ea2a4 100644 --- a/src/dotnet/install.sh +++ b/src/dotnet/install.sh @@ -10,6 +10,14 @@ DOTNET_VERSION="${VERSION:-"latest"}" ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" DOTNET_RUNTIME_VERSIONS="${DOTNETRUNTIMEVERSIONS:-""}" ASPNETCORE_RUNTIME_VERSIONS="${ASPNETCORERUNTIMEVERSIONS:-""}" +WORKLOADS="${WORKLOADS:-""}" + +# Prevent "Welcome to .NET" message from dotnet +export DOTNET_NOLOGO=true + +# Prevent generating a development certificate while running this script +# Otherwise it would be stored in the image, which is undesirable +export DOTNET_GENERATE_ASPNET_CERTIFICATE=false set -e @@ -111,6 +119,15 @@ for version in "${aspNetCoreRuntimeVersions[@]}"; do install_runtime "aspnetcore" "$version" done +workloads=() +for workload in $(split_csv "$WORKLOADS"); do + workloads+=("$workload") +done + +if [ ${#workloads[@]} -ne 0 ]; then + install_workloads "${workloads[@]}" +fi + # Clean up rm -rf /var/lib/apt/lists/* rm -rf scripts diff --git a/src/dotnet/scripts/dotnet-helpers.sh b/src/dotnet/scripts/dotnet-helpers.sh index bda0c9c3a..b7024c9d1 100644 --- a/src/dotnet/scripts/dotnet-helpers.sh +++ b/src/dotnet/scripts/dotnet-helpers.sh @@ -25,7 +25,6 @@ fetch_latest_version_in_channel() { else wget -qO- "https://dotnetcli.azureedge.net/dotnet/Sdk/$channel/latest.version" fi - } # Prints the latest dotnet version @@ -76,12 +75,11 @@ install_sdk() { fi # Currently this script does not make it possible to qualify the version, 'GA' is always implied - echo "Executing $DOTNET_INSTALL_SCRIPT --version $version --channel $channel --install-dir $DOTNET_INSTALL_DIR --no-path" + echo "Executing $DOTNET_INSTALL_SCRIPT --version $version --channel $channel --install-dir $DOTNET_INSTALL_DIR" "$DOTNET_INSTALL_SCRIPT" \ --version "$version" \ --channel "$channel" \ - --install-dir "$DOTNET_INSTALL_DIR" \ - --no-path + --install-dir "$DOTNET_INSTALL_DIR" } # Installs a version of the .NET Runtime @@ -117,3 +115,16 @@ install_runtime() { --install-dir "$DOTNET_INSTALL_DIR" \ --no-path } + +# Installs one or more .NET workloads +# Usage: install_workload [ ...] +# Reference: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-workload-install +install_workloads() { + local workloads="$@" + + echo "Installing .NET workload(s) $workloads" + dotnet workload install $workloads --temp-dir /tmp/dotnet-workload-temp-dir + + # Clean up + rm -r /tmp/dotnet-workload-temp-dir +} diff --git a/test/dotnet/dotnet_helpers.sh b/test/dotnet/dotnet_helpers.sh index 6c833b444..a24bd1ce2 100644 --- a/test/dotnet/dotnet_helpers.sh +++ b/test/dotnet/dotnet_helpers.sh @@ -15,7 +15,6 @@ fetch_latest_version_in_channel() { else wget -qO- "https://dotnetcli.azureedge.net/dotnet/Sdk/$channel/latest.version" fi - } # Prints the latest dotnet version @@ -47,7 +46,6 @@ is_dotnet_sdk_version_installed() { return $? } - # Asserts that the specified .NET Runtime version is installed # Returns a non-zero exit code if the check fails # Usage: is_dotnet_runtime_version_installed @@ -68,4 +66,14 @@ is_aspnetcore_runtime_version_installed() { local expected="$1" dotnet --list-runtimes | grep --fixed-strings --silent "Microsoft.AspNetCore.App $expected" return $? -} \ No newline at end of file +} + +# Asserts that the specified workload is installed +# Returns a non-zero exit code if the check fails +# Usage: is_dotnet_workload_installed +# Example: is_dotnet_workload_installed "aspire" +is_dotnet_workload_installed() { + local expected="$1" + dotnet workload list | grep --fixed-strings --silent "$expected" + return $? +} diff --git a/test/dotnet/install_dotnet_workloads.sh b/test/dotnet/install_dotnet_workloads.sh new file mode 100644 index 000000000..37c86a2d4 --- /dev/null +++ b/test/dotnet/install_dotnet_workloads.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Optional: Import test library bundled with the devcontainer CLI +# See https://github.com/devcontainers/cli/blob/HEAD/docs/features/test.md#dev-container-features-test-lib +# Provides the 'check' and 'reportResults' commands. +source dev-container-features-test-lib + +# Feature-specific tests +# The 'check' command comes from the dev-container-features-test-lib. Syntax is... +# check