From 5baf166f67683d6801c327d1c084d15f737947cd Mon Sep 17 00:00:00 2001 From: Leobaldo Alcantara Neto Date: Tue, 26 Mar 2024 14:05:35 -0300 Subject: [PATCH 001/224] Update DOCKER_DASH_COMPOSE_VERSION default to v2 (#920) * Update DOCKER_DASH_COMPOSE_VERSION default to v2 Changed it because devcontainer-feature.json documentation indicates that it's the default version. * Update devcontainer-feature.json version to 1.4.4 --- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 54fca5ec1..2a55fdbca 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.3", + "version": "1.4.4", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index dd39e1f68..6b63469f9 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -10,7 +10,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" -DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none +DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v2"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" SOURCE_SOCKET="${SOURCE_SOCKET:-"/var/run/docker-host.sock"}" From 0100f66138e2082cd76b92c30826cbb4544755a3 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 27 Mar 2024 05:25:37 +0530 Subject: [PATCH 002/224] [docker-in-docker]-docker_compose-fallback-github api (#914) * [docker-in-docker]-docker_compose-fallback-github api * changes misc. * misc. change * changes as required in review comments * changes miscellaneous * suggested changes * changes as requested by review comments on pr * Minor changes as suggested... --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 142 +++++++++--- .../docker_build_fallback_buildx.sh | 206 +++++++++++++----- .../docker_build_fallback_compose.sh | 174 +++++++++++++++ test/docker-in-docker/scenarios.json | 13 +- 5 files changed, 442 insertions(+), 95 deletions(-) create mode 100644 test/docker-in-docker/docker_build_fallback_compose.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 671f4e2ed..812db444c 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.1", + "version": "2.10.2", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index dbe34e2ea..0dc9e52d1 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -109,18 +109,73 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } -install_compose_switch_fallback() { - echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version#v} - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } ########################################### @@ -265,6 +320,16 @@ echo "Finished installing docker / moby!" docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" +# fallback for docker-compose +fallback_compose(){ + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + get_previous_version "${url}" "${repo_url}" compose_version + echo -e "\nAttempting to install v${compose_version}" + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} +} + # If 'docker-compose' command is to be included if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then case "${architecture}" in @@ -301,17 +366,16 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then fi else compose_version=${DOCKER_DASH_COMPOSE_VERSION#v} - find_version_from_git_tags compose_version "https://github.com/docker/compose" "tags/v" + docker_compose_url="https://github.com/docker/compose" + find_version_from_git_tags compose_version "$docker_compose_url" "tags/v" echo "(*) Installing docker-compose ${compose_version}..." - curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} - - if grep -q "Not Found" "${docker_compose_path}"; then - echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_version=${previous_version#v} - curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} - fi + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} || { + if [[ $DOCKER_DASH_COMPOSE_VERSION == "latest" ]]; then + fallback_compose "$docker_compose_url" + else + echo -e "Error: Failed to install docker-compose v${compose_version}" + fi + } chmod +x ${docker_compose_path} @@ -325,6 +389,16 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then fi fi +# fallback method for compose-switch +fallback_compose-switch() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + get_previous_version "$url" "$repo_url" compose_switch_version + echo -e "\nAttempting to install v${compose_switch_version}" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +} + # Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /dev/null 2>&1; then if type docker-compose > /dev/null 2>&1; then @@ -332,8 +406,9 @@ if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /d current_compose_path="$(which docker-compose)" target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" compose_switch_version="latest" - find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || install_compose_switch_fallback + compose_switch_url="https://github.com/docker/compose-switch" + find_version_from_git_tags compose_switch_version "$compose_switch_url" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || fallback_compose-switch "$compose_switch_url" chmod +x /usr/local/bin/compose-switch # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) @@ -360,29 +435,26 @@ fi usermod -aG docker ${USERNAME} -install_previous_version_artifacts() { - wget_exit_code=$? - if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. - echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." - repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL - previous_version=$(get_previous_version "${repo_url}") - buildx_file_name="buildx-${previous_version}.linux-${architecture}" - echo -e "\nAttempting to install ${previous_version}" - wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} - else - echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" - exit 1 - fi +# fallback for docker/buildx +fallback_buildx() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + get_previous_version "$url" "$repo_url" buildx_version + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + echo -e "\nAttempting to install v${buildx_version}" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} } if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then buildx_version="latest" - find_version_from_git_tags buildx_version "https://github.com/docker/buildx" "refs/tags/v" + docker_buildx_url="https://github.com/docker/buildx" + find_version_from_git_tags buildx_version "$docker_buildx_url" "refs/tags/v" echo "(*) Installing buildx ${buildx_version}..." buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" cd /tmp - wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || fallback_buildx "$docker_buildx_url" docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" diff --git a/test/docker-in-docker/docker_build_fallback_buildx.sh b/test/docker-in-docker/docker_build_fallback_buildx.sh index 707e0a68a..e139613a0 100644 --- a/test/docker-in-docker/docker_build_fallback_buildx.sh +++ b/test/docker-in-docker/docker_build_fallback_buildx.sh @@ -14,83 +14,177 @@ check "docker-build" docker build ./ check "docker-buildx" bash -c "docker buildx version" check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" -echo -e "\n๐Ÿ‘‰${HL} Creating a scenario for fallback${N}\n" # Code to test the made up scenario when latest version of docker/buildx fails on wget command for fetching the artifacts -repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL architecture="$(dpkg --print-architecture)" +case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 +esac -# Function to fetch the latest version of the plugin -get_latest_version() { - curl -s "$repo_url/latest" | jq -r '.tag_name' +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + err "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" } -# Function to fetch the previous version of the plugin -get_previous_version() { - # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e } -# Function to change the patch number in a semver version -change_patch_number() { - local version="$1" # Input version - local new_patch="$2" # New patch number - # Extract major, minor, and current patch numbers - local major=$(echo "$version" | cut -d. -f1) - local minor=$(echo "$version" | cut -d. -f2) - local current_patch=$(echo "$version" | cut -d. -f3) - # Construct the new version with the updated patch number - local new_version="$major.$minor.$new_patch" - echo "$new_version" +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + echo -e "\nAttempting to find latest version using Github Api." + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $mode != "install_from_github_api_valid" ]]; then + message="API rate limit exceeded" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAttempting to find latest version using Github Api Failed. Exceeded API Rate Limit." + echo -e "\nAttempting to find latest version using Github Tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using Github Api Succeeded." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } -change_version_to_fail() { - latest_version=$1 - new_patch_number="xyz" # for testing a tag not found scenario for docker/buildx plugin - latest_version=$(get_latest_version) # can take latest_version from fn get_latest_version - buildx_version_fallback_test=$(change_patch_number "$latest_version" "$new_patch_number") # for testing a tag not found scenario for docker/buildx plugin - echo "${buildx_version_fallback_test}" +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -install_previous_version_artifacts() { - wget_exit_code=$? - if [ $wget_exit_code -ne 0 ]; then # means wget command to fetch latest version failed - if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. - echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx ${buildx_version}..." - previous_version=$(get_previous_version) - echo -e "\nAttempting to install ${previous_version}" - buildx_file_name="buildx-${previous_version}.linux-${architecture}" - wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} - else - echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" - exit 1 - fi - fi +install_using_get_previous_version() { + local url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + get_previous_version "${url}" "${repo_url}" buildx_version "${mode}" + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + echo -e "\nAttempting to install v${buildx_version}" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} } -test_version=$(change_version_to_fail "$(get_latest_version)") -buildx_file_name="buildx-${test_version}.linux-${architecture}" -buildx_version=$test_version +install_docker_buildx() { + mode=$1 + echo -e "\n${HL} Creating a scenario for fallback${N}\n" -# This wget command will fail as the wrong version won't fetch artifact -wget https://github.com/docker/buildx/releases/download/${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + buildx_version="0.13.xyz" + echo "(*) Installing buildx ${buildx_version}..." + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + cd /tmp -docker_home="/usr/libexec/docker" -cli_plugins_dir="${docker_home}/cli-plugins" + docker_buildx_url="https://github.com/docker/buildx" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_using_get_previous_version "${docker_buildx_url}" "${mode}" + + docker_home="/usr/libexec/docker" + cli_plugins_dir="${docker_home}/cli-plugins" -mkdir -p ${cli_plugins_dir} -mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx -chmod +x ${cli_plugins_dir}/docker-buildx + mkdir -p ${cli_plugins_dir} + mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx + chmod +x ${cli_plugins_dir}/docker-buildx -chown -R "${USERNAME}:docker" "${docker_home}" -chmod -R g+r+w "${docker_home}" -find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s + chown -R "${USERNAME}:docker" "${docker_home}" + chmod -R g+r+w "${docker_home}" + find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s +} + +echo -e "\n๐Ÿ‘‰${HL} docker-buildx version as installed by docker-in-docker test ( installing by github api ) ${N}" +install_docker_buildx "install_from_github_api_valid" + +# Definition specific tests after test for fallback +check "docker-buildx" docker buildx version +check "docker-buildx" bash -c "docker buildx version" + +echo -e "\n๐Ÿ‘‰${HL} docker-buildx version as installed by docker-in-docker test ( installing by find_prev_version_from_git_tags ) ${N}" +install_docker_buildx # Definition specific tests after test for fallback -echo -e "\n๐Ÿ‘‰${HL} docker/buildx version as installed by test for fallback${N}" check "docker-buildx" docker buildx version -check "docker-build" docker build ./ check "docker-buildx" bash -c "docker buildx version" -check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" # Report result reportResults diff --git a/test/docker-in-docker/docker_build_fallback_compose.sh b/test/docker-in-docker/docker_build_fallback_compose.sh new file mode 100644 index 000000000..4a18640f0 --- /dev/null +++ b/test/docker-in-docker/docker_build_fallback_compose.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# Optional: Import test library +source dev-container-features-test-lib + +# Setup STDERR. +err() { + echo "(!) $*" >&2 +} + +HL="\033[1;33m" +N="\033[0;37m" +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker feature${N}" +check "docker-compose" bash -c "docker-compose version" + +architecture="$(dpkg --print-architecture)" +case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 +esac + +docker_compose_path="/usr/local/bin/docker-compose" +cli_plugins_dir="${docker_home}/cli-plugins" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + err "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + echo -e "\nAttempting to find latest version using Github Api." + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $mode != "install_from_github_api_valid" ]]; then + message="API rate limit exceeded" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAttempting to find latest version using Github Api Failed. Exceeded API Rate Limit." + echo -e "\nAttempting to find latest version using Github Tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using Github Api Succeeded." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_using_get_previous_version() { + local url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + get_previous_version "$url" "$repo_url" compose_version "$mode" + echo -e "\nAttempting to install v${compose_version}" + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} +} + +install_docker_compose() { + mode=$1 + compose_version="2.25.xyz" + docker_compose_url="https://github.com/docker/compose" + echo "(*) Installing docker-compose ${compose_version}..." + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} || install_using_get_previous_version "$docker_compose_url" "$mode" +} + +chmod +x ${docker_compose_path} + +# Download the SHA256 checksum +DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}.sha256" | awk '{print $1}')" +echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum +sha256sum -c docker-compose.sha256sum --ignore-missing + +mkdir -p ${cli_plugins_dir} +cp ${docker_compose_path} ${cli_plugins_dir} + +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker test ( installing by github api ) ${N}" +install_docker_compose "install_from_github_api_valid" + +check "docker-compose" bash -c "docker-compose version" + +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker test ( installing by find_prev_version_from_git_tags ) ${N}" +install_docker_compose + +check "docker-compose" bash -c "docker-compose version" diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index e209f3453..33333583d 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -1,4 +1,13 @@ { + "docker_build_fallback_compose": { + "image": "ubuntu:focal", + "features": { + "docker-in-docker": { + "version": "latest", + "dockerDashComposeVersion": "latest" + } + } + }, "dockerDefaultAddressPool": { "image": "mcr.microsoft.com/vscode/devcontainers/javascript-node:0-18", "remoteUser": "node", @@ -112,9 +121,7 @@ "features": { "docker-in-docker": { "version": "latest", - "installDockerBuildx": true, - "moby": "false", - "dockerDashComposeVersion": "v2" + "installDockerBuildx": true } } }, From 0a4fa18e1db577a60f569f7684dde62430a3179d Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 03:30:20 +0530 Subject: [PATCH 003/224] [azure-cli] - python3.12 - feature broken due to distutils module removed - code fix (#909) * [azure-cli] - python3.12 - feature broken due to distutils module removed - fix * update default for installUsingPython * removed the installUsingPython=true flag from the building scenario * bump update the patch version --------- Co-authored-by: Samruddhi Khandale --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 8 +++---- .../install_with_python_3_12_bookworm.sh | 21 +++++++++++++++++++ test/azure-cli/scenarios.json | 9 ++++++++ 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 test/azure-cli/install_with_python_3_12_bookworm.sh diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index 136b03fdc..f25e5f9a2 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.2", + "version": "1.2.3", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index 6d3426699..7f4c4ff1c 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -15,10 +15,10 @@ rm -rf /var/lib/apt/lists/* AZ_VERSION=${VERSION:-"latest"} AZ_EXTENSIONS=${EXTENSIONS} AZ_INSTALLBICEP=${INSTALLBICEP:-false} -INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-true} +INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -AZCLI_ARCHIVE_ARCHITECTURES="amd64" -AZCLI_ARCHIVE_VERSION_CODENAMES="stretch buster bullseye bionic focal jammy" +AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" +AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' @@ -183,7 +183,7 @@ install_with_complete_python_installation() { export DEBIAN_FRONTEND=noninteractive -# See if we're on x86_64 and if so, install via apt-get, otherwise use pip3 +# See if we're on x86_64 or AARCH64 and if so, install via apt-get, otherwise use pip3 echo "(*) Installing Azure CLI..." . /etc/os-release architecture="$(dpkg --print-architecture)" diff --git a/test/azure-cli/install_with_python_3_12_bookworm.sh b/test/azure-cli/install_with_python_3_12_bookworm.sh new file mode 100644 index 000000000..074876148 --- /dev/null +++ b/test/azure-cli/install_with_python_3_12_bookworm.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version + +echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"'; then + echo -e "\n\nโœ… Passed 'O.S is Linux 12 (bookworm)'!" +else + echo -e "\n\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!" +fi + + +# Report result +reportResults \ No newline at end of file diff --git a/test/azure-cli/scenarios.json b/test/azure-cli/scenarios.json index b7a732395..041f50731 100644 --- a/test/azure-cli/scenarios.json +++ b/test/azure-cli/scenarios.json @@ -38,5 +38,14 @@ "installUsingPython": true } } + }, + "install_with_python_3_12_bookworm": { + "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bookworm", + "user": "vscode", + "features": { + "azure-cli": { + "version": "latest" + } + } } } \ No newline at end of file From 988cdd2bb9eb9a96cf16ed30e2bfba32c68ad81c Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:41:51 +0530 Subject: [PATCH 004/224] [Terraform] feature fallback fix (#912) * implementation of fallback logic for cosign, terraform * tflint, terragrunt - wrote fallback logic for these * misc change * [Terraform] - fallback - apply * changes as requested by review comments * misc change * misc change * 2-step fallback implemented in install.sh, test cases yet to be updated * install jq for ubuntu:focal, jammy & debian:11, 12 * test for terraform_docs updated * tfsec test updated * terraform fallback test updated * misc change * added tflint fallback test file * terragrunt fallback test file added * minor change in tfsec test file * few more changes to cosign, terragrunt installations.. * changes to test files for terragrunt & cosign * Changes acc. to review comments for pr * changes requested * changes as requested in review comments ! --- src/terraform/devcontainer-feature.json | 2 +- src/terraform/install.sh | 192 ++++++++++-- test/terraform/scenarios.json | 40 +++ .../terraform/terraform_docs_fallback_test.sh | 220 ++++++++++++++ test/terraform/terraform_fallback_test.sh | 206 +++++++++++++ test/terraform/terragrunt_fallback_test.sh | 219 +++++++++++++ test/terraform/tflint_fallback_test.sh | 287 ++++++++++++++++++ test/terraform/tfsec_fallback_test.sh | 224 ++++++++++++++ 8 files changed, 1372 insertions(+), 18 deletions(-) create mode 100644 test/terraform/terraform_docs_fallback_test.sh create mode 100644 test/terraform/terraform_fallback_test.sh create mode 100644 test/terraform/terragrunt_fallback_test.sh create mode 100644 test/terraform/tflint_fallback_test.sh create mode 100644 test/terraform/tfsec_fallback_test.sh diff --git a/src/terraform/devcontainer-feature.json b/src/terraform/devcontainer-feature.json index 51fb9a27a..5c9c042cc 100644 --- a/src/terraform/devcontainer-feature.json +++ b/src/terraform/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "terraform", - "version": "1.3.5", + "version": "1.3.6", "name": "Terraform, tflint, and TFGrunt", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/terraform", "description": "Installs the Terraform CLI and optionally TFLint and Terragrunt. Auto-detects latest version and installs needed dependencies.", diff --git a/src/terraform/install.sh b/src/terraform/install.sh index c1a382597..dc7aef8c3 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -137,6 +137,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + find_sentinel_version_from_url() { local variable_name=$1 local requested_version=${!variable_name} @@ -177,6 +218,73 @@ check_packages() { fi } +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + INSTALLER_FN=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_cosign() { + COSIGN_VERSION=$1 + local URL=$2 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." +} + # Install 'cosign' for validating signatures # https://docs.sigstore.dev/cosign/overview/ ensure_cosign() { @@ -184,12 +292,10 @@ ensure_cosign() { if ! type cosign > /dev/null 2>&1; then echo "Installing cosign..." - LATEST_COSIGN_VERSION="latest" - find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' - curl -L "https://github.com/sigstore/cosign/releases/latest/download/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb" -o /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - - dpkg -i /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - rm /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb + COSIGN_VERSION="latest" + cosign_url='https://github.com/sigstore/cosign' + find_version_from_git_tags COSIGN_VERSION "${cosign_url}" + install_cosign "${COSIGN_VERSION}" "${cosign_url}" fi if ! type cosign > /dev/null 2>&1; then echo "(!) Failed to install cosign." @@ -207,18 +313,30 @@ if ! type git > /dev/null 2>&1; then check_packages git fi +terraform_url='https://github.com/hashicorp/terraform' +tflint_url='https://github.com/terraform-linters/tflint' +terragrunt_url='https://github.com/gruntwork-io/terragrunt' # Verify requested version is available, convert latest -find_version_from_git_tags TERRAFORM_VERSION 'https://github.com/hashicorp/terraform' -find_version_from_git_tags TFLINT_VERSION 'https://github.com/terraform-linters/tflint' -find_version_from_git_tags TERRAGRUNT_VERSION 'https://github.com/gruntwork-io/terragrunt' +find_version_from_git_tags TERRAFORM_VERSION "$terraform_url" +find_version_from_git_tags TFLINT_VERSION "$tflint_url" +find_version_from_git_tags TERRAGRUNT_VERSION "$terragrunt_url" + +install_terraform() { + local TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +} mkdir -p /tmp/tf-downloads cd /tmp/tf-downloads - # Install Terraform, tflint, Terragrunt echo "Downloading terraform..." terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" -curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +install_terraform "$TERRAFORM_VERSION" +if grep -q "The specified key does not exist." "${terraform_filename}"; then + install_previous_version TERRAFORM_VERSION $terraform_url "install_terraform" + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" +fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_SHA256}" = "automatic" ]; then receive_gpg_keys TERRAFORM_GPG_KEY @@ -233,10 +351,18 @@ fi unzip ${terraform_filename} mv -f terraform /usr/local/bin/ +install_tflint() { + TFLINT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} +} + if [ "${TFLINT_VERSION}" != "none" ]; then echo "Downloading tflint..." TFLINT_FILENAME="tflint_linux_${architecture}.zip" - curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} + install_tflint "$TFLINT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + install_previous_version TFLINT_VERSION "$tflint_url" "install_tflint" + fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then if [ "${TFLINT_SHA256}" != "automatic" ]; then @@ -277,10 +403,20 @@ if [ "${TFLINT_VERSION}" != "none" ]; then unzip /tmp/tf-downloads/${TFLINT_FILENAME} mv -f tflint /usr/local/bin/ fi + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} +} + if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." terragrunt_filename="terragrunt_linux_${architecture}" - curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} + install_terragrunt "$TERRAGRUNT_VERSION" + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then + install_previous_version TERRAGRUNT_VERSION $terragrunt_url "install_terragrunt" + fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS @@ -318,12 +454,23 @@ if [ "${INSTALL_SENTINEL}" = "true" ]; then mv -f /tmp/tf-downloads/sentinel /usr/local/bin/sentinel fi +install_tfsec() { + local TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + if [ "${INSTALL_TFSEC}" = "true" ]; then TFSEC_VERSION="latest" - find_version_from_git_tags TFSEC_VERSION 'https://github.com/aquasecurity/tfsec' + tfsec_url='https://github.com/aquasecurity/tfsec' + find_version_from_git_tags TFSEC_VERSION $tfsec_url tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" echo "(*) Downloading TFSec... ${tfsec_filename}" - curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + install_previous_version TFSEC_VERSION $tfsec_url "install_tfsec" + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then if [ "${TFSEC_SHA256}" = "automatic" ]; then curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt @@ -338,12 +485,23 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec fi +install_terraform_docs() { + local TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + if [ "${INSTALL_TERRAFORM_DOCS}" = "true" ]; then TERRAFORM_DOCS_VERSION="latest" - find_version_from_git_tags TERRAFORM_DOCS_VERSION 'https://github.com/terraform-docs/terraform-docs' + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' + find_version_from_git_tags TERRAFORM_DOCS_VERSION $terraform_docs_url tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" echo "(*) Downloading Terraform docs... ${tfdocs_filename}" - curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} + install_terraform_docs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url "install_terraform_docs" + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index 0a4d8956c..04693666c 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -15,6 +15,14 @@ } } }, + "tfsec_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTFsec": true + } + } + }, "install_terraform_docs": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { @@ -23,6 +31,38 @@ } } }, + "terraform_docs_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTerraformDocs": true + } + } + }, + "terraform_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "version": "latest" + } + } + }, + "terragrunt_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "terragrunt": "latest" + } + } + }, + "tflint_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "tflint": "latest" + } + } + }, "older_tflint": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh new file mode 100644 index 000000000..256ef2f85 --- /dev/null +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +# Terraform Docs specific tests +check "terraform-docs version as installed by feature" terraform-docs --version + +TERRAFORM_DOCS_SHA256="automatic" + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_terraform_docs() { + local TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + + +try_install_terraform_docs_dummy_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TERRAFORM_DOCS_VERSION="0.17.xyz" + echo -e "\nInstalling TERRAFORM_DOCS dummy version.." v${TERRAFORM_DOCS_VERSION} + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + echo "(*) Downloading Terraform docs... ${tfdocs_filename}" + install_terraform_docs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url $mode "install_terraform_docs" + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi + if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then + if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then + curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum + else + echo "${TERRAFORM_DOCS_SHA256} *${tfsec_filename}" > tfdocs_SHA256SUMS + fi + sha256sum --ignore-missing -c tfdocs_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfdocs + tar -xzf /tmp/tf-downloads/${tfdocs_filename} -C /tmp/tf-downloads/tfdocs + sudo chmod a+x /tmp/tf-downloads/tfdocs/terraform-docs + sudo mv -f /tmp/tf-downloads/tfdocs/terraform-docs /usr/local/bin/terraform-docs +} + +try_install_terraform_docs_dummy_version "mode1" + +check "terraform-docs version as installed by test (mode 1: install using find_prev_version_from_git_tags)" terraform-docs --version + +try_install_terraform_docs_dummy_version "mode2" + +check "terraform-docs version as installed by test (mode 2: install using GitHub Api)" terraform-docs --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh new file mode 100644 index 000000000..a5193daa4 --- /dev/null +++ b/test/terraform/terraform_fallback_test.sh @@ -0,0 +1,206 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +check "terraform version as installed by feature" terraform --version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_terraform() { + local TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +} + +try_install_dummy_terraform_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terraform_url='https://github.com/hashicorp/terraform' + TERRAFORM_VERSION="1.7.xyz" + echo -e "\nAttempting to install dummy version for Terraform v${TERRAFORM_VERSION}..." + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + install_terraform "$TERRAFORM_VERSION" + if grep -q "The specified key does not exist." "${terraform_filename}"; then + install_previous_version TERRAFORM_VERSION $terraform_url $mode "install_terraform" + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + fi + unzip ${terraform_filename} + sudo mv -f terraform /usr/local/bin/ +} + +try_install_dummy_terraform_version "mode1" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terraform --version + +try_install_dummy_terraform_version "mode2" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terraform --version + +# Report result +reportResults + diff --git a/test/terraform/terragrunt_fallback_test.sh b/test/terraform/terragrunt_fallback_test.sh new file mode 100644 index 000000000..83289380f --- /dev/null +++ b/test/terraform/terragrunt_fallback_test.sh @@ -0,0 +1,219 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +check "terragrunt version as installed by feature" terragrunt --version + +TERRAGRUNT_SHA256="automatic" + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} +} + + +try_install_dummy_terragrunt_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terragrunt_url='https://github.com/gruntwork-io/terragrunt' + TERRAGRUNT_VERSION="0.55.xyz" + echo -e "\nAttempting to install terragrunt dummy v${TERRAGRUNT_VERSION}" + echo "Downloading Terragrunt... v${TERRAGRUNT_VERSION}" + terragrunt_filename="terragrunt_linux_${architecture}" + install_terragrunt "$TERRAGRUNT_VERSION" + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then + install_previous_version TERRAGRUNT_VERSION $terragrunt_url $mode "install_terragrunt" + fi + if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then + if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then + curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS + else + echo "${TERRAGRUNT_SHA256} *${terragrunt_filename}" > terragrunt_SHA256SUMS + fi + sha256sum --ignore-missing -c terragrunt_SHA256SUMS + fi + sudo chmod a+x /tmp/tf-downloads/${terragrunt_filename} + sudo mv -f /tmp/tf-downloads/${terragrunt_filename} /usr/local/bin/terragrunt +} + +try_install_dummy_terragrunt_version "mode1" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terragrunt --version + +try_install_dummy_terragrunt_version "mode2" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terragrunt --version + +# Report result +reportResults + diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh new file mode 100644 index 000000000..5619ff4fb --- /dev/null +++ b/test/terraform/tflint_fallback_test.sh @@ -0,0 +1,287 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +TFLINT_SHA256="automatic" + +GPG_KEY_SERVERS="keyserver hkps://keyserver.ubuntu.com +keyserver hkps://keys.openpgp.org +keyserver hkps://keyserver.pgp.com" + +check "tflint version as installed by feature" tflint --version +check "cosign version as installed by feature" cosign version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_cosign() { + COSIGN_VERSION=$1 + local URL=$2 + local mode=$3 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION $mode + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." +} + +# Install 'cosign' for validating signatures +# https://docs.sigstore.dev/cosign/overview/ +ensure_cosign() { + mode=$1 + if ! type cosign > /dev/null 2>&1; then + echo -e "\nAttempting to install dummy cosign version..." + COSIGN_VERSION="2.2.xyz" + echo "Installing cosign... v${COSIGN_VERSION}" + cosign_url='https://github.com/sigstore/cosign' + install_cosign "${COSIGN_VERSION}" "${cosign_url}" $mode + fi + if ! type cosign > /dev/null 2>&1; then + echo "(!) Failed to install cosign." + exit 1 + fi + cosign version +} + +install_tflint() { + TFLINT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} +} + + +try_install_dummy_tflint_cosign_version() { + mode=$1 + tflint_url='https://github.com/terraform-linters/tflint' + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + echo -e "\nTrying to install dummy tflint version..." + TFLINT_VERSION="0.50.XYZ" + echo "Downloading tflint...v${TFLINT_VERSION}" + TFLINT_FILENAME="tflint_linux_${architecture}.zip" + install_tflint "$TFLINT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + install_previous_version TFLINT_VERSION "$tflint_url" $mode "install_tflint" + fi + if [ "${TFLINT_SHA256}" != "dev-mode" ]; then + + if [ "${TFLINT_SHA256}" != "automatic" ]; then + echo "${TFLINT_SHA256} *${TFLINT_FILENAME}" > tflint_checksums.txt + sha256sum --ignore-missing -c tflint_checksums.txt + else + curl -sSL -o tflint_checksums.txt https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt + + set +e + curl -sSL -o checksums.txt.keyless.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.keyless.sig + set -e + + # Check that checksums.txt.keyless.sig exists and is not empty + if [ -s checksums.txt.keyless.sig ]; then + # Validate checksums with cosign + curl -sSL -o checksums.txt.pem https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.pem + ensure_cosign $mode + cosign verify-blob \ + --certificate=/tmp/tf-downloads/checksums.txt.pem \ + --signature=/tmp/tf-downloads/checksums.txt.keyless.sig \ + --certificate-identity-regexp="^https://github.com/terraform-linters/tflint" \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + /tmp/tf-downloads/tflint_checksums.txt + # Ensure that checksums.txt has $TFLINT_FILENAME + grep ${TFLINT_FILENAME} /tmp/tf-downloads/tflint_checksums.txt + # Validate downloaded file + sha256sum --ignore-missing -c tflint_checksums.txt + else + # Fallback to older, GPG-based verification (pre-0.47.0 of tflint) + curl -sSL -o tflint_checksums.txt.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.sig + curl -sSL -o tflint_key "${TFLINT_GPG_KEY_URI}" + gpg -q --import tflint_key + gpg --verify tflint_checksums.txt.sig tflint_checksums.txt + fi + fi + fi + + unzip /tmp/tf-downloads/${TFLINT_FILENAME} + sudo mv -f tflint /usr/local/bin/ +} + +try_install_dummy_tflint_cosign_version "mode1" + +check "tflint version as installed when mode=1" tflint --version +check "cosign version as installed when mode=1" cosign version + +try_install_dummy_tflint_cosign_version "mode2" + +check "tflint version as installed when mode=2" tflint --version +check "cosign version as installed when mode=2" cosign version \ No newline at end of file diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh new file mode 100644 index 000000000..5c87d821e --- /dev/null +++ b/test/terraform/tfsec_fallback_test.sh @@ -0,0 +1,224 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +# Trap errors and call the error handling function +trap 'handle_error' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +TFSEC_SHA256="automatic" + +# TFSec specific tests +check "tfsec version as installed by feature" tfsec --version + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_tfsec() { + local TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + +try_install_tfsec_dummy_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TFSEC_VERSION="1.28.XYZ" + echo -e "\nInstalling TFSEC dummy version.." v${TFSEC_VERSION} + tfsec_url='https://github.com/aquasecurity/tfsec' + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + echo "(*) Downloading TFSec... ${tfsec_filename}" + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + install_previous_version TFSEC_VERSION $tfsec_url $mode "install_tfsec" + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi + if [ "${TFSEC_SHA256}" != "dev-mode" ]; then + if [ "${TFSEC_SHA256}" = "automatic" ]; then + curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt + else + echo "${TFSEC_SHA256} *${tfsec_filename}" > tfsec_SHA256SUMS + fi + sha256sum --ignore-missing -c tfsec_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfsec + tar -xzf /tmp/tf-downloads/${tfsec_filename} -C /tmp/tf-downloads/tfsec + chmod a+x /tmp/tf-downloads/tfsec/tfsec + sudo mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec +} + +try_install_tfsec_dummy_version "mode1" + +check "tfsec version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" tfsec --version + +try_install_tfsec_dummy_version "mode2" + +check "tfsec version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" tfsec --version + +# Report result +reportResults \ No newline at end of file From cf03551a529f39f4a50b37e39fac0964c7a579ca Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:58:53 +0530 Subject: [PATCH 005/224] [kubectl-helm-minikube] - helm - alternative fallback method implemented (#923) * [kubectl-helm-minikube] - helm - alternative fallback method implemented * bump patch version --- .../devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 80 +++++++- .../install_only_helm_fallback.sh | 192 ++++++++++++------ 3 files changed, 200 insertions(+), 74 deletions(-) diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index 41a8b8b68..8c5dfddc0 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.8", + "version": "1.1.9", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index d7de3984f..871a77ca1 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -88,6 +88,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + apt_get_update() { if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then @@ -157,9 +198,32 @@ fi # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name#v} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="v${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } get_helm() { @@ -173,7 +237,8 @@ get_helm() { if [ ${HELM_VERSION} != "none" ]; then # Install Helm, verify signature and checksum echo "Downloading Helm..." - find_version_from_git_tags HELM_VERSION "https://github.com/helm/helm" + helm_url="https://github.com/helm/helm" + find_version_from_git_tags HELM_VERSION "${helm_url}" if [ "${HELM_VERSION::1}" != 'v' ]; then HELM_VERSION="v${HELM_VERSION}" fi @@ -181,10 +246,9 @@ if [ ${HELM_VERSION} != "none" ]; then get_helm "${HELM_VERSION}" if grep -q "BlobNotFound" "${tmp_helm_filename}"; then echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." - repo_url=https://api.github.com/repos/helm/helm/releases - requested_version=$(get_previous_version "${repo_url}") - echo -e "\nAttempting to install ${requested_version}" - HELM_VERSION=${requested_version} + repo_url=$(get_github_api_repo_url "${helm_url}") + get_previous_version "${helm_url}" "${repo_url}" HELM_VERSION + echo -e "\nAttempting to install ${HELM_VERSION}" get_helm "${HELM_VERSION}" fi export GNUPGHOME="/tmp/helm/gnupg" diff --git a/test/kubectl-helm-minikube/install_only_helm_fallback.sh b/test/kubectl-helm-minikube/install_only_helm_fallback.sh index b8071a0b5..9f6a8e095 100644 --- a/test/kubectl-helm-minikube/install_only_helm_fallback.sh +++ b/test/kubectl-helm-minikube/install_only_helm_fallback.sh @@ -31,33 +31,118 @@ case $architecture in i?86) architecture="386";; *) echo "(!) Architecture $architecture unsupported"; exit 1 ;; esac -HELM_SHA256="${HELM_SHA256:-"automatic"}" -HELM_GPG_KEYS_URI="https://raw.githubusercontent.com/helm/helm/main/KEYS" -repo_url=https://api.github.com/repos/helm/helm/releases +helm_url="https://github.com/helm/helm" -# Function to fetch the latest version of the plugin -get_latest_version() { - curl -s "$repo_url/latest" | jq -r '.tag_name' +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" } -# Function to change the patch number in a semver version -change_patch_number() { - local version="$1" # Input version - local new_patch="$2" # New patch number - # Extract major, minor, and current patch numbers - local major=$(echo "$version" | cut -d. -f1) - local minor=$(echo "$version" | cut -d. -f2) - local current_patch=$(echo "$version" | cut -d. -f3) - # Construct the new version with the updated patch number - local new_version="$major.$minor.$new_patch" - echo "$new_version" +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e } -# Function to fetch the previous version of the plugin +# Function to fetch the version released prior to the latest version get_previous_version() { - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name#v} + + output=$(curl -s "$repo_url"); + + message=$(echo "$output" | jq -r '.message') + + if [ $mode == "mode1" ]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="v${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } get_helm() { @@ -68,53 +153,30 @@ get_helm() { sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" } -latest_version=$(get_latest_version) -NON_EXISTING_PATCH_VERSION="xyz" -HELM_VERSION="$(change_patch_number ${latest_version} ${NON_EXISTING_PATCH_VERSION})" -echo -e "\n๐Ÿ‘‰${HL} Trying to install HELM_VERSION = ${HELM_VERSION}${N}"; -sudo mkdir -p /tmp/helm -get_helm "${HELM_VERSION}" -if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then - echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." - requested_version=$(get_previous_version) - echo -e "\nAttempting to install ${requested_version}" - HELM_VERSION=${requested_version} +install_helm() { + mode=$1 + HELM_VERSION="v3.14.xyz" + echo -e "\n๐Ÿ‘‰Trying to install HELM_VERSION = ${HELM_VERSION}"; + sudo mkdir -p /tmp/helm get_helm "${HELM_VERSION}" -fi -export GNUPGHOME="/tmp/helm/gnupg" -sudo mkdir -p "${GNUPGHOME}" -sudo chmod 700 ${GNUPGHOME} -sudo curl -sSL "${HELM_GPG_KEYS_URI}" -o /tmp/helm/KEYS -sudo echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | sudo tee ${GNUPGHOME}/dirmngr.conf >/dev/null -sudo gpg -q --import "/tmp/helm/KEYS" -if ! sudo gpg --verify "${tmp_helm_filename}.asc" | sudo tee ${GNUPGHOME}/verify.log 2>&1; then - echo "Verification failed!" - sudo cat /tmp/helm/gnupg/verify.log - exit 1 -fi - -if [ "${HELM_SHA256}" = "automatic" ]; then - sudo curl -sSL "https://get.helm.sh/${helm_filename}.sha256" -o "${tmp_helm_filename}.sha256" - sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.sha256.asc" -o "${tmp_helm_filename}.sha256.asc" - if ! sudo gpg --verify "${tmp_helm_filename}.sha256.asc" | sudo tee /tmp/helm/gnupg/verify.log 2>&1; then - echo "Verification failed!" - sudo cat /tmp/helm/gnupg/verify.log - exit 1 + if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." + repo_url=$(get_github_api_repo_url "${helm_url}") + get_previous_version "${helm_url}" "${repo_url}" HELM_VERSION $mode + echo -e "\nAttempting to install ${HELM_VERSION}" + get_helm "${HELM_VERSION}" fi - HELM_SHA256="$(sudo cat "${tmp_helm_filename}.sha256")" -fi - -([ "${HELM_SHA256}" = "dev-mode" ] || (sudo echo "${HELM_SHA256} *${tmp_helm_filename}" | sha256sum -c -)) -sudo tar xf "${tmp_helm_filename}" -C /tmp/helm -sudo mv -f "/tmp/helm/linux-${architecture}/helm" /usr/local/bin/ -sudo chmod 0755 /usr/local/bin/helm -sudo rm -rf /tmp/helm -if ! type helm > /dev/null 2>&1; then - echo '(!) Helm installation failed!' - exit 1 -fi - -echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}:" +} + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}: (mode1: installation using find_prev_version_using_git_tags() fn)" +install_helm "mode1" + +set +e + check "helm version" helm version +set -e + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}: (mode2: installation using GitHub api)" +install_helm "mode2" set +e check "helm version" helm version From 32d2b5baf8071f00297fa0cb9fcfd79b9f238387 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 23:00:49 +0530 Subject: [PATCH 006/224] [docker-outside-of-docker] - alternative fallback implementation (#924) --- .../devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 84 ++++++++- .../docker_build_compose_fallback.sh | 161 ++++++++++++++++-- 3 files changed, 224 insertions(+), 23 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 2a55fdbca..2506031ae 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.4", + "version": "1.4.5", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 6b63469f9..16ad15f60 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -99,19 +99,84 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } -# Function to fetch the previous version of the plugin +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} install_compose_switch_fallback() { + compose_switch_url=$1 + repo_url=$(get_github_api_repo_url "${compose_switch_url}") echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version#v} + get_previous_version "${compose_switch_url}" "${repo_url}" compose_switch_version + echo -e "\nAttempting to install v${compose_switch_version}" curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose } @@ -270,8 +335,9 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then else echo "(*) Installing compose-switch as docker-compose..." compose_switch_version="latest" - find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + compose_switch_url="https://github.com/docker/compose-switch" + find_version_from_git_tags compose_switch_version "${compose_switch_url}" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback "${compose_switch_url}" chmod +x /usr/local/bin/docker-compose # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 fi diff --git a/test/docker-outside-of-docker/docker_build_compose_fallback.sh b/test/docker-outside-of-docker/docker_build_compose_fallback.sh index e1c0885e6..193d79d28 100644 --- a/test/docker-outside-of-docker/docker_build_compose_fallback.sh +++ b/test/docker-outside-of-docker/docker_build_compose_fallback.sh @@ -3,34 +3,169 @@ # Optional: Import test library source dev-container-features-test-lib +echo -e "\n๐Ÿ‘‰ Checking version of compose-switch installed as docker-compose as installed by feature"; check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" +trap 'echo "Last executed command failed at line ${LINENO}"' ERR + # Fetch host/container arch. architecture="$(dpkg --print-architecture)" -repo_url="https://api.github.com/repos/docker/compose-switch/releases" +sudo mkdir -p /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" -# Function to fetch the previous version of the plugin + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - sudo curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $mode == 'mode1' ]]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } install_compose_switch_fallback() { - echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch ${test_compose_switch_version}..." - previous_version=$(get_previous_version) - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version} - sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose + compose_switch_url=$1 + mode=$2 + repo_url=$(get_github_api_repo_url "${compose_switch_url}") + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + get_previous_version "${compose_switch_url}" "${repo_url}" compose_switch_version $mode + echo -e "\nAttempting to install v${compose_switch_version}" + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose } install_compose-switch_as_docker-compose() { + mode=$1 echo "(*) Installing compose-switch as docker-compose..." - test_compose_switch_version="1.2.xyz" - echo -e "\nTesting with $test_compose_switch_version..." - sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${test_compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + compose_switch_version="1.0.6" + compose_switch_url="https://github.com/docker/compose-switch" + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback "${compose_switch_url}" $mode sudo chmod +x /usr/local/bin/docker-compose } -install_compose-switch_as_docker-compose +echo -e "\n๐Ÿ‘‰ Trying to install compose-switch as docker-compose using mode 1 ( find_prev_version_from_git_tags method )"; +install_compose-switch_as_docker-compose "mode1" +check "installs compose-switch as docker-compose mode 1" bash -c "[[ -f /usr/local/bin/docker-compose ]]" -check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file +echo -e "\n๐Ÿ‘‰ Trying to install compose-switch as docker-compose using mode 2 ( GitHub Api )"; +install_compose-switch_as_docker-compose "mode2" +check "installs compose-switch as docker-compose mode 2" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file From 55dbe138fc159c462a6c4554b4619db3ff3b04ab Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Tue, 2 Apr 2024 06:05:32 +0530 Subject: [PATCH 007/224] [PowerShell]- Fallback method with find_prev_vers_frm_git_tags() implemented (#926) * [PowerShell]- Fallback method with find_prev_vers_frm_git_tags() done * bumped the feature version --- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 81 ++++++++- .../install_powershell_fallback_test.sh | 164 ++++++++++++++---- 3 files changed, 201 insertions(+), 46 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index dd7a96b90..82ef39a30 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.4", + "version": "1.3.5", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 4ba8bcf5c..533da6f37 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -106,17 +106,83 @@ install_using_apt() { apt-get install -yq powershell${version_suffix} || return 1 } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + check_packages jq + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + + install_prev_pwsh() { + pwsh_url=$1 + repo_url=$(get_github_api_repo_url $pwsh_url) echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." - previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") - echo -e "\nAttempting to install ${previous_version}" - POWERSHELL_VERSION="${previous_version#v}" + get_previous_version $pwsh_url $repo_url POWERSHELL_VERSION + echo -e "\nAttempting to install v${POWERSHELL_VERSION}" install_pwsh "${POWERSHELL_VERSION}" } @@ -138,10 +204,11 @@ install_using_github() { if [ "${architecture}" = "amd64" ]; then architecture="x64" fi - find_version_from_git_tags POWERSHELL_VERSION https://github.com/PowerShell/PowerShell + pwsh_url="https://github.com/PowerShell/PowerShell" + find_version_from_git_tags POWERSHELL_VERSION $pwsh_url install_pwsh "${POWERSHELL_VERSION}" if grep -q "Not Found" "${powershell_filename}"; then - install_prev_pwsh + install_prev_pwsh $pwsh_url fi # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. diff --git a/test/powershell/install_powershell_fallback_test.sh b/test/powershell/install_powershell_fallback_test.sh index 852d7d103..39c8bc49e 100644 --- a/test/powershell/install_powershell_fallback_test.sh +++ b/test/powershell/install_powershell_fallback_test.sh @@ -15,16 +15,125 @@ check "Powershell version as installed by feature" bash -c "pwsh --version" . /etc/os-release architecture="$(dpkg --print-architecture)" +sudo mkdir -p /var/lib/apt/lists/ + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [ $mode == "mode1" ]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } install_prev_pwsh() { + local pwsh_url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url $pwsh_url) echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." - previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") - echo -e "\nAttempting to install ${previous_version}" - POWERSHELL_VERSION="${previous_version#v}" + get_previous_version $pwsh_url $repo_url POWERSHELL_VERSION $mode + echo -e "\nAttempting to install v${POWERSHELL_VERSION}" install_pwsh "${POWERSHELL_VERSION}" } @@ -37,44 +146,18 @@ install_pwsh() { sudo curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - sudo apt-get update -y - fi -} - -check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - sudo chmod +x /var/lib/apt/lists/ - sudo mkdir -p /var/lib/apt/lists/partial - sudo chmod +rx /var/lib/dpkg/lock-frontend - apt_get_update - sudo apt-get -y install --no-install-recommends "$@" - fi -} - install_using_github() { - # Fall back on direct download if no apt package exists in microsoft pool - check_packages curl ca-certificates gnupg2 dirmngr libc6 libgcc1 libgssapi-krb5-2 libstdc++6 libunwind8 libuuid1 zlib1g libicu[0-9][0-9] - if ! type git > /dev/null 2>&1; then - check_packages git - fi + mode=$1 if [ "${architecture}" = "amd64" ]; then architecture="x64" fi - - echo -e "\nTrying to install a non-existing version for Powershell..." - - POWERSHELL_VERSION="1.2.XYZ" + pwsh_url="https://github.com/PowerShell/PowerShell" + POWERSHELL_VERSION="7.4.xyz" install_pwsh "${POWERSHELL_VERSION}" - if grep -q "Not Found" "${powershell_filename}"; then - install_prev_pwsh + install_prev_pwsh $pwsh_url $mode fi - echo -e "\n" $POWERSHELL_VERSION "=powershell_version\n"; # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. sudo curl -sSL -o "release.html" "https://github.com/PowerShell/PowerShell/releases/tag/v${POWERSHELL_VERSION}" powershell_archive_sha256="$(cat release.html | tr '\n' ' ' | sed 's|<[^>]*>||g' | grep -oP "${powershell_filename}\s+\K[0-9a-fA-F]{64}" || echo '')" @@ -86,12 +169,17 @@ install_using_github() { fi sudo tar xf "${powershell_filename}" -C "${powershell_target_path}" sudo ln -s "${powershell_target_path}/pwsh" /usr/local/bin/pwsh - sudo rm -rf /tmp/pwsh + sudo rm -rf /tmp/pwsh /usr/local/bin/pwsh + } -install_using_github +echo -e "\nInstalling Powershell with find_prev_version_from_git_tags() fn ๐Ÿ‘ˆ๐Ÿป" +install_using_github "mode1" +check "Powershell version as installed by test (find_prev_version_from_git_tags() fn)" bash -c "pwsh --version" -check "Powershell version as installed by test" bash -c "pwsh --version" +echo -e "\nInstalling Powershell with GitHub Api ๐Ÿ‘ˆ๐Ÿป" +install_using_github "mode2" +check "Powershell version as installed by test (GitHub Api)" bash -c "pwsh --version" # Report result reportResults From a7c8c9a83b3ce0f7f0c546537d814009f9e48590 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 3 Apr 2024 03:06:19 +0530 Subject: [PATCH 008/224] [Php] - fallback to previous version - code fix (#908) * [php]- php fallback to prev. version - fix * update patch version for php feature * few changes.. * small change * test completes successfully pointing at the new fallbacked php version * changes acc. to review comments.. --- src/php/devcontainer-feature.json | 2 +- src/php/install.sh | 69 +++++++++- test/php/scenarios.json | 8 ++ test/php/test_php_fallback.sh | 201 ++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 6 deletions(-) create mode 100644 test/php/test_php_fallback.sh diff --git a/src/php/devcontainer-feature.json b/src/php/devcontainer-feature.json index 77fcf9812..a4edcca95 100644 --- a/src/php/devcontainer-feature.json +++ b/src/php/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "php", - "version": "1.1.2", + "version": "1.1.3", "name": "PHP", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/php", "options": { diff --git a/src/php/install.sh b/src/php/install.sh index 48140e1b3..357395e88 100755 --- a/src/php/install.sh +++ b/src/php/install.sh @@ -121,6 +121,46 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Install PHP Composer addcomposer() { "${PHP_SRC}" -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" @@ -130,8 +170,7 @@ addcomposer() { "${PHP_SRC}" -r "unlink('composer-setup.php');" } -install_php() { - PHP_VERSION="$1" +init_php_install() { PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" if [ -d "${PHP_INSTALL_DIR}" ]; then echo "(!) PHP version ${PHP_VERSION} already exists." @@ -142,7 +181,6 @@ install_php() { groupadd -r php fi usermod -a -G php "${USERNAME}" - PHP_URL="https://www.php.net/distributions/php-${PHP_VERSION}.tar.gz" PHP_INI_DIR="${PHP_INSTALL_DIR}/ini" @@ -155,7 +193,26 @@ install_php() { PHP_SRC_DIR="/usr/src/php" mkdir -p $PHP_SRC_DIR cd $PHP_SRC_DIR - wget -O php.tar.xz "$PHP_URL" +} + +install_previous_version() { + PHP_VERSION=$1 + if [[ "$ORIGINAL_PHP_VERSION" == "latest" ]]; then + find_prev_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" + echo -e "\nAttempting to install previous version v${PHP_VERSION}" + init_php_install + wget -O php.tar.xz "$PHP_URL" + else + echo -e "\nFailed to install v$PHP_VERSION" + fi +} + +install_php() { + PHP_VERSION="$1" + + init_php_install + + wget -O php.tar.xz "$PHP_URL" || install_previous_version "$PHP_VERSION" tar -xf $PHP_SRC_DIR/php.tar.xz -C "$PHP_SRC_DIR" --strip-components=1 cd $PHP_SRC_DIR; @@ -195,7 +252,7 @@ install_php() { if [ "${PHP_VERSION}" != "none" ]; then # Persistent / runtime dependencies - RUNTIME_DEPS="wget ca-certificates git build-essential xz-utils" + RUNTIME_DEPS="wget ca-certificates git build-essential xz-utils curl" # PHP dependencies PHP_DEPS="libssl-dev libcurl4-openssl-dev libedit-dev libsqlite3-dev libxml2-dev zlib1g-dev libsodium-dev libonig-dev" @@ -214,6 +271,8 @@ if [ "${PHP_VERSION}" != "none" ]; then # Install dependencies check_packages $RUNTIME_DEPS $PHP_DEPS $PHPIZE_DEPS + # storing value of PHP_VERSION before it changes + ORIGINAL_PHP_VERSION=$PHP_VERSION find_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" install_php "${PHP_VERSION}" diff --git a/test/php/scenarios.json b/test/php/scenarios.json index 1e4df063d..e53bb67ce 100644 --- a/test/php/scenarios.json +++ b/test/php/scenarios.json @@ -32,5 +32,13 @@ "installComposer": true } } + }, + "test_php_fallback": { + "image": "ubuntu:focal", + "features": { + "php": { + "version": "latest" + } + } } } diff --git a/test/php/test_php_fallback.sh b/test/php/test_php_fallback.sh new file mode 100644 index 000000000..84ed83d77 --- /dev/null +++ b/test/php/test_php_fallback.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +echo -e "\nInstalled PHP Version by Feature: ๐Ÿ‘‡ "; php -v; + +USERNAME="root" +PHP_DIR="/usr/local/php" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + echo "${!variable_name}" + echo "$(echo "${requested_version}" | grep -o "." | wc -l)" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + echo "${!variable_name}" + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +init_php_install() { + PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" + if [ -d "${PHP_INSTALL_DIR}" ]; then + echo "(!) PHP version ${PHP_VERSION} already exists." + exit 1 + fi + + if ! cat /etc/group | grep -e "^php:" > /dev/null 2>&1; then + groupadd -r php + fi + usermod -a -G php "${USERNAME}" + + PHP_URL="https://www.php.net/distributions/php-${PHP_VERSION}.tar.gz" + + PHP_INI_DIR="${PHP_INSTALL_DIR}/ini" + CONF_DIR="${PHP_INI_DIR}/conf.d" + mkdir -p "${CONF_DIR}"; + + PHP_EXT_DIR="${PHP_INSTALL_DIR}/extensions" + mkdir -p "${PHP_EXT_DIR}" + + PHP_SRC_DIR="/usr/src/php" + mkdir -p $PHP_SRC_DIR + cd $PHP_SRC_DIR +} + +install_previous_version() { + echo -e "\nInstalling Previous Version..." + find_prev_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" + echo -e "\nNow installing this version as a fallback previous version: ${PHP_VERSION} ๐Ÿคž๐Ÿป" + init_php_install + wget -O php.tar.xz "$PHP_URL" +} + +install_php() { + # trying to install with a possible new tag not having a released source binary yet + PHP_VERSION="8.3.xyz" + + init_php_install + + wget -O php.tar.xz "$PHP_URL" || install_previous_version + + tar -xf $PHP_SRC_DIR/php.tar.xz -C "$PHP_SRC_DIR" --strip-components=1 + cd $PHP_SRC_DIR; + + # PHP 7.4+, the pecl/pear installers are officially deprecated and are removed in PHP 8+ + # Thus, requiring an explicit "--with-pear" + IFS="." + read -a versions <<< "${PHP_VERSION}" + PHP_MAJOR_VERSION=${versions[0]} + PHP_MINOR_VERSION=${versions[1]} + + VERSION_CONFIG="" + if (( $(($PHP_MAJOR_VERSION)) >= 8 )) || (( $(($PHP_MAJOR_VERSION)) == 7 && $(($PHP_MINOR_VERSION)) >= 4 )); then + VERSION_CONFIG="--with-pear" + fi + + ./configure --prefix="${PHP_INSTALL_DIR}" --with-config-file-path="$PHP_INI_DIR" --with-config-file-scan-dir="$CONF_DIR" --enable-option-checking=fatal --with-curl --with-libedit --enable-mbstring --with-openssl --with-zlib --with-password-argon2 --with-sodium=shared "$VERSION_CONFIG" EXTENSION_DIR="$PHP_EXT_DIR"; + + make -j "$(nproc)" + find -type f -name '*.a' -delete + make install + find "${PHP_INSTALL_DIR}" -type f -executable -exec strip --strip-all '{}' + || true + make clean + + cp -v $PHP_SRC_DIR/php.ini-* "$PHP_INI_DIR/"; + cp "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" + + # Install xdebug + "${PHP_INSTALL_DIR}/bin/pecl" install xdebug + XDEBUG_INI="${CONF_DIR}/xdebug.ini" + + echo "zend_extension=${PHP_EXT_DIR}/xdebug.so" > "${XDEBUG_INI}" + echo "xdebug.mode = debug" >> "${XDEBUG_INI}" + echo "xdebug.start_with_request = yes" >> "${XDEBUG_INI}" + echo "xdebug.client_port = 9003" >> "${XDEBUG_INI}" +} + +apt-get purge php.* +PHP_DIR="/usr/local/php" +PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" +PHP_SRC_DIR="/usr/src/php" + +install_php +PHP_SRC="${PHP_INSTALL_DIR}/bin/php" + +updaterc() { + echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." + if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/bash.bashrc + fi + if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/zsh/zshrc + fi +} + +if [ "${PHP_VERSION}" != "none" ]; then + CURRENT_DIR="${PHP_DIR}/current" + if [[ ! -d "${CURRENT_DIR}" ]]; then + ln -s -r "${PHP_INSTALL_DIR}" ${CURRENT_DIR} + fi + + if [[ $(ls -l ${CURRENT_DIR}) != *"-> ${PHP_INSTALL_DIR}"* ]] ; then + rm "${CURRENT_DIR}" + ln -s -r "${PHP_INSTALL_DIR}" "${CURRENT_DIR}" + fi + + rm -rf "${PHP_SRC_DIR}" + updaterc "if [[ \"\${PATH}\" != *\"${CURRENT_DIR}\"* ]]; then export PATH=\"${CURRENT_DIR}/bin:\${PATH}\"; fi" + + chown -R "${USERNAME}:php" "${PHP_DIR}" + chmod -R g+r+w "${PHP_DIR}" + find "${PHP_DIR}" -type d -print0 | xargs -n 1 -0 chmod g+s +fi + +echo -e "\nInstalled PHP Version by Test: ๐Ÿ‘‡ "; php -v; + From 9ccc19e1378ba3b569784236bbb32499dcea138e Mon Sep 17 00:00:00 2001 From: WarrenS Date: Tue, 2 Apr 2024 17:42:58 -0400 Subject: [PATCH 009/224] Added versions 1.71-1.76 to Rust feature version (#929) * Update devcontainer-feature.json * Updated version --- src/rust/devcontainer-feature.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rust/devcontainer-feature.json b/src/rust/devcontainer-feature.json index 7ee455f3f..c4a7dd5b9 100644 --- a/src/rust/devcontainer-feature.json +++ b/src/rust/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "rust", - "version": "1.1.1", + "version": "1.1.2", "name": "Rust", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/rust", "description": "Installs Rust, common Rust utilities, and their required dependencies", @@ -10,6 +10,12 @@ "proposals": [ "latest", "none", + "1.76", + "1.75", + "1.74", + "1.73", + "1.72", + "1.71", "1.70", "1.69", "1.68", From 203dc3f5bde1a8ca25525234757ac54e9b8da64c Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Fri, 5 Apr 2024 04:23:20 +0530 Subject: [PATCH 010/224] oryx dotnet 8.0.1 cleanup (#927) --- src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 1 + test/oryx/test_python_project.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index a075096bf..41bd1f759 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.1", + "version": "1.3.2", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index 407406141..6c193f863 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -241,6 +241,7 @@ if [[ "${PINNED_SDK_VERSION}" != "" ]]; then MAJOR_MINOR_PATCH1_VERSION=${PINNED_SDK_VERSION%??} rm -rf /usr/share/dotnet/shared/Microsoft.NETCore.App/$MAJOR_MINOR_PATCH1_VERSION rm -rf /usr/share/dotnet/shared/Microsoft.AspNetCore.App/$MAJOR_MINOR_PATCH1_VERSION + rm -rf /usr/share/dotnet/templates/$MAJOR_MINOR_PATCH1_VERSION fi diff --git a/test/oryx/test_python_project.sh b/test/oryx/test_python_project.sh index a0d5eef08..610a9687e 100644 --- a/test/oryx/test_python_project.sh +++ b/test/oryx/test_python_project.sh @@ -28,5 +28,7 @@ check "oryx-build-python" oryx build --property python_version="${pythonVersion} check "oryx-build-python-installed" python3 -m pip list | grep mpmath check "oryx-build-python-result" python3 ./src/solve.py +check "templates/8.0.1-does-not-exist" test ! -d "/usr/share/dotnet/templates/8.0.1" + # Report result reportResults From 760f2bf10b30340e55a2cdd9862a994262f13d81 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Louazel Date: Thu, 11 Apr 2024 18:40:05 +0200 Subject: [PATCH 011/224] Set `DEBIAN_FRONTEND=noninteractive` for nvidia-cuda feature (#933) * Set DEBIAN_FRONTEND to noninteractive Signed-off-by: Jean-Baptiste Louazel * Bump nvidia-cuda to 1.1.1 Signed-off-by: Jean-Baptiste Louazel * Install `liburcu6` Signed-off-by: Jean-Baptiste Louazel * Revert "Install `liburcu6`" This reverts commit b7b2931f8bed57826f2019ea884cbe2440e6c9e8. --------- Signed-off-by: Jean-Baptiste Louazel --- src/nvidia-cuda/devcontainer-feature.json | 2 +- src/nvidia-cuda/install.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nvidia-cuda/devcontainer-feature.json b/src/nvidia-cuda/devcontainer-feature.json index 78ad10cd9..bb63ae1c7 100644 --- a/src/nvidia-cuda/devcontainer-feature.json +++ b/src/nvidia-cuda/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "nvidia-cuda", - "version": "1.1.0", + "version": "1.1.1", "name": "NVIDIA CUDA", "description": "Installs shared libraries for NVIDIA CUDA.", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/nvidia-cuda", diff --git a/src/nvidia-cuda/install.sh b/src/nvidia-cuda/install.sh index cb66d3955..d8658964e 100644 --- a/src/nvidia-cuda/install.sh +++ b/src/nvidia-cuda/install.sh @@ -33,6 +33,8 @@ check_packages() { fi } +export DEBIAN_FRONTEND=noninteractive + check_packages wget ca-certificates # Add NVIDIA's package repository to apt so that we can download packages From e7dd9fafd9aeede11d0a59a0ace819e3b774de2a Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Fri, 12 Apr 2024 23:33:24 +0530 Subject: [PATCH 012/224] cp command to follow symlink for systemctl (#937) --- src/common-utils/devcontainer-feature.json | 2 +- src/common-utils/main.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index a058864af..308e256ef 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.4.2", + "version": "2.4.3", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index 8b74830cc..5d7592cfc 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -564,7 +564,7 @@ chmod +rx /usr/local/bin/code # systemctl shim for Debian/Ubuntu - tells people to use 'service' if systemd is not running if [ "${ADJUSTED_ID}" = "debian" ]; then - cp -f "${FEATURE_DIR}/bin/systemctl" /usr/local/bin/systemctl + cp -fL "${FEATURE_DIR}/bin/systemctl" /usr/local/bin/systemctl chmod +rx /usr/local/bin/systemctl fi From b98f5a164be78317af27118e9491d38c17eb16a4 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 17 Apr 2024 04:28:57 +0530 Subject: [PATCH 013/224] [az-cli] - To separate the two methods of installation - using apt, using python (#922) * [az-cli] - To separate the two methods of installation - apt & python * bump patch version * changes as requested by review comments * no need to keep * added test script missing * changes for review comment --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 10 +++++---- ..._using_python_with_python_3_11_bullseye.sh | 21 +++++++++++++++++++ .../install_with_python_3_12_bookworm.sh | 12 +++++------ test/azure-cli/scenarios.json | 10 +++++++++ 5 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 test/azure-cli/install_using_python_with_python_3_11_bullseye.sh diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index f25e5f9a2..6b26ef6fc 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.3", + "version": "1.2.4", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index 7f4c4ff1c..a1b254779 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -15,7 +15,7 @@ rm -rf /var/lib/apt/lists/* AZ_VERSION=${VERSION:-"latest"} AZ_EXTENSIONS=${EXTENSIONS} AZ_INSTALLBICEP=${INSTALLBICEP:-false} -INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-false} +INSTALL_USING_PYTHON=${INSTALLUSINGPYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" @@ -188,13 +188,15 @@ echo "(*) Installing Azure CLI..." . /etc/os-release architecture="$(dpkg --print-architecture)" CACHED_AZURE_VERSION="${AZ_VERSION}" # In case we need to fallback to pip and the apt path has modified the AZ_VERSION variable. -if [[ "${AZCLI_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${AZCLI_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then - install_using_apt || use_pip="true" +if [ "${INSTALL_USING_PYTHON}" != "true" ]; then + if [[ "${AZCLI_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${AZCLI_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then + install_using_apt || use_pip="true" + fi else use_pip="true" fi -if [ "${use_pip}" = "true" ]; then +if [ "${use_pip}" = "true" ]; then AZ_VERSION=${CACHED_AZURE_VERSION} install_using_pip_strategy diff --git a/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh b/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh new file mode 100644 index 000000000..b9957843e --- /dev/null +++ b/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version + +echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"'; then + echo -e "\n\nโœ… Passed 'O.S is Linux 11 (bullseye)'!" +else + echo -e "\n\nโŒ Failed 'O.S is other than Linux 11 (bullseye)'!" +fi + + +# Report result +reportResults \ No newline at end of file diff --git a/test/azure-cli/install_with_python_3_12_bookworm.sh b/test/azure-cli/install_with_python_3_12_bookworm.sh index 074876148..2c8e1fd72 100644 --- a/test/azure-cli/install_with_python_3_12_bookworm.sh +++ b/test/azure-cli/install_with_python_3_12_bookworm.sh @@ -5,17 +5,17 @@ set -e # Import test library for `check` command source dev-container-features-test-lib -# Check to make sure the user is vscode -check "user is vscode" whoami | grep vscode -check "version" az --version -echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +echo -e "\n๐Ÿ”„ Testing 'O.S'" if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"'; then - echo -e "\n\nโœ… Passed 'O.S is Linux 12 (bookworm)'!" + echo -e "\nโœ… Passed 'O.S is Linux 12 (bookworm)'!\n" else - echo -e "\n\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!" + echo -e "\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!\n" fi +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version # Report result reportResults \ No newline at end of file diff --git a/test/azure-cli/scenarios.json b/test/azure-cli/scenarios.json index 041f50731..3ec910399 100644 --- a/test/azure-cli/scenarios.json +++ b/test/azure-cli/scenarios.json @@ -47,5 +47,15 @@ "version": "latest" } } + }, + "install_using_python_with_python_3_11_bullseye": { + "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye", + "user": "vscode", + "features": { + "azure-cli": { + "version": "latest", + "installUsingPython": "true" + } + } } } \ No newline at end of file From 6f4e59866169405c7b7a8ff65e3f2ac3ced6a26e Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 17 Apr 2024 04:35:23 +0530 Subject: [PATCH 014/224] [Ruby]- rvm - fallback code fix (#931) * [Ruby] - Install using fallback - draft * [Ruby] - Rvm - fallback logic implementation * misc changes * changes for review comments.. --- src/ruby/devcontainer-feature.json | 2 +- src/ruby/install.sh | 129 ++++++++++++-- test/ruby/ruby_fallback_test.sh | 277 +++++++++++++++++++++++++++++ test/ruby/scenarios.json | 8 + 4 files changed, 396 insertions(+), 20 deletions(-) create mode 100644 test/ruby/ruby_fallback_test.sh diff --git a/src/ruby/devcontainer-feature.json b/src/ruby/devcontainer-feature.json index 73bcbcced..3722cab06 100644 --- a/src/ruby/devcontainer-feature.json +++ b/src/ruby/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "ruby", - "version": "1.2.0", + "version": "1.2.1", "name": "Ruby (via rvm)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/ruby", "description": "Installs Ruby, rvm, rbenv, common Ruby utilities, and needed dependencies.", diff --git a/src/ruby/install.sh b/src/ruby/install.sh index 7e4514bba..8f95829da 100755 --- a/src/ruby/install.sh +++ b/src/ruby/install.sh @@ -140,6 +140,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + apt_get_update() { if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then @@ -173,25 +214,46 @@ if ! type git > /dev/null 2>&1; then check_packages git fi +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + #install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" "_" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name' | tr '_' '.') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + # Figure out correct version of a three part version number is not passed -find_version_from_git_tags RUBY_VERSION "https://github.com/ruby/ruby" "tags/v" "_" +RUBY_URL="https://github.com/ruby/ruby" +ORIGINAL_RUBY_VERSION=$RUBY_VERSION +find_version_from_git_tags RUBY_VERSION $RUBY_URL "tags/v" "_" -# Just install Ruby if RVM already installed -if rvm --version > /dev/null; then - echo "Ruby Version Manager already exists." - if [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then - echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." - elif [ "${RUBY_VERSION}" != "none" ]; then - echo "Installing specified Ruby version." - su ${USERNAME} -c "rvm install ruby ${RUBY_VERSION}" - fi - SKIP_GEM_INSTALL="false" - SKIP_RBENV_RBUILD="true" -else - # Install RVM - receive_gpg_keys RVM_GPG_KEYS - # Determine appropriate settings for rvm installer +set_rvm_install_args() { + RUBY_VERSION=$1 if [ "${RUBY_VERSION}" = "none" ]; then RVM_INSTALL_ARGS="" elif [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then @@ -210,19 +272,48 @@ else DEFAULT_GEMS="" fi fi +} + +install_previous_version() { + if [[ $ORIGINAL_RUBY_VERSION == "latest" ]]; then + repo_url=$(get_github_api_repo_url "$RUBY_URL") + get_previous_version "${RUBY_URL}" "${repo_url}" RUBY_VERSION + set_rvm_install_args $RUBY_VERSION + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 + else + echo "Failed to install Ruby version $ORIGINAL_RUBY_VERSION. Exiting..." + fi +} + +# Just install Ruby if RVM already installed +if rvm --version > /dev/null; then + echo "Ruby Version Manager already exists." + if [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then + echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." + elif [ "${RUBY_VERSION}" != "none" ]; then + echo "Installing specified Ruby version." + su ${USERNAME} -c "rvm install ruby ${RUBY_VERSION}" + fi + SKIP_GEM_INSTALL="false" + SKIP_RBENV_RBUILD="true" +else + # Install RVM + receive_gpg_keys RVM_GPG_KEYS + # Determine appropriate settings for rvm installer + set_rvm_install_args $RUBY_VERSION # Create rvm group as a system group to reduce the odds of conflict with local user UIDs if ! cat /etc/group | grep -e "^rvm:" > /dev/null 2>&1; then groupadd -r rvm fi # Install rvm - curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 || install_previous_version usermod -aG rvm ${USERNAME} source /usr/local/rvm/scripts/rvm rvm fix-permissions system rm -rf ${GNUPGHOME} fi -if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then +if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then # Non-root user may not have "gem" in path when script is run and no ruby version # is installed by rvm, so handle this by using root's default gem in this case ROOT_GEM="$(which gem || echo "")" @@ -239,7 +330,7 @@ if [ ! -z "${ADDITIONAL_VERSIONS}" ]; then read -a additional_versions <<< "$ADDITIONAL_VERSIONS" for version in "${additional_versions[@]}"; do # Figure out correct version of a three part version number is not passed - find_version_from_git_tags version "https://github.com/ruby/ruby" "tags/v" "_" + find_version_from_git_tags version $RUBY_URL "tags/v" "_" source /usr/local/rvm/scripts/rvm rvm install ruby ${version} done diff --git a/test/ruby/ruby_fallback_test.sh b/test/ruby/ruby_fallback_test.sh new file mode 100644 index 000000000..fe1a9c77b --- /dev/null +++ b/test/ruby/ruby_fallback_test.sh @@ -0,0 +1,277 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +USERNAME="automatic" +echo -e "\nRVM version installed previously by ruby feature ..." +check "rvm" rvm --version +check "ruby" ruby -v + +trap 'echo "Last executed command failed at line ${LINENO}"' ERR + +RVM_GPG_KEYS="409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB" +GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com +keyserver hkp://keyserver.ubuntu.com:80 +keyserver hkps://keys.openpgp.org +keyserver hkp://keyserver.pgp.com" + +# Clean up +rm -rf /var/lib/apt/lists/* + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +architecture="$(uname -m)" +if [ "${architecture}" != "amd64" ] && [ "${architecture}" != "x86_64" ] && [ "${architecture}" != "arm64" ] && [ "${architecture}" != "aarch64" ]; then + echo "(!) Architecture $architecture unsupported" + exit 1 +fi + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Import the specified key in a variable name passed in as +receive_gpg_keys() { + local keys=${!1} + local keyring_args="" + if [ ! -z "$2" ]; then + keyring_args="--no-default-keyring --keyring \"$2\"" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | tee ${GNUPGHOME}/dirmngr.conf > /dev/null + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + set +e + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + #install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $mode == "mode1" ]]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" "_" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name' | tr '_' '.') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + + +# Figure out correct version of a three part version number is not passed +ruby_url="https://github.com/ruby/ruby" + +RUBY_VERSION="3.1.xyz" + +set_rvm_install_args() { + RUBY_VERSION=$1 + if [ "${RUBY_VERSION}" = "none" ]; then + RVM_INSTALL_ARGS="" + elif [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then + echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." + RVM_INSTALL_ARGS="" + else + if [ "${RUBY_VERSION}" = "latest" ] || [ "${RUBY_VERSION}" = "current" ] || [ "${RUBY_VERSION}" = "lts" ]; then + RVM_INSTALL_ARGS="--ruby" + RUBY_VERSION="" + else + RVM_INSTALL_ARGS="--ruby=${RUBY_VERSION}" + fi + if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then + SKIP_GEM_INSTALL="true" + else + DEFAULT_GEMS="" + fi + fi +} + +install_previous_version() { + mode=$1 + repo_url=$(get_github_api_repo_url "$ruby_url") + get_previous_version "${ruby_url}" "${repo_url}" RUBY_VERSION $mode + set_rvm_install_args $RUBY_VERSION + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 +} + +install_rvm() { + mode=$1 + # Install RVM + receive_gpg_keys RVM_GPG_KEYS + # Determine appropriate settings for rvm installer + set_rvm_install_args $RUBY_VERSION + # Create rvm group as a system group to reduce the odds of conflict with local user UIDs + if ! cat /etc/group | grep -e "^rvm:" > /dev/null 2>&1; then + groupadd -r rvm + fi + # Install rvm + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 || install_previous_version "$mode" + sudo usermod -aG rvm ${USERNAME} + source /usr/local/rvm/scripts/rvm + rvm fix-permissions system + rm -rf ${GNUPGHOME} +} + +install_rvm "mode1" +echo -e "\n๐Ÿ‘‰๐Ÿป๐Ÿ‘‰๐ŸปRVM version installed by test file ... (mode: 1 - install using find_prev_version_from_git_tags):" +check "rvm" rvm --version + +install_rvm "mode2" +echo -e "\n๐Ÿ‘‰๐Ÿป๐Ÿ‘‰๐ŸปRVM version installed by test file ... (mode: 1 - install using GitHub Api):" +check "rvm" rvm --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/ruby/scenarios.json b/test/ruby/scenarios.json index 04cac73f5..cfa2d8554 100644 --- a/test/ruby/scenarios.json +++ b/test/ruby/scenarios.json @@ -13,5 +13,13 @@ "features": { "ruby": {} } + }, + "ruby_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:bullseye", + "features": { + "ruby": { + "version": "latest" + } + } } } \ No newline at end of file From bb7b7ea29f84ea31262c1290a2e14e9984286295 Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:50:07 +0530 Subject: [PATCH 015/224] upgrade cuda version to 11.7 and cudnn to 8.5.0 (#942) upgrade cuda version 11.7 and cudnn 8.5.0 --- src/nvidia-cuda/devcontainer-feature.json | 2 +- src/nvidia-cuda/install.sh | 7 +++++++ test/nvidia-cuda/install_cudnn_nvxt_version.sh | 8 ++++---- test/nvidia-cuda/scenarios.json | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/nvidia-cuda/devcontainer-feature.json b/src/nvidia-cuda/devcontainer-feature.json index bb63ae1c7..4a0fb0834 100644 --- a/src/nvidia-cuda/devcontainer-feature.json +++ b/src/nvidia-cuda/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "nvidia-cuda", - "version": "1.1.1", + "version": "1.1.2", "name": "NVIDIA CUDA", "description": "Installs shared libraries for NVIDIA CUDA.", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/nvidia-cuda", diff --git a/src/nvidia-cuda/install.sh b/src/nvidia-cuda/install.sh index d8658964e..79a7a9a20 100644 --- a/src/nvidia-cuda/install.sh +++ b/src/nvidia-cuda/install.sh @@ -12,6 +12,8 @@ INSTALL_TOOLKIT=${INSTALLTOOLKIT} CUDA_VERSION=${CUDAVERSION} CUDNN_VERSION=${CUDNNVERSION} +. /etc/os-release + if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 @@ -33,6 +35,11 @@ check_packages() { fi } +if [ $VERSION_CODENAME = "bookworm" ] || [ $VERSION_CODENAME = "jammy" ] && [ $CUDA_VERSION \< 11.7 ]; then + echo "(!) Unsupported distribution version '${VERSION_CODENAME}' for CUDA < 11.7" + exit 1 +fi + export DEBIAN_FRONTEND=noninteractive check_packages wget ca-certificates diff --git a/test/nvidia-cuda/install_cudnn_nvxt_version.sh b/test/nvidia-cuda/install_cudnn_nvxt_version.sh index a7f46bdd6..4817ae479 100644 --- a/test/nvidia-cuda/install_cudnn_nvxt_version.sh +++ b/test/nvidia-cuda/install_cudnn_nvxt_version.sh @@ -5,11 +5,11 @@ set -e # Optional: Import test library source dev-container-features-test-lib -# Check installation of libcudnn8 (8.3.2) -check "libcudnn.so.8.3.2" test 1 -eq "$(find /usr -name 'libcudnn.so.8.3.2' | wc -l)" +# Check installation of libcudnn8 (8.5.0) +check "libcudnn.so.8.5.0" test 1 -eq "$(find /usr -name 'libcudnn.so.8.5.0' | wc -l)" -# Check installation of cuda-nvtx-11-5 (11.5) -check "cuda-11-5+nvtx" test -e '/usr/local/cuda-11.5/targets/x86_64-linux/include/nvtx3' +# Check installation of cuda-nvtx-11-7 (11.7) +check "cuda-11-7+nvtx" test -e '/usr/local/cuda-11.7/targets/x86_64-linux/include/nvtx3' # Report result reportResults diff --git a/test/nvidia-cuda/scenarios.json b/test/nvidia-cuda/scenarios.json index 3018330e2..bd73263b3 100644 --- a/test/nvidia-cuda/scenarios.json +++ b/test/nvidia-cuda/scenarios.json @@ -16,8 +16,8 @@ "nvidia-cuda": { "installCudnn": true, "installNvtx": true, - "cudaVersion": "11.5", - "cudnnVersion": "8.3.2.44" + "cudaVersion": "11.7", + "cudnnVersion": "8.5.0.96" } } } From d8e9d335952d0b8d488cc3445ad076003bc3c22c Mon Sep 17 00:00:00 2001 From: Barrett Strausser Date: Tue, 7 May 2024 11:10:39 -0400 Subject: [PATCH 016/224] Fix rustup-init sha256sum check (#962) * Fix rustup-init sha256sum check * Semver update --------- Co-authored-by: bstrausser --- src/rust/devcontainer-feature.json | 2 +- src/rust/install.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rust/devcontainer-feature.json b/src/rust/devcontainer-feature.json index c4a7dd5b9..70013442f 100644 --- a/src/rust/devcontainer-feature.json +++ b/src/rust/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "rust", - "version": "1.1.2", + "version": "1.1.3", "name": "Rust", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/rust", "description": "Installs Rust, common Rust utilities, and their required dependencies", diff --git a/src/rust/install.sh b/src/rust/install.sh index 00c0a6e72..4db9edc1e 100755 --- a/src/rust/install.sh +++ b/src/rust/install.sh @@ -186,6 +186,7 @@ else curl -sSL --proto '=https' --tlsv1.2 "https://static.rust-lang.org/rustup/dist/${download_architecture}-unknown-linux-gnu/rustup-init" -o /tmp/rustup/target/${download_architecture}-unknown-linux-gnu/release/rustup-init curl -sSL --proto '=https' --tlsv1.2 "https://static.rust-lang.org/rustup/dist/${download_architecture}-unknown-linux-gnu/rustup-init.sha256" -o /tmp/rustup/rustup-init.sha256 cd /tmp/rustup + cp /tmp/rustup/target/${download_architecture}-unknown-linux-gnu/release/rustup-init /tmp/rustup/rustup-init sha256sum -c rustup-init.sha256 chmod +x target/${download_architecture}-unknown-linux-gnu/release/rustup-init target/${download_architecture}-unknown-linux-gnu/release/rustup-init -y --no-modify-path --profile ${RUSTUP_PROFILE} ${default_toolchain_arg} From 67c10a660868260c284b02f2878dcfdcfd91279f Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 13 May 2024 17:37:11 +0100 Subject: [PATCH 017/224] =?UTF-8?q?=E2=9C=A8=20Add=20Ubuntu=2024=20Noble?= =?UTF-8?q?=20to=20`docker-in-docker`=20(#971)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Ubuntu Noble Signed-off-by: GitHub * Changes Signed-off-by: GitHub --------- Signed-off-by: GitHub --- .github/workflows/test-all.yaml | 1 + .github/workflows/test-pr.yaml | 1 + src/docker-in-docker/devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index 40f5efa75..7a9554314 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -48,6 +48,7 @@ jobs: "debian:12", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", + "mcr.microsoft.com/devcontainers/base:noble" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 3aae841e2..776a9731f 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -55,6 +55,7 @@ jobs: "debian:12", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", + "mcr.microsoft.com/devcontainers/base:noble" ] steps: - uses: actions/checkout@v3 diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 812db444c..4897ebf3e 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.2", + "version": "2.11.0", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 0dc9e52d1..ee9cb6ee6 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -18,8 +18,8 @@ USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" INSTALL_DOCKER_COMPOSE_SWITCH="${INSTALLDOCKERCOMPOSESWITCH:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" -DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy noble" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy noble" # Default: Exit on any failure. set -e From 4d2dabec57722e23ffe547038923a799e332e291 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 17 May 2024 00:12:03 +0530 Subject: [PATCH 018/224] [Desktop-lite]- libasound2 not installing in noble - issue (#973) * [Desktop-lite]- libasound2 not installing in noble - issue * bump to patch version in Desktop-lite feature * misc change * Changes for comments ( review comments ) * changes based on review comments.. --- src/desktop-lite/devcontainer-feature.json | 2 +- src/desktop-lite/install.sh | 11 +++++++- test/desktop-lite/test.sh | 30 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/desktop-lite/devcontainer-feature.json b/src/desktop-lite/devcontainer-feature.json index 5387138b5..5386eb31d 100644 --- a/src/desktop-lite/devcontainer-feature.json +++ b/src/desktop-lite/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "desktop-lite", - "version": "1.0.8", + "version": "1.1.0", "name": "Light-weight Desktop", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/desktop-lite", "description": "Adds a lightweight Fluxbox based desktop to the container that can be accessed using a VNC viewer or the web. GUI-based commands executed from the built-in VS code terminal will open on the desktop automatically.", diff --git a/src/desktop-lite/install.sh b/src/desktop-lite/install.sh index df4390eca..13a524ada 100755 --- a/src/desktop-lite/install.sh +++ b/src/desktop-lite/install.sh @@ -41,7 +41,6 @@ package_list=" libnotify4 \ libnss3 \ libxss1 \ - libasound2 \ xfonts-base \ xfonts-terminus \ fonts-noto \ @@ -198,6 +197,16 @@ fi # Install X11, fluxbox and VS Code dependencies check_packages ${package_list} +# if Ubuntu-24.04, noble(numbat) found, then will install libasound2-dev instead of libasound2. +# this change is temporary, https://packages.ubuntu.com/noble/libasound2 will switch to libasound2 once it is available for Ubuntu-24.04, noble(numbat) +. /etc/os-release +if [ "${ID}" = "ubuntu" ] && [ "${VERSION_CODENAME}" = "noble" ]; then + echo "Ubuntu 24.04, Noble(Numbat) detected. Installing libasound2-dev package..." + check_packages "libasound2-dev" +else + check_packages "libasound2" +fi + # On newer versions of Ubuntu (22.04), # we need an additional package that isn't provided in earlier versions if ! type vncpasswd > /dev/null 2>&1; then diff --git a/test/desktop-lite/test.sh b/test/desktop-lite/test.sh index 9009aa9c6..5d11dd424 100755 --- a/test/desktop-lite/test.sh +++ b/test/desktop-lite/test.sh @@ -5,9 +5,39 @@ set -e # Optional: Import test library source dev-container-features-test-lib +echoStderr() +{ + echo "$@" 1>&2 +} + +checkOSPackage() { + LABEL=$1 + PACKAGE_NAME=$2 + echo -e "\n๐Ÿงช Testing $LABEL" + # Check if the package exists and retrieve its exact version + if [ "$(dpkg-query -W -f='${Status}' "$PACKAGE_NAME" 2>/dev/null | grep -c "ok installed")" -eq 1 ]; then + echo "โœ… Package '$PACKAGE_NAME' is installed." + exit 0 + else + echo "โŒ Package '$PACKAGE_NAME' is not installed." + exit 1 + fi +} + check "desktop-init-exists" bash -c "ls /usr/local/share/desktop-init.sh" check "log-exists" bash -c "ls /tmp/container-init.log" check "fluxbox-exists" bash -c "ls -la ~/.fluxbox" +. /etc/os-release +if [ "${ID}" = "ubuntu" ]; then + if [ "${VERSION_CODENAME}" = "noble" ]; then + checkOSPackage "if libasound2-dev exists !" "libasound2-dev" + else + checkOSPackage "if libasound2 exists !" "libasound2" + fi +else + checkOSPackage "if libasound2 exists !" "libasound2" +fi + # Report result reportResults \ No newline at end of file From ecbfd50952e513db872d8d3380e069ccf74a70a8 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Fri, 17 May 2024 09:16:22 -0700 Subject: [PATCH 019/224] [Updates] Automated vendor dotnet-install script (#970) * Automated dotnet-install script update * Bump version --------- Co-authored-by: github-actions --- src/dotnet/devcontainer-feature.json | 2 +- src/dotnet/scripts/vendor/dotnet-install.sh | 26 +++++++++++++++++---- src/oryx/devcontainer-feature.json | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 78a061d23..8b8ffaf2a 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.5", + "version": "2.0.6", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", diff --git a/src/dotnet/scripts/vendor/dotnet-install.sh b/src/dotnet/scripts/vendor/dotnet-install.sh index f6b08d1f8..42c201af4 100755 --- a/src/dotnet/scripts/vendor/dotnet-install.sh +++ b/src/dotnet/scripts/vendor/dotnet-install.sh @@ -298,6 +298,10 @@ get_machine_architecture() { if command -v uname > /dev/null; then CPUName=$(uname -m) case $CPUName in + armv1*|armv2*|armv3*|armv4*|armv5*|armv6*) + echo "armv6-or-below" + return 0 + ;; armv*l) echo "arm" return 0 @@ -339,7 +343,13 @@ get_normalized_architecture_from_architecture() { local architecture="$(to_lowercase "$1")" if [[ $architecture == \ ]]; then - echo "$(get_machine_architecture)" + machine_architecture="$(get_machine_architecture)" + if [[ "$machine_architecture" == "armv6-or-below" ]]; then + say_err "Architecture \`$machine_architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" + return 1 + fi + + echo $machine_architecture return 0 fi @@ -1013,7 +1023,7 @@ extract_dotnet_package() { rm -rf "$temp_out_path" if [ -z ${keep_zip+x} ]; then - rm -f "$zip_path" && say_verbose "Temporary zip file $zip_path was removed" + rm -f "$zip_path" && say_verbose "Temporary archive file $zip_path was removed" fi if [ "$failed" = true ]; then @@ -1261,6 +1271,12 @@ get_download_link_from_aka_ms() { http_codes=$( echo "$response" | awk '$1 ~ /^HTTP/ {print $2}' ) # They all need to be 301, otherwise some links are broken (except for the last, which is not a redirect but 200 or 404). broken_redirects=$( echo "$http_codes" | sed '$d' | grep -v '301' ) + # The response may end without final code 2xx/4xx/5xx somehow, e.g. network restrictions on www.bing.com causes redirecting to bing.com fails with connection refused. + # In this case it should not exclude the last. + last_http_code=$( echo "$http_codes" | tail -n 1 ) + if ! [[ $last_http_code =~ ^(2|4|5)[0-9][0-9]$ ]]; then + broken_redirects=$( echo "$http_codes" | grep -v '301' ) + fi # All HTTP codes are 301 (Moved Permanently), the redirect link exists. if [[ -z "$broken_redirects" ]]; then @@ -1512,7 +1528,7 @@ install_dotnet() { mkdir -p "$install_root" zip_path="${zip_path:-$(mktemp "$temporary_file_template")}" - say_verbose "Zip path: $zip_path" + say_verbose "Archive path: $zip_path" for link_index in "${!download_links[@]}" do @@ -1536,7 +1552,7 @@ install_dotnet() { say "Failed to download $link_type link '$download_link': $download_error_msg" ;; esac - rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" + rm -f "$zip_path" 2>&1 && say_verbose "Temporary archive file $zip_path was removed" else download_completed=true break @@ -1551,7 +1567,7 @@ install_dotnet() { remote_file_size="$(get_remote_file_size "$download_link")" - say "Extracting zip from $download_link" + say "Extracting archive from $download_link" extract_dotnet_package "$zip_path" "$install_root" "$remote_file_size" || return 1 # Check if the SDK version is installed; if not, fail the installation. diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index 41bd1f759..9e6e698a1 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.2", + "version": "1.3.3", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", From f5787eed01022f177475a99084327e023a84ddaf Mon Sep 17 00:00:00 2001 From: Andy Li Date: Wed, 22 May 2024 00:57:40 +0100 Subject: [PATCH 020/224] Add Ubuntu 24 Noble to `docker-outside-of-docker` (#978) --- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 2506031ae..19018ea8e 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.5", + "version": "1.5.0", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 16ad15f60..de5212fe6 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -19,8 +19,8 @@ USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" -DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy noble" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy noble" set -e From 02b71cbd6cf972ca29059cd409dc0fe8c3b60e65 Mon Sep 17 00:00:00 2001 From: hellodword <46193371+hellodword@users.noreply.github.com> Date: Tue, 28 May 2024 23:03:23 +0000 Subject: [PATCH 021/224] [python] add default formatter (#903) Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 9f211244f..ef16ee643 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.4.2", + "version": "1.5.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", @@ -73,10 +73,14 @@ "vscode": { "extensions": [ "ms-python.python", - "ms-python.vscode-pylance" + "ms-python.vscode-pylance", + "ms-python.autopep8" ], "settings": { - "python.defaultInterpreterPath": "/usr/local/python/current/bin/python" + "python.defaultInterpreterPath": "/usr/local/python/current/bin/python", + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8" + } } } }, @@ -84,4 +88,4 @@ "ghcr.io/devcontainers/features/common-utils", "ghcr.io/devcontainers/features/oryx" ] -} \ No newline at end of file +} From 10ea0b7dd5a653b08266039527f7e095c02591a8 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 28 May 2024 16:20:00 -0700 Subject: [PATCH 022/224] Automated documentation update (#984) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/python/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/README.md b/src/python/README.md index db6ad1039..90d79aee8 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -31,6 +31,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho - `ms-python.python` - `ms-python.vscode-pylance` +- `ms-python.autopep8` From 476a68d0523b004112498dc161c2b6de1bd9fe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Wed, 29 May 2024 21:43:03 +0200 Subject: [PATCH 023/224] [desktop-lite] Allow password-less VNC connections (#982) * [desktop-lite] Allow password-less VNC connections Closes #611 * Restore readme * Update src/desktop-lite/install.sh Co-authored-by: Samruddhi Khandale * Fix --------- Co-authored-by: Samruddhi Khandale --- src/desktop-lite/devcontainer-feature.json | 13 +++++----- src/desktop-lite/install.sh | 29 +++++++++++++++++++--- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/desktop-lite/devcontainer-feature.json b/src/desktop-lite/devcontainer-feature.json index 5386eb31d..417787c51 100644 --- a/src/desktop-lite/devcontainer-feature.json +++ b/src/desktop-lite/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "desktop-lite", - "version": "1.1.0", + "version": "1.2.0", "name": "Light-weight Desktop", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/desktop-lite", "description": "Adds a lightweight Fluxbox based desktop to the container that can be accessed using a VNC viewer or the web. GUI-based commands executed from the built-in VS code terminal will open on the desktop automatically.", @@ -19,17 +19,18 @@ "1.2.0" ], "default": "1.2.0", - "description": "NoVnc Version" + "description": "The noVNC version to use" }, "password": { "type": "string", "proposals": [ "vscode", "codespaces", - "password" + "password", + "noPassword" ], "default": "vscode", - "description": "Enter a password for desktop connections" + "description": "Enter a password for desktop connections. If \"noPassword\", connections from the local host can be established without entering a password" }, "webPort": { "type": "string", @@ -37,7 +38,7 @@ "6080" ], "default": "6080", - "description": "Enter a port for the VNC web client" + "description": "Enter a port for the VNC web client (noVNC)" }, "vncPort": { "type": "string", @@ -45,7 +46,7 @@ "5901" ], "default": "5901", - "description": "Enter a port for the desktop VNC server" + "description": "Enter a port for the desktop VNC server (TigerVNC)" } }, "init": true, diff --git a/src/desktop-lite/install.sh b/src/desktop-lite/install.sh index 13a524ada..ef8b603c6 100755 --- a/src/desktop-lite/install.sh +++ b/src/desktop-lite/install.sh @@ -9,6 +9,9 @@ NOVNC_VERSION="${NOVNCVERSION:-"1.2.0"}" # TODO: Add in a 'latest' auto-detect and swap name to 'version' VNC_PASSWORD=${PASSWORD:-"vscode"} +if [ "$VNC_PASSWORD" = "noPassword" ]; then + unset VNC_PASSWORD +fi NOVNC_PORT="${WEBPORT:-6080}" VNC_PORT="${VNCPORT:-5901}" @@ -372,7 +375,15 @@ sudoIf chown root:\${group_name} /tmp/.X11-unix if [ "\$(echo "\${VNC_RESOLUTION}" | tr -cd 'x' | wc -c)" = "1" ]; then VNC_RESOLUTION=\${VNC_RESOLUTION}x16; fi screen_geometry="\${VNC_RESOLUTION%*x*}" screen_depth="\${VNC_RESOLUTION##*x}" -startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "tigervncserver \${DISPLAY} -geometry \${screen_geometry} -depth \${screen_depth} -rfbport ${VNC_PORT} -dpi \${VNC_DPI:-96} -localhost -desktop fluxbox -fg -passwd /usr/local/etc/vscode-dev-containers/vnc-passwd" + +# Check if VNC_PASSWORD is set and use the appropriate command +common_options="tigervncserver \${DISPLAY} -geometry \${screen_geometry} -depth \${screen_depth} -rfbport ${VNC_PORT} -dpi \${VNC_DPI:-96} -localhost -desktop fluxbox -fg" + +if [ -n "\${VNC_PASSWORD+x}" ]; then + startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "\${common_options} -passwd /usr/local/etc/vscode-dev-containers/vnc-passwd" +else + startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "\${common_options} -SecurityTypes None" +fi # Spin up noVNC if installed and not running. if [ -d "/usr/local/novnc" ] && [ "\$(ps -ef | grep /usr/local/novnc/noVNC*/utils/launch.sh | grep -v grep)" = "" ]; then @@ -388,7 +399,9 @@ exec "\$@" log "** SCRIPT EXIT **" EOF -echo "${VNC_PASSWORD}" | vncpasswd -f > /usr/local/etc/vscode-dev-containers/vnc-passwd +if [ -n "${VNC_PASSWORD+x}" ]; then + echo "${VNC_PASSWORD}" | vncpasswd -f > /usr/local/etc/vscode-dev-containers/vnc-passwd +fi chmod +x /usr/local/share/desktop-init.sh /usr/local/bin/set-resolution # Set up fluxbox config @@ -401,15 +414,23 @@ fi # Clean up rm -rf /var/lib/apt/lists/* +# Determine the message based on whether VNC_PASSWORD is set +if [ -n "${VNC_PASSWORD+x}" ]; then + PASSWORD_MESSAGE="In both cases, use the password \"${VNC_PASSWORD}\" when connecting" +else + PASSWORD_MESSAGE="In both cases, no password is required." +fi + +# Display the message cat << EOF You now have a working desktop! Connect to in one of the following ways: -- Forward port ${NOVNC_PORT} and use a web browser start the noVNC client (recommended) +- Forward port ${NOVNC_PORT} and use a web browser to start the noVNC client (recommended) - Forward port ${VNC_PORT} using VS Code client and connect using a VNC Viewer -In both cases, use the password "${VNC_PASSWORD}" when connecting +${PASSWORD_MESSAGE} (*) Done! From 32797f4f693a3bf18a18928ad28bc55589a7ada6 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Wed, 29 May 2024 12:54:52 -0700 Subject: [PATCH 024/224] Automated documentation update (#989) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/desktop-lite/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/desktop-lite/README.md b/src/desktop-lite/README.md index 7094a2425..6f2d67ce2 100644 --- a/src/desktop-lite/README.md +++ b/src/desktop-lite/README.md @@ -16,10 +16,10 @@ Adds a lightweight Fluxbox based desktop to the container that can be accessed u | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Currently Unused! | string | latest | -| noVncVersion | NoVnc Version | string | 1.2.0 | -| password | Enter a password for desktop connections | string | vscode | -| webPort | Enter a port for the VNC web client | string | 6080 | -| vncPort | Enter a port for the desktop VNC server | string | 5901 | +| noVncVersion | The noVNC version to use | string | 1.2.0 | +| password | Enter a password for desktop connections. If "noPassword", connections from the local host can be established without entering a password | string | vscode | +| webPort | Enter a port for the VNC web client (noVNC) | string | 6080 | +| vncPort | Enter a port for the desktop VNC server (TigerVNC) | string | 5901 | ## Connecting to the desktop From c1df45b189afae33be72c1ce478452fbfc044abe Mon Sep 17 00:00:00 2001 From: Prabhakar Kumar <64955767+prabhakk-mw@users.noreply.github.com> Date: Thu, 30 May 2024 03:00:18 +0530 Subject: [PATCH 025/224] Adds /home/USER/.local/bin/ to PATH in /etc/sudoers.d/vscode, (#887) * Adds /home/USER/.local/bin/ to PATH in /etc/sudoers.d/vscode, fixes devcontainers/features#870 * Bumping up version to 1.4.2 * Adds to sudoers file if already present * Tests to ensure Default secure_path is not overwritten * Bump to version 1.4.4 * Fix version as 1.4.3 Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Prabhakar Kumar <64955767+prabhakk-mw@users.noreply.github.com> * Update src/python/devcontainer-feature.json --------- Co-authored-by: Samruddhi Khandale Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 57 ++++++++++++------- test/python/install_jupyterlab.sh | 3 + ...nstall_jupyterlab_existing_sudoers_file.sh | 36 ++++++++++++ .../Dockerfile | 5 ++ .../sudoers.test | 2 + test/python/scenarios.json | 13 +++++ 7 files changed, 97 insertions(+), 21 deletions(-) create mode 100755 test/python/install_jupyterlab_existing_sudoers_file.sh create mode 100644 test/python/install_jupyterlab_existing_sudoers_file/Dockerfile create mode 100644 test/python/install_jupyterlab_existing_sudoers_file/sudoers.test diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index ef16ee643..7c2c6200a 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.5.0", + "version": "1.6.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index e8a9d24e1..1aad3f631 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -130,7 +130,7 @@ updaterc() { fi } -# Import the specified key in a variable name passed in as +# Import the specified key in a variable name passed in as receive_gpg_keys() { local keys=${!1} local keyring_args="" @@ -152,7 +152,7 @@ receive_gpg_keys() { local retry_count=0 local gpg_ok="false" set +e - until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do echo "(*) Downloading GPG key..." ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" @@ -222,7 +222,7 @@ find_version_from_git_tags() { local repository=$2 local prefix=${3:-"tags/v"} local separator=${4:-"."} - local last_part_optional=${5:-"false"} + local last_part_optional=${5:-"false"} if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator=${separator//./\\.} local last_part @@ -282,7 +282,7 @@ find_prev_version_from_git_tags() { ((breakfix=breakfix-1)) if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then declare -g ${variable_name}="${major}.${minor}" - else + else declare -g ${variable_name}="${major}.${minor}.${breakfix}" fi fi @@ -378,13 +378,13 @@ check_packages() { add_symlink() { if [[ ! -d "${CURRENT_PATH}" ]]; then - ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" fi if [ "${OVERRIDE_DEFAULT_VERSION}" = "true" ]; then if [[ $(ls -l ${CURRENT_PATH}) != *"-> ${INSTALL_PATH}"* ]] ; then rm "${CURRENT_PATH}" - ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" fi fi } @@ -397,7 +397,7 @@ install_openssl3() { openssl3_version="3.0" # Find version using soft match find_version_from_git_tags openssl3_version "https://github.com/openssl/openssl" "openssl-" - local tgz_filename="openssl-${openssl3_version}.tar.gz" + local tgz_filename="openssl-${openssl3_version}.tar.gz" local tgz_url="https://github.com/openssl/openssl/releases/download/openssl-${openssl3_version}/${tgz_filename}" echo "Downloading ${tgz_filename}..." curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" @@ -434,7 +434,7 @@ install_cpython() { } install_from_source() { - VERSION=$1 + VERSION=$1 echo "(*) Building Python ${VERSION} from source..." if ! type git > /dev/null 2>&1; then check_packages git @@ -444,7 +444,7 @@ install_from_source() { find_version_from_git_tags VERSION "https://github.com/python/cpython" # Some platforms/os versions need modern versions of openssl installed - # via common package repositories, for now rhel-7 family, use case statement to + # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand case ${VERSION_CODENAME} in centos7|rhel7) @@ -455,7 +455,7 @@ install_from_source() { esac install_cpython "${VERSION}" - if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then + if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then if grep -q "404 Not Found" "/tmp/python-src/${cpython_tgz_filename}"; then install_prev_vers_cpython "${VERSION}" fi @@ -512,9 +512,9 @@ install_from_source() { } install_using_oryx() { - VERSION=$1 + VERSION=$1 INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" - + if [ -d "${INSTALL_PATH}" ]; then echo "(!) Python version ${VERSION} already exists." exit 1 @@ -727,7 +727,7 @@ if [ "${PYTHON_VERSION}" != "none" ]; then usermod -a -G python "${USERNAME}" CURRENT_PATH="${PYTHON_INSTALL_PATH}/current" - + install_python ${PYTHON_VERSION} # Additional python versions to be installed but not be set as default. @@ -748,7 +748,7 @@ if [ "${PYTHON_VERSION}" != "none" ]; then updaterc "if [[ \"\${PATH}\" != *\"${CURRENT_PATH}/bin\"* ]]; then export PATH=${CURRENT_PATH}/bin:\${PATH}; fi" PATH="${INSTALL_PATH}/bin:${PATH}" fi - + # Updates the symlinks for os-provided, or the installed python version in other cases chown -R "${USERNAME}:python" "${PYTHON_INSTALL_PATH}" chmod -R g+r+w "${PYTHON_INSTALL_PATH}" @@ -776,7 +776,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then umask 0002 mkdir -p ${PIPX_BIN_DIR} chown -R "${USERNAME}:pipx" ${PIPX_HOME} - chmod -R g+r+w "${PIPX_HOME}" + chmod -R g+r+w "${PIPX_HOME}" find "${PIPX_HOME}" -type d -print0 | xargs -0 -n 1 chmod g+s # Update pip if not using os provided python @@ -805,21 +805,21 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then echo "${util} already installed. Skipping." fi done - + # Temporary: Removes โ€œsetup toolsโ€ metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 - if [[ $SKIP_VULNERABILITY_PATCHING = "false" ]]; then + if [[ $SKIP_VULNERABILITY_PATCHING = "false" ]]; then VULNERABLE_VERSIONS=("3.10" "3.11") RUN_TIME_PY_VER_DETECT=$(${PYTHON_SRC} --version 2>&1) PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info - if [[ -e "/usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" ]]; then + if [[ -e "/usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" ]]; then # remove the vulnerable setuptools-65.5.0-py3-none-any.whl file rm /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl # create and change to the setuptools_downloaded directory mkdir -p /tmp/setuptools_downloaded cd /tmp/setuptools_downloaded - # download the source distribution for setuptools using pip + # download the source distribution for setuptools using pip pip download setuptools==65.5.1 --no-binary :all: # extract the filename of the setuptools-*.tar.gz file filename=$(find . -maxdepth 1 -type f) @@ -833,7 +833,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then python setup.py bdist_wheel # move inside the dist directory in pwd cd dist - # copy this file to the ensurepip/_bundled directory + # copy this file to the ensurepip/_bundled directory cp setuptools-65.5.1-py3-none-any.whl /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/ # replace the version in __init__.py file with the installed version sed -i 's/_SETUPTOOLS_VERSION = \"65\.5\.0\"/_SETUPTOOLS_VERSION = "65.5.1"/g' /usr/local/lib/"python${PY_MAJOR_MINOR_VER}"/ensurepip/__init__.py @@ -865,6 +865,23 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then install_user_package $INSTALL_UNDER_ROOT jupyterlab install_user_package $INSTALL_UNDER_ROOT jupyterlab-git + if [ "$INSTALL_UNDER_ROOT" = false ]; then + # JupyterLab would have installed into /home/${USERNAME}/.local/bin + # Adding it to default path for Codespaces which use non-login shells + SUDOERS_FILE="/etc/sudoers.d/$USERNAME" + SEARCH_STR="Defaults secure_path=" + REPLACE_STR="Defaults secure_path=/home/${USERNAME}/.local/bin" + + if grep -qs ${SEARCH_STR} ${SUDOERS_FILE}; then + # string found and file is present + sed -i "s|${SEARCH_STR}|${REPLACE_STR}:|g" "${SUDOERS_FILE}" + else + # either string is not found, or file is not present + # In either case take same action, note >> places at end of file + echo "${REPLACE_STR}:${PATH}" >> ${SUDOERS_FILE} + fi + fi + # Configure JupyterLab if needed if [ -n "${CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN}" ]; then # Resolve config directory diff --git a/test/python/install_jupyterlab.sh b/test/python/install_jupyterlab.sh index 58c4f7f54..033b44edd 100755 --- a/test/python/install_jupyterlab.sh +++ b/test/python/install_jupyterlab.sh @@ -22,5 +22,8 @@ check "jupyterlab_git" grep jupyterlab_git <<< "$packages" # Check for correct JupyterLab configuration check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py +# Check for PATH modification +check "default path has jupyterlab" sudo grep "/home/${user}/.local/bin" /etc/sudoers.d/$user + # Report result reportResults diff --git a/test/python/install_jupyterlab_existing_sudoers_file.sh b/test/python/install_jupyterlab_existing_sudoers_file.sh new file mode 100755 index 000000000..22bbc2f92 --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Check for correct JupyterLab configuration +check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py + +# Check for PATH modification +check "default path has jupyterlab" grep "Defaults secure_path=/home/${user}/.local/bin" /etc/sudoers.d/$user + +# Check if previous PATH exists +check "existing default path is preserved" grep "Defaults secure_path=.*original_content_of_sudoers_file" /etc/sudoers.d/$user + +# Check if PATH modification includes original and new paths +check "existing path included with jupyterlab" grep "Defaults secure_path.*/home/${user}/.local/bin.*original_content_of_sudoers_file" /etc/sudoers.d/$user + + +# Report result +reportResults diff --git a/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile b/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile new file mode 100644 index 000000000..5acb58a37 --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile @@ -0,0 +1,5 @@ +# Builds an image with a preconfigured SUDOERS file +# Used to test the install script for JupyterLab which modifies this file +FROM mcr.microsoft.com/devcontainers/base:focal + +COPY --chown=root sudoers.test /etc/sudoers.d/vscode diff --git a/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test b/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test new file mode 100644 index 000000000..9a26d777b --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test @@ -0,0 +1,2 @@ +# Sudoers File for testing, after install script runs the Defaults secure_path should be appended to +Defaults secure_path=/original_content_of_sudoers_file \ No newline at end of file diff --git a/test/python/scenarios.json b/test/python/scenarios.json index be37869df..e4fe0f0cd 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -68,6 +68,19 @@ } } }, + "install_jupyterlab_existing_sudoers_file": { + "build": { + "dockerfile": "Dockerfile" + }, + "remoteUser": "vscode", + "features": { + "python": { + "version": "latest", + "installJupyterlab": true, + "configureJupyterlabAllowOrigin": "*" + } + } + }, "install_jupyterlab_rhel_family": { "image": "almalinux:8", "remoteUser": "vscode", From b32aa5f0f207e9dc9c5ad36524ed45c6aaec20dd Mon Sep 17 00:00:00 2001 From: Rambaud Pierrick <12rambau@users.noreply.github.com> Date: Thu, 30 May 2024 19:56:16 +0200 Subject: [PATCH 026/224] fix centos-7 build (#985) * fix centos-7 build * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * bump(python): 1.5.0 -> 1.5.1 --------- Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 7c2c6200a..f57568cc6 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.6.0", + "version": "1.6.1", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index 1aad3f631..94007a9b0 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -390,7 +390,6 @@ add_symlink() { } install_openssl3() { - local _prefix=$1 mkdir /tmp/openssl3 ( cd /tmp/openssl3 @@ -403,7 +402,7 @@ install_openssl3() { curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" tar xzf ${tgz_filename} cd openssl-${openssl3_version} - ./config --prefix=${_prefix} --openssldir=${_prefix} --libdir=lib + ./config --libdir=lib make -j $(nproc) make install_dev ) @@ -446,11 +445,12 @@ install_from_source() { # Some platforms/os versions need modern versions of openssl installed # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand + SSL_INSTALL_PATH="/usr/local" case ${VERSION_CODENAME} in centos7|rhel7) check_packages perl-IPC-Cmd - install_openssl3 ${INSTALL_PATH} - ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH} --with-openssl-rpath=${INSTALL_PATH}/lib" + install_openssl3 + ADDL_CONFIG_ARGS="--with-openssl=${SSL_INSTALL_PATH} --with-openssl-rpath=${SSL_INSTALL_PATH}/lib" ;; esac From dbb135408311512248bfb2b161d52d12936bbaa6 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 31 May 2024 05:36:41 +0530 Subject: [PATCH 027/224] [azure-cli] - add support for noble numbat (#986) * [azure-cli] - add support for noble numbat * changes requested --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index 6b26ef6fc..e73d2295c 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.4", + "version": "1.2.5", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index a1b254779..2b52ca158 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -18,7 +18,7 @@ AZ_INSTALLBICEP=${INSTALLBICEP:-false} INSTALL_USING_PYTHON=${INSTALLUSINGPYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" -AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" +AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy noble" if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' From 1e44a6741d33d65bfaf340b5af6107f0a95441ce Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Mon, 3 Jun 2024 22:15:24 +0530 Subject: [PATCH 028/224] [Java] - Document additionalVersions functionality (#987) * [Java] - Document additionalVersions functionality * changes requested * changes as requested in pr review * bump patch version * changes requested --- src/java/devcontainer-feature.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/java/devcontainer-feature.json b/src/java/devcontainer-feature.json index 9ed72ad59..c43862610 100644 --- a/src/java/devcontainer-feature.json +++ b/src/java/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "java", - "version": "1.4.1", + "version": "1.5.0", "name": "Java (via SDKMAN!)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/java", "description": "Installs Java, SDKMAN! (if not installed), and needed dependencies.", @@ -17,6 +17,11 @@ "default": "latest", "description": "Select or enter a Java version to install" }, + "additionalVersions": { + "type": "string", + "default": "", + "description": "Enter additional Java versions, separated by commas." + }, "jdkDistro": { "type": "string", "proposals": [ From 6a9dd0777c5ac0c9a998728491d0e2e0bca11081 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 3 Jun 2024 09:52:05 -0700 Subject: [PATCH 029/224] Automated documentation update (#992) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/java/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/java/README.md b/src/java/README.md index 938cd0f89..1a2d91851 100644 --- a/src/java/README.md +++ b/src/java/README.md @@ -16,6 +16,7 @@ Installs Java, SDKMAN! (if not installed), and needed dependencies. | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select or enter a Java version to install | string | latest | +| additionalVersions | Enter additional Java versions, separated by commas. | string | - | | jdkDistro | Select or enter a JDK distribution | string | ms | | installGradle | Install Gradle, a build automation tool for multi-language software development | boolean | false | | gradleVersion | Select or enter a Gradle version | string | latest | From 865f69c6a2683603090be0d8c531da1cbf549c9b Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Wed, 12 Jun 2024 01:06:35 +0530 Subject: [PATCH 030/224] #963 specific powershell module version install (#993) * #963 specific powershell module version install * review comments addressed * added test for the version specific module installation and addressed review comments * Update src/powershell/install.sh * Update src/powershell/install.sh * Update src/powershell/install.sh --------- Co-authored-by: Samruddhi Khandale --- src/powershell/devcontainer-feature.json | 4 ++-- src/powershell/install.sh | 16 +++++++++++++--- test/powershell/install_modules_version.sh | 13 +++++++++++++ test/powershell/scenarios.json | 8 ++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 test/powershell/install_modules_version.sh diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 82ef39a30..d51b5c03f 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.5", + "version": "1.4.0", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", @@ -18,7 +18,7 @@ "modules": { "type": "string", "default": "", - "description": "Optional comma separated list of PowerShell modules to install." + "description": "Optional comma separated list of PowerShell modules to install. If you need to install a specific version of a module, use '==' to specify the version (e.g. 'az.resources==2.5.0')" }, "powershellProfileURL": { "type": "string", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 533da6f37..43773b3f6 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -242,14 +242,24 @@ if [ "${use_github}" = "true" ]; then install_using_github fi -# If PowerShell modules are requested, loop through and install +# If PowerShell modules are requested, loop through and install if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then echo "Installing PowerShell Modules: ${POWERSHELL_MODULES}" modules=(`echo ${POWERSHELL_MODULES} | tr ',' ' '`) for i in "${modules[@]}" do - echo "Installing ${i}" - pwsh -Command "Install-Module -Name ${i} -AllowClobber -Force -Scope AllUsers" || continue + module_parts=(`echo ${i} | tr '==' ' '`) + module_name="${module_parts[0]}" + args="-Name ${module_name} -AllowClobber -Force -Scope AllUsers" + if [ "${#module_parts[@]}" -eq 2 ]; then + module_version="${module_parts[1]}" + echo "Installing ${module_name} v${module_version}" + args+=" -RequiredVersion ${module_version}" + else + echo "Installing latest version for ${i} module" + fi + + pwsh -Command "Install-Module $args" || continue done fi diff --git a/test/powershell/install_modules_version.sh b/test/powershell/install_modules_version.sh new file mode 100644 index 000000000..ea1a3a98c --- /dev/null +++ b/test/powershell/install_modules_version.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Extension-specific tests +check "az.resources" pwsh -Command "(Get-Module -ListAvailable -Name Az.Resources).Version.ToString()" | grep 2.5.0 +check "az.storage" pwsh -Command "(Get-Module -ListAvailable -Name Az.Storage).Version.ToString()" | grep 4.3.0 + +# Report result +reportResults diff --git a/test/powershell/scenarios.json b/test/powershell/scenarios.json index 6a810b0c9..757a8cc97 100644 --- a/test/powershell/scenarios.json +++ b/test/powershell/scenarios.json @@ -16,5 +16,13 @@ "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" } } + }, + "install_modules_version": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "powershell": { + "modules": "az.resources==2.5.0, az.storage==4.3.0" + } + } } } From 22ee16e26000d47f6f2ea03a09d68a7487e4603d Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:09:20 -0700 Subject: [PATCH 031/224] Automated documentation update (#998) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/powershell/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powershell/README.md b/src/powershell/README.md index f018778c0..31199a4e1 100644 --- a/src/powershell/README.md +++ b/src/powershell/README.md @@ -16,7 +16,7 @@ Installs PowerShell along with needed dependencies. Useful for base Dockerfiles | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select or enter a version of PowerShell. | string | latest | -| modules | Optional comma separated list of PowerShell modules to install. | string | - | +| modules | Optional comma separated list of PowerShell modules to install. If you need to install a specific version of a module, use '==' to specify the version (e.g. 'az.resources==2.5.0') | string | - | | powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | ## Customizations From 15320f018d0cd72490ba073edb8900968b864ade Mon Sep 17 00:00:00 2001 From: Steven Date: Fri, 14 Jun 2024 02:02:14 +0200 Subject: [PATCH 032/224] dotnet: add ability to install workloads (#997) * dotnet: add ability to install workloads * Bump dotnet feature version * Update NOTES instead of README * Simplify workloads example * Fix temp-dir path oopsie * Improve log message * Fix typo imstalling->installing * Install all workloads at once, fix DOTNET vars not taking effect --- src/dotnet/NOTES.md | 18 +++++++++++------- src/dotnet/devcontainer-feature.json | 7 ++++++- src/dotnet/install.sh | 17 +++++++++++++++++ src/dotnet/scripts/dotnet-helpers.sh | 19 +++++++++++++++---- test/dotnet/dotnet_helpers.sh | 14 +++++++++++--- test/dotnet/install_dotnet_workloads.sh | 24 ++++++++++++++++++++++++ test/dotnet/scenarios.json | 12 +++++++++++- 7 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 test/dotnet/install_dotnet_workloads.sh diff --git a/src/dotnet/NOTES.md b/src/dotnet/NOTES.md index 578aceaf3..584c90cef 100644 --- a/src/dotnet/NOTES.md +++ b/src/dotnet/NOTES.md @@ -2,8 +2,7 @@ Installing only the latest .NET SDK version (the default). -``` json -{ +``` jsonc "features": { "ghcr.io/devcontainers/features/dotnet:2": "latest" // or "" or {} } @@ -12,7 +11,6 @@ Installing only the latest .NET SDK version (the default). Installing an additional SDK version. Multiple versions can be specified as comma-separated values. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "additionalVersions": "lts" @@ -23,7 +21,6 @@ Installing an additional SDK version. Multiple versions can be specified as comm Installing specific SDK versions. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0", @@ -35,7 +32,6 @@ Installing specific SDK versions. Installing a specific SDK feature band. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0.4xx", @@ -46,7 +42,6 @@ Installing a specific SDK feature band. Installing a specific SDK patch version. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0.412", @@ -57,7 +52,6 @@ Installing a specific SDK patch version. Installing only the .NET Runtime or the ASP.NET Core Runtime. (The SDK includes all runtimes so this configuration is only useful if you need to run .NET apps without building them from source.) ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "none", @@ -67,6 +61,16 @@ Installing only the .NET Runtime or the ASP.NET Core Runtime. (The SDK includes } ``` +Installing .NET workloads. Multiple workloads can be specified as comma-separated values. + +``` json +"features": { + "ghcr.io/devcontainers/features/dotnet:2": { + "workloads": "aspire, wasm-tools" + } +} +``` + ## OS Support This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 8b8ffaf2a..fa80799a5 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.6", + "version": "2.1.0", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", @@ -32,6 +32,11 @@ "type": "string", "default": "", "description": "Enter additional ASP.NET Core runtime versions, separated by commas. Use 'latest' for the latest version, 'lts' for the latest LTS version, 'X.Y' or 'X.Y.Z' for a specific version." + }, + "workloads": { + "type": "string", + "default": "", + "description": "Enter additional .NET SDK workloads, separated by commas. Use 'dotnet workload search' to learn what workloads are available to install." } }, "containerEnv": { diff --git a/src/dotnet/install.sh b/src/dotnet/install.sh index 237a8a0be..d289ea2a4 100644 --- a/src/dotnet/install.sh +++ b/src/dotnet/install.sh @@ -10,6 +10,14 @@ DOTNET_VERSION="${VERSION:-"latest"}" ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" DOTNET_RUNTIME_VERSIONS="${DOTNETRUNTIMEVERSIONS:-""}" ASPNETCORE_RUNTIME_VERSIONS="${ASPNETCORERUNTIMEVERSIONS:-""}" +WORKLOADS="${WORKLOADS:-""}" + +# Prevent "Welcome to .NET" message from dotnet +export DOTNET_NOLOGO=true + +# Prevent generating a development certificate while running this script +# Otherwise it would be stored in the image, which is undesirable +export DOTNET_GENERATE_ASPNET_CERTIFICATE=false set -e @@ -111,6 +119,15 @@ for version in "${aspNetCoreRuntimeVersions[@]}"; do install_runtime "aspnetcore" "$version" done +workloads=() +for workload in $(split_csv "$WORKLOADS"); do + workloads+=("$workload") +done + +if [ ${#workloads[@]} -ne 0 ]; then + install_workloads "${workloads[@]}" +fi + # Clean up rm -rf /var/lib/apt/lists/* rm -rf scripts diff --git a/src/dotnet/scripts/dotnet-helpers.sh b/src/dotnet/scripts/dotnet-helpers.sh index bda0c9c3a..b7024c9d1 100644 --- a/src/dotnet/scripts/dotnet-helpers.sh +++ b/src/dotnet/scripts/dotnet-helpers.sh @@ -25,7 +25,6 @@ fetch_latest_version_in_channel() { else wget -qO- "https://dotnetcli.azureedge.net/dotnet/Sdk/$channel/latest.version" fi - } # Prints the latest dotnet version @@ -76,12 +75,11 @@ install_sdk() { fi # Currently this script does not make it possible to qualify the version, 'GA' is always implied - echo "Executing $DOTNET_INSTALL_SCRIPT --version $version --channel $channel --install-dir $DOTNET_INSTALL_DIR --no-path" + echo "Executing $DOTNET_INSTALL_SCRIPT --version $version --channel $channel --install-dir $DOTNET_INSTALL_DIR" "$DOTNET_INSTALL_SCRIPT" \ --version "$version" \ --channel "$channel" \ - --install-dir "$DOTNET_INSTALL_DIR" \ - --no-path + --install-dir "$DOTNET_INSTALL_DIR" } # Installs a version of the .NET Runtime @@ -117,3 +115,16 @@ install_runtime() { --install-dir "$DOTNET_INSTALL_DIR" \ --no-path } + +# Installs one or more .NET workloads +# Usage: install_workload [ ...] +# Reference: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-workload-install +install_workloads() { + local workloads="$@" + + echo "Installing .NET workload(s) $workloads" + dotnet workload install $workloads --temp-dir /tmp/dotnet-workload-temp-dir + + # Clean up + rm -r /tmp/dotnet-workload-temp-dir +} diff --git a/test/dotnet/dotnet_helpers.sh b/test/dotnet/dotnet_helpers.sh index 6c833b444..a24bd1ce2 100644 --- a/test/dotnet/dotnet_helpers.sh +++ b/test/dotnet/dotnet_helpers.sh @@ -15,7 +15,6 @@ fetch_latest_version_in_channel() { else wget -qO- "https://dotnetcli.azureedge.net/dotnet/Sdk/$channel/latest.version" fi - } # Prints the latest dotnet version @@ -47,7 +46,6 @@ is_dotnet_sdk_version_installed() { return $? } - # Asserts that the specified .NET Runtime version is installed # Returns a non-zero exit code if the check fails # Usage: is_dotnet_runtime_version_installed @@ -68,4 +66,14 @@ is_aspnetcore_runtime_version_installed() { local expected="$1" dotnet --list-runtimes | grep --fixed-strings --silent "Microsoft.AspNetCore.App $expected" return $? -} \ No newline at end of file +} + +# Asserts that the specified workload is installed +# Returns a non-zero exit code if the check fails +# Usage: is_dotnet_workload_installed +# Example: is_dotnet_workload_installed "aspire" +is_dotnet_workload_installed() { + local expected="$1" + dotnet workload list | grep --fixed-strings --silent "$expected" + return $? +} diff --git a/test/dotnet/install_dotnet_workloads.sh b/test/dotnet/install_dotnet_workloads.sh new file mode 100644 index 000000000..37c86a2d4 --- /dev/null +++ b/test/dotnet/install_dotnet_workloads.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Optional: Import test library bundled with the devcontainer CLI +# See https://github.com/devcontainers/cli/blob/HEAD/docs/features/test.md#dev-container-features-test-lib +# Provides the 'check' and 'reportResults' commands. +source dev-container-features-test-lib + +# Feature-specific tests +# The 'check' command comes from the dev-container-features-test-lib. Syntax is... +# check