diff --git a/.evergreen/config_generator/components/misc.py b/.evergreen/config_generator/components/misc.py deleted file mode 100644 index f609ce233fe..00000000000 --- a/.evergreen/config_generator/components/misc.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Iterable - -from shrub.v3.evg_build_variant import BuildVariant -from shrub.v3.evg_task import EvgTask, EvgTaskRef - -from . import earthly - - -def tasks() -> Iterable[EvgTask]: - yield EvgTask( - name="create-silk-asset-group", - commands=[ - earthly.earthly_exec( - kind="setup", - target="create-silk-asset-group", - args={ - "branch": r"${branch_name}", - }, - secrets={ - "SILK_CLIENT_ID": r"${silk_client_id}", - "SILK_CLIENT_SECRET": r"${silk_client_secret}", - }, - ) - ], - run_on=earthly.CONTAINER_RUN_DISTROS, - tags=["misc", "pr-merge-gate"], - ) - - -def variants() -> Iterable[BuildVariant]: - yield BuildVariant( - name="misc", - tasks=[EvgTaskRef(name=".misc")], - display_name="Miscellaneous", - ) diff --git a/.evergreen/config_generator/components/sbom.py b/.evergreen/config_generator/components/sbom.py new file mode 100644 index 00000000000..dca242148df --- /dev/null +++ b/.evergreen/config_generator/components/sbom.py @@ -0,0 +1,119 @@ +from config_generator.etc.distros import find_small_distro +from config_generator.etc.function import Function, merge_defns +from config_generator.etc.utils import bash_exec + +from shrub.v3.evg_build_variant import BuildVariant +from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, expansions_update, s3_put +from shrub.v3.evg_task import EvgTask, EvgTaskRef + +from pydantic import ConfigDict +from typing import Optional + + +TAG = 'sbom' + + +class CustomCommand(BuiltInCommand): + command: str + model_config = ConfigDict(arbitrary_types_allowed=True) + + +def ec2_assume_role( + role_arn: Optional[str] = None, + policy: Optional[str] = None, + duration_seconds: Optional[int] = None, + command_type: Optional[EvgCommandType] = None, +) -> CustomCommand: + return CustomCommand( + command="ec2.assume_role", + params={ + "role_arn": role_arn, + "policy": policy, + "duration_seconds": duration_seconds, + }, + type=command_type, + ) + + +class SBOM(Function): + name = 'sbom' + commands = [ + ec2_assume_role( + command_type=EvgCommandType.SETUP, + role_arn='${kondukto_role_arn}', + ), + bash_exec( + command_type=EvgCommandType.SETUP, + include_expansions_in_env=[ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SESSION_TOKEN', + ], + script='''\ + set -o errexit + set -o pipefail + kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" + printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml + ''', + ), + expansions_update( + command_type=EvgCommandType.SETUP, + file='expansions.kondukto.yml', + ), + bash_exec( + command_type=EvgCommandType.TEST, + working_dir='mongoc', + include_expansions_in_env=[ + 'artifactory_password', + 'artifactory_username', + 'branch_name', + 'KONDUKTO_TOKEN', + ], + script='.evergreen/scripts/sbom.sh', + ), + s3_put( + command_type=EvgCommandType.TEST, + aws_key='${aws_key}', + aws_secret='${aws_secret}', + bucket='mciuploads', + content_type='application/json', + display_name='Augmented SBOM', + local_file='mongoc/augmented-sbom.json', + permissions='public-read', + remote_file='${project}/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented-sbom.json', + ), + ] + + @classmethod + def call(cls, **kwargs): + return cls.default_call(**kwargs) + + +def functions(): + return merge_defns( + SBOM.defn(), + ) + + +def tasks(): + distro_name = 'rhel80' + distro = find_small_distro(distro_name) + + yield EvgTask( + name='sbom', + tags=[TAG, distro_name], + run_on=distro.name, + commands=[ + SBOM.call(), + ], + ) + + +def variants(): + return [ + BuildVariant( + name=TAG, + display_name='SBOM', + tasks=[EvgTaskRef(name=f'.{TAG}')], + ), + ] diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 66c11a1d42a..cf232d7717a 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -499,6 +499,54 @@ functions: args: - -c - .evergreen/scripts/compile.sh + sbom: + - command: ec2.assume_role + type: setup + params: + role_arn: ${kondukto_role_arn} + - command: subprocess.exec + type: setup + params: + binary: bash + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + args: + - -c + - | + set -o errexit + set -o pipefail + kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" + printf "KONDUKTO_TOKEN: %s\n" "$kondukto_token" >|expansions.kondukto.yml + - command: expansions.update + type: setup + params: + file: expansions.kondukto.yml + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: mongoc + include_expansions_in_env: + - artifactory_password + - artifactory_username + - branch_name + - KONDUKTO_TOKEN + args: + - -c + - .evergreen/scripts/sbom.sh + - command: s3.put + type: test + params: + display_name: Augmented SBOM + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: application/json + local_file: mongoc/augmented-sbom.json + permissions: public-read + remote_file: ${project}/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented-sbom.json scan-build: - command: subprocess.exec type: test diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index c75cbc5c250..9c680817c97 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1410,27 +1410,6 @@ tasks: tags: [clang-format] commands: - func: clang-format - - name: create-silk-asset-group - run_on: - - ubuntu2204-large - - debian10-large - - debian11-large - - amazon2 - tags: [misc, pr-merge-gate] - commands: - - command: subprocess.exec - type: setup - params: - binary: ./tools/earthly.sh - working_dir: mongoc - env: - SILK_CLIENT_ID: ${silk_client_id} - SILK_CLIENT_SECRET: ${silk_client_secret} - args: - - --secret=SILK_CLIENT_ID - - --secret=SILK_CLIENT_SECRET - - +create-silk-asset-group - - --branch=${branch_name} - name: cse-sasl-cyrus-darwinssl-macos-11-arm64-clang-compile run_on: macos-11-arm64 tags: [cse-matrix-darwinssl, compile, macos-11-arm64, clang, cse, sasl-cyrus] @@ -4934,6 +4913,11 @@ tasks: - func: bootstrap-mongo-orchestration - func: run-simple-http-server - func: run-tests + - name: sbom + run_on: rhel80-small + tags: [sbom, rhel80] + commands: + - func: sbom - name: scan-build-macos-14-arm64-clang run_on: macos-14-arm64 tags: [scan-build-matrix, macos-14-arm64, clang] diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 7d354eee1de..91057d170a5 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -174,10 +174,6 @@ buildvariants: display_name: loadbalanced tasks: - name: .loadbalanced - - name: misc - display_name: Miscellaneous - tasks: - - name: .misc - name: mock-server-test display_name: Mock Server Test expansions: @@ -282,6 +278,10 @@ buildvariants: expansions: {} tasks: - name: .sasl-matrix-winssl + - name: sbom + display_name: SBOM + tasks: + - name: .sbom - name: scan-build-matrix display_name: scan-build-matrix tasks: diff --git a/.evergreen/scripts/sbom.sh b/.evergreen/scripts/sbom.sh new file mode 100755 index 00000000000..b683b737874 --- /dev/null +++ b/.evergreen/scripts/sbom.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +: "${artifactory_username:?}" +: "${artifactory_password:?}" +: "${branch_name:?}" +: "${KONDUKTO_TOKEN:?}" + +command -v podman >/dev/null || { + echo "missing required program podman" 1>&2 + exit 1 +} + +podman login --password-stdin --username "${artifactory_username:?}" artifactory.corp.mongodb.com <<<"${artifactory_password:?}" + +silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0" + +# Ensure latest version of SilkBomb is being used. +podman pull "${silkbomb:?}" + +silkbomb_augment_flags=( + --repo mongodb/mongo-c-driver + --branch "${branch_name:?}" + --sbom-in /pwd/etc/cyclonedx.sbom.json + --sbom-out /pwd/augmented-sbom.json + + # Any notable updates to the Augmented SBOM version should be done manually after careful inspection. + # Otherwise, it should be equal to the SBOM Lite version, which should normally be `1`. + --no-update-sbom-version +) + +# First validate the SBOM Lite. +podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \ + validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira + +# Then download the Augmented SBOM. Allow the timestamp to be updated. +podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" \ + augment "${silkbomb_augment_flags[@]:?}" + +[[ -f ./augmented-sbom.json ]] || { + echo "failed to download Augmented SBOM" 1>&2 + exit 1 +} diff --git a/Earthfile b/Earthfile index 9a889bba535..fc5c6161a77 100644 --- a/Earthfile +++ b/Earthfile @@ -176,7 +176,6 @@ multibuild: release-archive: FROM artifactory.corp.mongodb.com/dockerhub/library/alpine:3.20 RUN apk add git bash - ARG --required sbom_branch ARG --required prefix ARG --required ref @@ -198,7 +197,9 @@ release-archive: LET base = "mongo_c_driver_latest_release" END - COPY (+sbom-download/augmented-sbom.json --branch=$sbom_branch) cyclonedx.sbom.json + # The augmented SBOM must be manually obtained from a recent execution of + # the `sbom` task in an Evergreen patch or commit build in advance. + COPY etc/augmented-sbom.json cyclonedx.sbom.json # The full link to the build for this commit LET waterfall_url = "https://spruce.mongodb.com/version/${base}_${revision}" @@ -250,8 +251,6 @@ sign-file: signed-release: FROM artifactory.corp.mongodb.com/dockerhub/library/alpine:3.20 RUN apk add git - # We need to know which branch to get the SBOM from - ARG --required sbom_branch # The version of the release. This affects the filepaths of the output and is the default for --ref ARG --required version # The Git revision of the repository to be archived. By default, archives the tag of the given version @@ -269,7 +268,7 @@ signed-release: LET rel_tgz = "$rel_dir/$stem.tar.gz" LET rel_asc = "$rel_dir/$stem.tar.gz.asc" # Make the release archive: - COPY (+release-archive/ --branch=$sbom_branch --prefix=$stem --ref=$ref) $rel_dir/ + COPY (+release-archive/ --prefix=$stem --ref=$ref) $rel_dir/ RUN mv $rel_dir/release.tar.gz $rel_tgz # Sign the release archive: COPY (+sign-file/signature.asc --file $rel_tgz) $rel_asc @@ -280,15 +279,15 @@ signed-release: # This target is simply an environment in which the SilkBomb executable is available. silkbomb: - FROM artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 - # Alias the silkbom executable to a simpler name: + FROM artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0 + # Alias the silkbomb executable to a simpler name: RUN ln -s /python/src/sbom/silkbomb/bin /usr/local/bin/silkbomb # sbom-generate : # Generate/update the etc/cyclonedx.sbom.json file from the etc/purls.txt file. # # This target will update the existing etc/cyclonedx.sbom.json file in-place based -# on the content of etc/purls.txt. +# on the content of etc/purls.txt and etc/cyclonedx.sbom.json. sbom-generate: FROM +silkbomb # Copy in the relevant files: @@ -296,63 +295,47 @@ sbom-generate: COPY etc/purls.txt etc/cyclonedx.sbom.json /s/ # Update the SBOM file: RUN silkbomb update \ + --refresh \ + --no-update-sbom-version \ --purls purls.txt \ --sbom-in cyclonedx.sbom.json \ --sbom-out cyclonedx.sbom.json # Save the result back to the host: SAVE ARTIFACT /s/cyclonedx.sbom.json AS LOCAL etc/cyclonedx.sbom.json -# sbom-download : -# Download an augmented SBOM from the Silk server for the given branch. Exports -# the artifact as /augmented-sbom.json +# sbom-generate-new-serial-number: +# Equivalent to +sbom-generate but includes the --generate-new-serial-number +# flag to generate a new unique serial number and reset the SBOM version to 1. # -# Requires credentials for silk access. -sbom-download: - FROM artifactory.corp.mongodb.com/dockerhub/library/alpine:3.20 - ARG --required branch - # Run the SilkBomb tool to download the artifact that matches the requested branch +# This target will update the existing etc/cyclonedx.sbom.json file in-place based +# on the content of etc/purls.txt and etc/cyclonedx.sbom.json. +sbom-generate-new-serial-number: FROM +silkbomb - # Set --no-cache, because the remote artifact could change arbitrarily over time - RUN --no-cache \ - --secret SILK_CLIENT_ID \ - --secret SILK_CLIENT_SECRET \ - silkbomb download \ - --sbom-out augmented-sbom.json \ - --silk-asset-group mongo-c-driver-${branch} - # Export as /augmented-sbom.json - SAVE ARTIFACT augmented-sbom.json - -# create-silk-asset-group : -# Create an asset group in Silk for the Git branch if one is not already defined. -# -# Requires credentials for Silk access. -# -# If --branch is not specified, it will be inferred from the current Git branch -create-silk-asset-group: - ARG branch - # Get a default value for $branch - FROM artifactory.corp.mongodb.com/dockerhub/library/alpine:3.19 - IF test "${branch}" = "" - LOCALLY - LET branch=$(git rev-parse --abbrev-ref HEAD) - RUN --no-cache echo "Inferred asset-group name from Git HEAD to be “${branch}”" - END - # Reset to alpine from the LOCALLY above - FROM artifactory.corp.mongodb.com/dockerhub/library/alpine:3.19 - RUN apk add python3 - # Copy in the script - COPY tools/create-silk-asset-group.py /opt/ - # # Run the creation script. Refer to tools/create-silk-asset-group.py for details - RUN --no-cache \ - --secret SILK_CLIENT_ID \ - --secret SILK_CLIENT_SECRET \ - python /opt/create-silk-asset-group.py \ - --branch=${branch} \ - --project=mongo-c-driver \ - --code-repo-url=https://github.com/mongodb/mongo-c-driver \ - --sbom-lite-path=etc/cyclonedx.sbom.json \ - --exist-ok + # Copy in the relevant files: + WORKDIR /s + COPY etc/purls.txt etc/cyclonedx.sbom.json /s/ + # Update the SBOM file: + RUN silkbomb update \ + --refresh \ + --generate-new-serial-number \ + --purls purls.txt \ + --sbom-in cyclonedx.sbom.json \ + --sbom-out cyclonedx.sbom.json + # Save the result back to the host: + SAVE ARTIFACT /s/cyclonedx.sbom.json AS LOCAL etc/cyclonedx.sbom.json +# sbom-validate: +# Validate the SBOM Lite for the given branch. +sbom-validate: + FROM +silkbomb + # Copy in the relevant files: + WORKDIR /s + COPY etc/purls.txt etc/cyclonedx.sbom.json /s/ + # Run the SilkBomb tool to download the artifact that matches the requested branch + RUN silkbomb validate \ + --purls purls.txt \ + --sbom-in cyclonedx.sbom.json \ + --exclude jira snyk: FROM --platform=linux/amd64 artifactory.corp.mongodb.com/dockerhub/library/ubuntu:24.04 diff --git a/docs/dev/deps.rst b/docs/dev/deps.rst index dec5cb56e19..44253035acb 100644 --- a/docs/dev/deps.rst +++ b/docs/dev/deps.rst @@ -47,10 +47,9 @@ copyright statements, URLs, and license identifiers of the dependencies. .. file:: etc/cyclonedx.sbom.json - The `SBOM-lite`_ for the C driver project. This is injested automatically and - asynchronously by Silk to produce the `augmented SBOM`_. This file is - generated semi-automatically from `etc/purls.txt` and the `+sbom-generate` - Earthly target. + The `SBOM-lite`_ for the C driver project. This file is generated + semi-automatically from `etc/purls.txt` and the `+sbom-generate` Earthly + target. This file is used by SilkBomb to produce an `augmented SBOM`_. .. warning:: This file is **partially generated**! Prefer to edit `etc/purls.txt`! Refer to: `sbom-lite-updating` @@ -98,38 +97,10 @@ The augmented SBOM contains extra data about the dependencies from the `SBOM-lite `, including vulnerabilities known at the time of the augmented SBOM's generation. [#asbom-vulns]_ -The augmented SBOM is produced automatically and asynchronously as part of an -external process that is not contained within the repository itself. The -augmented SBOM is downloaded from an internal service using the `+sbom-download` -Earthly target, which is automatically included in the release archive for the -`+release-archive` target. - -.. _silk-asset-group: - -Silk Asset Groups -***************** - -.. note:: A Silk asset group will be created automatically for each branch that - is executed in CI. - -We use Silk's *asset groups* to allow tracking of multiple versions of the -SBOM-lite_ simultaneously (i.e. one for each release branch). These asset groups -correspond to branches within the repository, and are created automatically when -CI executes for the first time on a particular branch. If you need an asset -group for a branch that has not run in CI, use the `+create-silk-asset-group` -Earthly target to create the asset group on-demand. - -Note that Silk pulls from the upstream Git repository for an asset group, so -creating an asset group for a branch that does not exist in the main upstream -repository will not work. - -.. file:: tools/create-silk-asset-group.py - - A Python script that will create an `asset group ` in Silk - based on a set of parameters. Execute with ``--help`` for more information. - For the C driver, it is easier to use the `+create-silk-asset-group` Earthly - target. - +The augmented SBOM is produced by an external process that is not contained +within the repository itself. The augmented SBOM must be downloaded from a +recent execution of the ``sbom`` task in an Evergreen patch or commit build. +This file is included in the release archive for the `+release-archive` target. .. _snyk scanning: @@ -152,8 +123,7 @@ detect the other dependencies within the project. The `+snyk-test` Earthly target is written to avoid this issue and allow Snyk to accurately detect other dependencies within the project. -Due to difficulty coordinating the behavior of Snyk and Silk at time of -writing, vulnerability collection is partially a manual process. This is +For now, vulnerability collection is partially a manual process. This is especially viable as the native code contains a very small number of dependencies and it is trivial to validate the output of Snyk by hand. diff --git a/docs/dev/earthly.rst b/docs/dev/earthly.rst index c8ab95db3a0..2bbfb838da8 100644 --- a/docs/dev/earthly.rst +++ b/docs/dev/earthly.rst @@ -90,11 +90,6 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository `+sign-file/signature.asc` for the release. The exported filenames are based on the `--version` argument. - .. rubric:: Parameters - .. option:: --sbom_branch - - Forwarded to `+release-archive --sbom_branch` - .. option:: --version Affects the output filename and archive prefix paths in @@ -108,22 +103,22 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository .. rubric:: Secrets - Secrets for the `+sbom-download`, `+snyk-test`, and `+sign-file` targets are - required for this target. + Secrets for the `+snyk-test` and `+sign-file` targets are required for this + target. .. program:: +release-archive .. earthly-target:: +release-archive - Generate a source release archive of the repository at a specifiy branch. - Requires the secrets for `+sbom-download` and `+snyk-test`. + Generate a source release archive of the repository for the specified branch. + Requires the secrets for `+snyk-test`. + Requires ``etc/augmented-sbom.json`` is present (obtained from Evergreen). .. earthly-artifact:: +release-archive/release.tar.gz The resulting source distribution archive for the specified branch. The generated archive includes the source tree, but also includes other - release artifacts that are generated on-the-fly when invoked e.g. the - `+sbom-download/augmented-sbom.json` artifact. + release artifacts that are generated on-the-fly when invoked. .. earthly-artifact:: +release-archive/ssdlc_compliance_report.md @@ -131,11 +126,6 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository content of ``etc/ssdlc.md``, which has certain substrings replaced based on attributes of the release. - .. rubric:: Parameters - .. option:: --sbom_branch - - Forwarded as `+sbom-download --branch` to download the augmented SBOM. - .. option:: --ref Specifies the Git revision that is used when we use ``git archive`` to @@ -150,38 +140,10 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository no effect on the files archived, which is selected by `+release-archive --ref`. +.. program:: +sbom-validate +.. earthly-target:: +sbom-validate -.. program:: +sbom-download -.. earthly-target:: +sbom-download - - Download an `augmented SBOM ` from Silk for a given project - branch. This target explicitly disables caching, because the upstream SBOM - file can change arbitrarily. - - .. earthly-artifact:: +sbom-download/augmented-sbom.json - - The `augmented SBOM ` downloaded from Silk for the requested branch. - - .. rubric:: Parameters - .. option:: --branch - - **Required**. Specifies the branch of the repository from which we are - requesting an SBOM. - - .. note:: - - It is *required* that the `Silk asset group ` has - been created for the given branch before the `+sbom-download` target - can succeed. See: `+create-silk-asset-group` - - .. rubric:: Secrets - .. envvar:: - SILK_CLIENT_ID - SILK_CLIENT_SECRET - - **Required**. [#creds]_ - - .. seealso:: `earthly.secrets` + Validate the `etc/cyclonedx.sbom.json`. .. program:: +sign-file .. earthly-target:: +sign-file @@ -249,32 +211,15 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository .. seealso:: `sbom-lite` and `sbom-lite-updating` +.. earthly-target:: +sbom-generate-new-serial-number -.. program:: +create-silk-asset-group -.. earthly-target:: +create-silk-asset-group - - Creates a new `Silk asset group ` for a branch in the - repository. This target executes the `tools/create-silk-asset-group.py` - script with the appropriate arguments. - - .. note:: For branches that execute in CI, running this target manually is - not necessary, as it is run automatically for every build. - - .. rubric:: Parameters - .. option:: --branch - - The repository branch for which to create the new asset group. If not - specified, the branch name will be inferred by asking Git. - - .. rubric:: Secrets - .. envvar:: - SILK_CLIENT_ID - SILK_CLIENT_SECRET - :noindex: + Equivalent to `+sbom-generate` but uses the ``--generate-new-serial-number`` + flag to generate a new unique serial number and reset the SBOM version to 1. - **Required**. [#creds]_ + After running this target, the contents of the `etc/cyclonedx.sbom.json` file + may change. - .. seealso:: `earthly.secrets` + .. seealso:: `sbom-lite` and `sbom-lite-updating` .. program:: +snyk-monitor-snapshot .. earthly-target:: +snyk-monitor-snapshot diff --git a/docs/dev/releasing.rst b/docs/dev/releasing.rst index 61a5fda9f3a..d4fbc960528 100644 --- a/docs/dev/releasing.rst +++ b/docs/dev/releasing.rst @@ -381,15 +381,15 @@ running :any:`+signed-release`, one will need to set up some environment that is required for it to succeed: 1. :ref:`Authenticate with Artifactory ` -2. Set the Earthly secrets required for the :any:`+sign-file` and - :any:`+sbom-download` targets. +2. Set the Earthly secrets required for the :any:`+sign-file` target. +3. Download an augmented SBOM from a recent execution of the ``sbom`` task in + an Evergreen patch or commit build and save it to ``etc/augmented-sbom.json``. Once these prerequesites are met, creating the release archive can be done using the :any:`+signed-release` target.:: - $ ./tools/earthly.sh --artifact +signed-release/dist dist --sbom_branch=$SBOM_BRANCH --version=$NEW_VERSION + $ ./tools/earthly.sh --artifact +signed-release/dist dist --version=$NEW_VERSION -.. note:: `$SBOM_BRANCH` must be ``master`` for a minor release, or ``$RELEASE_BRANCH`` for a patch release. .. note:: `$NEW_VERSION` must correspond to the Git tag created by the release. The above command will create a `dist/` directory in the working directory that @@ -633,4 +633,3 @@ updating the mongo-cxx-driver container image files to use the newly released C version. `Details for this process are documented here`__ __ https://github.com/mongodb/mongo-cxx-driver/blob/5f2077f98140ea656983ea5881de31d73bb3f735/etc/releasing.md#docker-image-build-and-publish - diff --git a/tools/create-silk-asset-group.py b/tools/create-silk-asset-group.py deleted file mode 100644 index 2a76760ca54..00000000000 --- a/tools/create-silk-asset-group.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python - -""" -Reusable utility script for creating an asset group in Silk. Invoke this script -with "--help" for details -""" - -from __future__ import annotations - -import argparse -import http -import http.client -import json -import os -import sys -import urllib.error -import urllib.request -from dataclasses import dataclass -from typing import Protocol, Sequence, cast - -JSON_MIME = "application/json" -JSON_HEADERS = {"Accept": JSON_MIME, "Content-Type": JSON_MIME} - - -class CommandParams(Protocol): - silk_endpoint: str - silk_client_id: None | str - silk_client_secret: None | str - exist_ok: bool - project: str - code_repo_url: str - branch: str - asset_id: str | None - sbom_lite_path: str - - -def parse_argv(argv: Sequence[str]) -> CommandParams: - parser = argparse.ArgumentParser( - description=""" - This script creates an asset group in Silk using the Silk HTTP API. This - script requires credentials to access Silk. - """, - epilog=""" - The --silk-client-id and --silk-client-secret can be passed via their - associated environment variables (recommended). If credentials are - not provided, then this script will immediately fail with an error. - """, - allow_abbrev=False, - ) - parser.add_argument( - "--project", - required=True, - metavar="", - help="The name of the project that owns the asset group", - ) - parser.add_argument( - "--branch", - metavar="", - required=True, - help="The name of the project's branch", - ) - parser.add_argument( - "--code-repo-url", - required=True, - metavar="", - help="The repository URL where the project can be found", - ) - parser.add_argument( - "--asset-id", - metavar="", - help='The asset ID to be created (Default is "-")', - ) - parser.add_argument( - "--sbom-lite-path", - required=True, - metavar="", - help="Relative file path within the project respository on the specified branch where the CycloneDX SBOM-Lite can be found", - ) - parser.add_argument( - "--exist-ok", - action="store_true", - help="If specified, do not generate an error if the asset group already exists", - ) - parser.add_argument( - "--silk-client-id", - default=os.environ.get("SILK_CLIENT_ID"), - metavar="", - help="The Silk API client ID (env: SILK_CLIENT_ID)", - ) - parser.add_argument( - "--silk-client-secret", - default=os.environ.get("SILK_CLIENT_SECRET"), - help="The Silk API secret access token (env: SILK_CLIENT_SECRET)", - metavar="", - ) - parser.add_argument( - "--silk-endpoint", - metavar="", - help="The base API URL for Silk access", - default="https://silkapi.us1.app.silk.security/api/v1", - ) - return cast(CommandParams, parser.parse_args(argv)) - - -@dataclass -class SimpleSilkClient: - endpoint: str - """The base API endpoint for Silk access""" - auth_token: str - """The Authorization token that can be used to act on the Silk server""" - - @classmethod - def open(cls, api_ep: str, client_id: str, secret: str) -> SimpleSilkClient: - """ - Open a new authenticated client against the Silk server. - """ - auth_ep = f"{api_ep}/authenticate" - req = urllib.request.Request( - auth_ep, - method="POST", - headers=JSON_HEADERS, - data=json.dumps( - { - "client_id": client_id, - "client_secret": secret, - } - ).encode(), - ) - resp: http.client.HTTPResponse - try: - with urllib.request.urlopen(req) as resp: - resp_json = json.loads(resp.read()) - except urllib.error.HTTPError as err: - raise RuntimeError( - f"Attempting to authenticate with Silk failed (see context)" - ) from err - match resp_json: - case {"token": str(tk)}: - return SimpleSilkClient(api_ep, tk) - case _: - assert False, f"Abnormal authentication response from Silk {resp_json=}" - - def create_asset_group( - self, - *, - branch: str, - asset_id: str, - sbom_lite_path: str, - code_repo_url: str, - project: str, - exist_ok: bool, - ) -> None: - """ - Create a new Silk asset group with the given parameters. - """ - req = urllib.request.Request( - f"{self.endpoint}/raw/asset_group", - data=json.dumps( - { - "active": True, - "name": project, - "code_repo_url": code_repo_url, - "branch": branch, - "file_paths": ["*"], - "metadata": { - "sbom_lite_path": sbom_lite_path, - }, - "asset_id": asset_id, - } - ).encode(), - headers=JSON_HEADERS | {"Authorization": self.auth_token}, - ) - try: - with urllib.request.urlopen(req) as resp: - print(json.loads(resp.read())) # No-op, but validates that JSON was returned - except urllib.error.HTTPError as err: - if err.status == 409 and exist_ok: - # 409: Conflict - print(f"No-op: Asset group {asset_id!r} already exists") - return - raise RuntimeError( - f"Creating Silk asset group {asset_id!r} failed (see context)" - ) from err - else: - print(f"Asset group {asset_id!r} was created for {project}#{branch}") - - -def main(argv: Sequence[str]) -> int: - args = parse_argv(argv) - # Enforce that we have a client ID and secret - cid = args.silk_client_id - if cid is None: - raise ValueError("A --silk-client-id/$SILK_CLIENT_ID is required") - secret = args.silk_client_secret - if secret is None: - raise ValueError("A --silk-client-secret/$SILK_CLIENT_SECRET is required") - - cl = SimpleSilkClient.open(args.silk_endpoint, cid, secret) - cl.create_asset_group( - branch=args.branch, - asset_id=args.asset_id or f"{args.project}-{args.branch}", - sbom_lite_path=args.sbom_lite_path, - code_repo_url=args.code_repo_url, - project=args.project, - exist_ok=args.exist_ok, - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:]))