diff --git a/.github/workflows/__per-language-bundle.yml b/.github/workflows/__per-language-bundle.yml index 4245a3d3a..c94c79867 100644 --- a/.github/workflows/__per-language-bundle.yml +++ b/.github/workflows/__per-language-bundle.yml @@ -80,6 +80,22 @@ jobs: exit 1 fi done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, where + # a later job analyzing a different language could pick it up. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [ -d "$RUNNER_TOOL_CACHE/CodeQL" ]; then + echo "::error::The toolcache contains CodeQL, but no bundle should have been added to it." + ls -R "$RUNNER_TOOL_CACHE/CodeQL" + exit 1 + fi - uses: ./../action/analyze with: upload-database: false diff --git a/lib/entry-points.js b/lib/entry-points.js index ffae24dc4..c06d53289 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -151514,6 +151514,17 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { // src/per-language-bundles.ts var semver7 = __toESM(require_semver2()); var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "99.99.99"; +var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; +function tryGetBundleLanguageFromUrl(url2) { + let assetName; + try { + assetName = new URL(url2).pathname.split("/").pop() ?? ""; + } catch { + return void 0; + } + const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + return match2 ? parseBuiltInLanguage(match2[1]) : void 0; +} var PER_LANGUAGE_BUNDLE_PLATFORMS = { ["actions" /* actions */]: "linux64", ["cpp" /* cpp */]: "linux64", @@ -152432,6 +152443,13 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO ); } compressionMethod = method; + const language = tryGetBundleLanguageFromUrl(url2); + if (language !== void 0) { + logger.info( + `${url2} appears to be a CodeQL bundle that contains only ${language}.` + ); + perLanguageBundle = { language }; + } } if (cliVersion2) { logger.info(`Using CodeQL CLI version ${cliVersion2} sourced from ${url2} .`); @@ -152637,7 +152655,7 @@ async function downloadCodeQLBundle(source, apiDetails, tarVersion, tempDir, fea } }; } catch (e) { - if (asHTTPError(e)?.status !== 404) { + if (perLanguageBundle.combinedBundleURL === void 0 || asHTTPError(e)?.status !== 404) { throw e; } logger.warning( diff --git a/pr-checks/checks/per-language-bundle.yml b/pr-checks/checks/per-language-bundle.yml index bf500e2aa..032089d42 100644 --- a/pr-checks/checks/per-language-bundle.yml +++ b/pr-checks/checks/per-language-bundle.yml @@ -33,6 +33,22 @@ steps: exit 1 fi done + - name: Check that the bundle was not added to the toolcache + env: + CODEQL_PATH: ${{ steps.init.outputs.codeql-path }} + run: | + # A bundle that is missing most of its extractors must never be left in the toolcache, where + # a later job analyzing a different language could pick it up. + echo "CodeQL is at $CODEQL_PATH" + if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then + echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH." + exit 1 + fi + if [ -d "$RUNNER_TOOL_CACHE/CodeQL" ]; then + echo "::error::The toolcache contains CodeQL, but no bundle should have been added to it." + ls -R "$RUNNER_TOOL_CACHE/CodeQL" + exit 1 + fi - uses: ./../action/analyze with: upload-database: false diff --git a/src/per-language-bundles.test.ts b/src/per-language-bundles.test.ts index ea34347f5..43b2fe1ca 100644 --- a/src/per-language-bundles.test.ts +++ b/src/per-language-bundles.test.ts @@ -8,6 +8,7 @@ import { getPerLanguageBundleLanguage, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, PerLanguageBundleOptions, + tryGetBundleLanguageFromUrl, } from "./per-language-bundles"; import { createFeatures, setupTests } from "./testing-utils"; import { GitHubVariant } from "./util"; @@ -141,6 +142,53 @@ test.serial("requires the feature flag", async (t) => { t.is(await checkEligibility({}, []), undefined); }); +test.serial("recognizes a per-language bundle from its URL", (t) => { + const url = (name: string) => + `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-java-linux64.tar.zst")), + BuiltInLanguage.java, + ); + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-swift-osx64.tar.zst")), + BuiltInLanguage.swift, + ); + // We do not publish these, but should still recognize them if we ever do. + t.is( + tryGetBundleLanguageFromUrl(url("codeql-bundle-csharp-win64.tar.gz")), + BuiltInLanguage.csharp, + ); + // A token in the URL must not prevent us from recognizing the bundle. + t.is( + tryGetBundleLanguageFromUrl( + `${url("codeql-bundle-ruby-linux64.tar.zst")}?token=secret`, + ), + BuiltInLanguage.ruby, + ); +}); + +test.serial("does not mistake other bundles for per-language ones", (t) => { + const url = (name: string) => + `https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`; + + for (const name of [ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-osx64.tar.gz", + "codeql-bundle-win64.tar.zst", + // The all-platform bundle. + "codeql-bundle.tar.gz", + // A platform we do not publish per-language bundles for, whose name also contains a hyphen. + "codeql-bundle-linux-arm64.tar.zst", + // Not a language we know about. + "codeql-bundle-cobol-linux64.tar.zst", + ]) { + t.is(tryGetBundleLanguageFromUrl(url(name)), undefined, name); + } + + t.is(tryGetBundleLanguageFromUrl("not a url"), undefined); +}); + function decrementPatchVersion(version: string): string { const [major, minor, patch] = version.split(".").map(Number); return `${major}.${minor}.${patch - 1}`; diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index 34369ea87..d84e79604 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -17,6 +17,39 @@ import { GitHubVariant } from "./util"; */ export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "99.99.99"; +/** + * Matches the name of a bundle that contains a single language, capturing that language. + * + * The language comes before the platform so that the name of a per-language bundle never has the + * name of the combined bundle for the same platform as a prefix. + */ +const PER_LANGUAGE_BUNDLE_NAME = + /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/; + +/** + * Determines whether a URL points at a bundle that contains only a single language. + * + * We need to recognise these even when we did not choose to download one ourselves, since a bundle + * that is missing most of its extractors must not be added to the toolcache no matter how we came + * to be downloading it. + * + * @returns The language that the bundle contains, or `undefined` if the URL does not point at a + * bundle for a single language. + */ +export function tryGetBundleLanguageFromUrl( + url: string, +): BuiltInLanguage | undefined { + let assetName: string; + try { + assetName = new URL(url).pathname.split("/").pop() ?? ""; + } catch { + return undefined; + } + + const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME); + return match ? parseBuiltInLanguage(match[1]) : undefined; +} + /** * The platform of the per-language bundle we use for each language. * diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index e80aff40e..453133c21 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -837,7 +837,7 @@ test.serial( ); t.is(source.perLanguageBundle?.language, BuiltInLanguage.java); t.true( - source.perLanguageBundle?.combinedBundleURL.endsWith( + source.perLanguageBundle?.combinedBundleURL?.endsWith( "/codeql-bundle-linux64.tar.zst", ), ); @@ -1040,6 +1040,69 @@ test.serial( }, ); +test.serial( + "setupCodeQLBundle keeps an explicitly requested per-language bundle out of the toolcache", + async (t) => { + const downloadStub = sinon.stub(setupCodeql, "downloadCodeQL").resolves({ + codeqlFolder: "codeql", + statusReport: { totalDurationMs: 100 }, + toolsVersion: "9.9.9", + }); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + "https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ); + + // Even though we did not choose this bundle, it is still missing most of its extractors. + t.true(downloadStub.calledOnce); + t.true(isPerLanguageBundleArg(downloadStub.firstCall)); + t.is( + result.toolsDownloadStatusReport?.bundleLanguage, + BuiltInLanguage.ruby, + ); + }); + }, +); + +test.serial( + "setupCodeQLBundle does not substitute a bundle for an explicitly requested one that is missing", + async (t) => { + const downloadStub = sinon + .stub(setupCodeql, "downloadCodeQL") + .rejects(new util.HTTPError("Not Found", 404)); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + "https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst", + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + SAMPLE_DEFAULT_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ), + ); + + // Falling back would silently ignore the bundle that was asked for. + t.true(downloadStub.calledOnce); + }); + }, +); + test.serial( "getEnabledVersionsWithOverlayBaseDatabases returns flag-enabled versions present in cache, sorted desc", async (t) => { diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index ed3983a60..fe31eecf0 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -33,7 +33,10 @@ import { import { BuiltInLanguage } from "./languages"; import { Logger } from "./logging"; import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching"; -import { getPerLanguageBundleLanguage } from "./per-language-bundles"; +import { + getPerLanguageBundleLanguage, + tryGetBundleLanguageFromUrl, +} from "./per-language-bundles"; import * as tar from "./tar"; import { deleteToolcacheBundles, @@ -263,7 +266,11 @@ export type CodeQLToolsSource = */ perLanguageBundle?: { language: BuiltInLanguage; - combinedBundleURL: string; + /** + * Absent when the bundle was requested explicitly rather than chosen by us, since in that + * case we should honor the request rather than substituting a different bundle. + */ + combinedBundleURL?: string; }; sourceType: "download"; /** Human-readable description of the source of the tools for telemetry purposes. */ @@ -749,7 +756,7 @@ export async function getCodeQLSource( let compressionMethod: tar.CompressionMethod; let perLanguageBundle: - | { language: BuiltInLanguage; combinedBundleURL: string } + | { language: BuiltInLanguage; combinedBundleURL?: string } | undefined; if (!url) { @@ -801,6 +808,16 @@ export async function getCodeQLSource( ); } compressionMethod = method; + + // The bundle was requested explicitly rather than chosen by us, but we still need to know + // whether it contains a single language so that we do not add it to the toolcache. + const language = tryGetBundleLanguageFromUrl(url); + if (language !== undefined) { + logger.info( + `${url} appears to be a CodeQL bundle that contains only ${language}.`, + ); + perLanguageBundle = { language }; + } } if (cliVersion) { @@ -1157,7 +1174,12 @@ async function downloadCodeQLBundle( }, }; } catch (e) { - if (util.asHTTPError(e)?.status !== 404) { + // Without a combined bundle to fall back to, the bundle was requested explicitly, so there is + // nothing we can substitute for it. + if ( + perLanguageBundle.combinedBundleURL === undefined || + util.asHTTPError(e)?.status !== 404 + ) { throw e; } logger.warning(