Merge pull request #3126 from github/mbg/add/properties-api

Add support for the repository properties API
This commit is contained in:
Michael B. Gale
2025-09-23 10:17:52 +01:00
committed by GitHub
24 changed files with 795 additions and 39 deletions
@@ -6,6 +6,16 @@ import * as assert from 'assert'
const actualConfig = loadActualConfig()
function sortConfigArrays(config) {
for (const key of Object.keys(config)) {
const value = config[key];
if (key === 'queries' && Array.isArray(value)) {
config[key] = value.sort();
}
}
return config;
}
const rawExpectedConfig = process.argv[3].trim()
if (!rawExpectedConfig) {
core.setFailed('No expected configuration provided')
@@ -18,8 +28,8 @@ if (!rawExpectedConfig) {
const expectedConfig = rawExpectedConfig ? JSON.parse(rawExpectedConfig) : undefined;
assert.deepStrictEqual(
actualConfig,
expectedConfig,
sortConfigArrays(actualConfig),
sortConfigArrays(expectedConfig),
'Expected configuration does not match actual configuration'
);
@@ -180,13 +180,13 @@ jobs:
with:
expected-config-file-contents: |
{
"queries": [
{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql" },
{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" }
],
"packs": {
"javascript": ["codeql-testing/codeql-pack1@1.0.0", "codeql-testing/codeql-pack2", "codeql/javascript-queries" ]
}
},
"queries": [
{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql" },
{ "uses": "./codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql" }
]
}
languages: javascript
queries: + ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
+5
View File
@@ -117922,6 +117922,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+5
View File
@@ -91156,6 +91156,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+5
View File
@@ -78660,6 +78660,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+5
View File
@@ -129255,6 +129255,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+137 -11
View File
@@ -86128,6 +86128,12 @@ function computeAutomationID(analysis_key, environment) {
}
return automationID;
}
async function getRepositoryProperties(repositoryNwo) {
return getApiClient().request("GET /repos/:owner/:repo/properties/values", {
owner: repositoryNwo.owner,
repo: repositoryNwo.repo
});
}
// src/caching-utils.ts
var core6 = __toESM(require_core());
@@ -86223,6 +86229,9 @@ function getConfigFileFormatInvalidMessage(configFile) {
function getConfigFileDirectoryGivenMessage(configFile) {
return `The configuration file "${configFile}" looks like a directory, not a file`;
}
function getEmptyCombinesError() {
return `A '+' was used to specify that you want to add extra arguments to the configuration, but no extra arguments were specified. Please either remove the '+' or specify some extra arguments.`;
}
function getConfigFilePropertyError(configFile, property, error2) {
if (configFile === void 0) {
return `The workflow property "${property}" is invalid: ${error2}`;
@@ -86230,6 +86239,9 @@ function getConfigFilePropertyError(configFile, property, error2) {
return `The configuration file "${configFile}" is invalid: property "${property}" ${error2}`;
}
}
function getRepoPropertyError(propertyName, error2) {
return `The repository property "${propertyName}" is invalid: ${error2}`;
}
function getPacksStrInvalid(packStr, configFile) {
return configFile ? getConfigFilePropertyError(
configFile,
@@ -86244,6 +86256,52 @@ function getUnknownLanguagesError(languages) {
return `Did not recognize the following languages: ${languages.join(", ")}`;
}
// src/feature-flags/properties.ts
var RepositoryPropertyName = /* @__PURE__ */ ((RepositoryPropertyName2) => {
RepositoryPropertyName2["EXTRA_QUERIES"] = "github-codeql-extra-queries";
return RepositoryPropertyName2;
})(RepositoryPropertyName || {});
async function loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo) {
if (gitHubVersion.type === 1 /* GHES */) {
return {};
}
try {
const response = await getRepositoryProperties(repositoryNwo);
const remoteProperties = response.data;
if (!Array.isArray(remoteProperties)) {
throw new Error(
`Expected repository properties API to return an array, but got: ${JSON.stringify(response.data)}`
);
}
logger.debug(
`Retrieved ${remoteProperties.length} repository properties: ${remoteProperties.map((p) => p.property_name).join(", ")}`
);
const knownProperties = new Set(Object.values(RepositoryPropertyName));
const properties = {};
for (const property of remoteProperties) {
if (property.property_name === void 0) {
throw new Error(
`Expected property object to have a 'property_name', but got: ${JSON.stringify(property)}`
);
}
if (knownProperties.has(property.property_name)) {
properties[property.property_name] = property.value;
}
}
logger.debug("Loaded the following values for the repository properties:");
for (const [property, value] of Object.entries(properties).sort(
([nameA], [nameB]) => nameA.localeCompare(nameB)
)) {
logger.debug(` ${property}: ${value}`);
}
return properties;
} catch (e) {
throw new Error(
`Encountered an error while trying to determine repository properties: ${e}`
);
}
}
// src/config/db-config.ts
function shouldCombine(inputValue) {
return !!inputValue?.trim().startsWith("+");
@@ -86336,7 +86394,7 @@ function parsePacksFromInput(rawPacksInput, languages, packsInputCombines) {
}, [])
};
}
async function calculateAugmentation(rawPacksInput, rawQueriesInput, languages) {
async function calculateAugmentation(rawPacksInput, rawQueriesInput, repositoryProperties, languages) {
const packsInputCombines = shouldCombine(rawPacksInput);
const packsInput = parsePacksFromInput(
rawPacksInput,
@@ -86348,19 +86406,38 @@ async function calculateAugmentation(rawPacksInput, rawQueriesInput, languages)
rawQueriesInput,
queriesInputCombines
);
const repoExtraQueries = repositoryProperties["github-codeql-extra-queries" /* EXTRA_QUERIES */];
const repoExtraQueriesCombines = shouldCombine(repoExtraQueries);
const repoPropertyQueries = {
combines: repoExtraQueriesCombines,
input: parseQueriesFromInput(
repoExtraQueries,
repoExtraQueriesCombines,
new ConfigurationError(
getRepoPropertyError(
"github-codeql-extra-queries" /* EXTRA_QUERIES */,
getEmptyCombinesError()
)
)
)
};
return {
packsInputCombines,
packsInput: packsInput?.[languages[0]],
queriesInput,
queriesInputCombines
queriesInputCombines,
repoPropertyQueries
};
}
function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) {
function parseQueriesFromInput(rawQueriesInput, queriesInputCombines, errorToThrow) {
if (!rawQueriesInput) {
return void 0;
}
const trimmedInput = queriesInputCombines ? rawQueriesInput.trim().slice(1).trim() : rawQueriesInput?.trim() ?? "";
if (queriesInputCombines && trimmedInput.length === 0) {
if (errorToThrow) {
throw errorToThrow;
}
throw new ConfigurationError(
getConfigFilePropertyError(
void 0,
@@ -86371,17 +86448,43 @@ function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) {
}
return trimmedInput.split(",").map((query) => ({ uses: query.trim() }));
}
function generateCodeScanningConfig(originalUserInput, augmentationProperties) {
const augmentedConfig = cloneObject(originalUserInput);
if (augmentationProperties.queriesInput) {
if (augmentationProperties.queriesInputCombines) {
augmentedConfig.queries = (augmentedConfig.queries || []).concat(
augmentationProperties.queriesInput
function combineQueries(logger, config, augmentationProperties) {
const result = [];
if (augmentationProperties.repoPropertyQueries && augmentationProperties.repoPropertyQueries.input) {
logger.info(
`Found query configuration in the repository properties (${"github-codeql-extra-queries" /* EXTRA_QUERIES */}): ${augmentationProperties.repoPropertyQueries.input.map((q) => q.uses).join(", ")}`
);
if (!augmentationProperties.repoPropertyQueries.combines) {
logger.info(
`The queries configured in the repository properties don't allow combining with other query settings. Any queries configured elsewhere will be ignored.`
);
return augmentationProperties.repoPropertyQueries.input;
} else {
augmentedConfig.queries = augmentationProperties.queriesInput;
result.push(...augmentationProperties.repoPropertyQueries.input);
}
}
if (augmentationProperties.queriesInput) {
if (!augmentationProperties.queriesInputCombines) {
return result.concat(augmentationProperties.queriesInput);
} else {
result.push(...augmentationProperties.queriesInput);
}
}
if (config.queries) {
result.push(...config.queries);
}
return result;
}
function generateCodeScanningConfig(logger, originalUserInput, augmentationProperties) {
const augmentedConfig = cloneObject(originalUserInput);
augmentedConfig.queries = combineQueries(
logger,
augmentedConfig,
augmentationProperties
);
logger.debug(
`Combined queries: ${augmentedConfig.queries?.map((q) => q.uses).join(",")}`
);
if (augmentedConfig.queries?.length === 0) {
delete augmentedConfig.queries;
}
@@ -86983,6 +87086,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
@@ -87523,6 +87631,7 @@ async function initActionState({
sourceRoot,
githubVersion,
features,
repositoryProperties,
logger
}, userConfig) {
const analysisKinds = await parseAnalysisKinds(analysisKindsInput);
@@ -87546,8 +87655,18 @@ async function initActionState({
const augmentationProperties = await calculateAugmentation(
packsInput,
queriesInput,
repositoryProperties,
languages
);
if (analysisKinds.length === 1 && analysisKinds.includes("code-quality" /* CodeQuality */) && augmentationProperties.repoPropertyQueries.input) {
logger.info(
`Ignoring queries configured in the repository properties, because query customisations are not supported for Code Quality analyses.`
);
augmentationProperties.repoPropertyQueries = {
combines: false,
input: void 0
};
}
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
codeql,
@@ -87555,6 +87674,7 @@ async function initActionState({
logger
);
const computedConfig = generateCodeScanningConfig(
logger,
userConfig,
augmentationProperties
);
@@ -87577,7 +87697,8 @@ async function initActionState({
dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled),
extraQueryExclusions: [],
overlayDatabaseMode: "none" /* None */,
useOverlayDatabaseCaching: false
useOverlayDatabaseCaching: false,
repositoryProperties
};
}
async function downloadCacheWithTime(trapCachingEnabled, codeQL, languages, logger) {
@@ -90451,6 +90572,10 @@ async function run() {
getTemporaryDirectory(),
logger
);
const enableRepoProps = await features.getValue(
"use_repository_properties" /* UseRepositoryProperties */
);
const repositoryProperties = enableRepoProps ? await loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo) : {};
const jobRunUuid = v4_default();
logger.info(`Job run UUID is ${jobRunUuid}.`);
core13.exportVariable("JOB_RUN_UUID" /* JOB_RUN_UUID */, jobRunUuid);
@@ -90550,6 +90675,7 @@ async function run() {
githubVersion: gitHubVersion,
apiDetails,
features,
repositoryProperties,
logger
});
await checkInstallPython311(config.languages, codeql);
+5
View File
@@ -78651,6 +78651,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+5
View File
@@ -117331,6 +117331,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+5
View File
@@ -89347,6 +89347,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+5
View File
@@ -117494,6 +117494,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+5
View File
@@ -89335,6 +89335,11 @@ var featureConfig = {
minimumVersion: void 0,
toolsFeature: "pythonDefaultIsToNotExtractStdlib" /* PythonDefaultIsToNotExtractStdlib */
},
["use_repository_properties" /* UseRepositoryProperties */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: void 0
},
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+9 -1
View File
@@ -4,7 +4,7 @@ import * as retry from "@octokit/plugin-retry";
import consoleLogLevel from "console-log-level";
import { getActionVersion, getRequiredInput } from "./actions-util";
import { getRepositoryNwo } from "./repository";
import { getRepositoryNwo, RepositoryNwo } from "./repository";
import {
ConfigurationError,
getRequiredEnvParam,
@@ -240,6 +240,14 @@ export async function deleteActionsCache(id: number) {
});
}
/** Retrieve all custom repository properties. */
export async function getRepositoryProperties(repositoryNwo: RepositoryNwo) {
return getApiClient().request("GET /repos/:owner/:repo/properties/values", {
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
});
}
export function wrapApiConfigurationError(e: unknown) {
if (isHTTPError(e)) {
if (
+84 -3
View File
@@ -496,6 +496,8 @@ const injectedConfigMacro = test.macro({
expectedConfig: any,
) => {
await util.withTmpDir(async (tempDir) => {
sinon.stub(actionsUtil, "isDefaultSetup").resolves(false);
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await stubCodeql();
@@ -505,6 +507,7 @@ const injectedConfigMacro = test.macro({
tempDir,
};
thisStubConfig.computedConfig = generateCodeScanningConfig(
getRunnerLogger(true),
thisStubConfig.originalUserInput,
augmentationProperties,
);
@@ -659,15 +662,15 @@ test(
},
{
queries: [
{
uses: "zzz",
},
{
uses: "xxx",
},
{
uses: "yyy",
},
{
uses: "zzz",
},
],
},
);
@@ -713,6 +716,84 @@ test(
{},
);
test(
"repo property queries have the highest precedence",
injectedConfigMacro,
{
...defaultAugmentationProperties,
queriesInputCombines: true,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
repoPropertyQueries: {
combines: false,
input: [{ uses: "zzz" }, { uses: "aaa" }],
},
},
{
originalUserInput: {
queries: [{ uses: "uu" }, { uses: "vv" }],
},
},
{
queries: [{ uses: "zzz" }, { uses: "aaa" }],
},
);
test(
"repo property queries combines with queries input",
injectedConfigMacro,
{
...defaultAugmentationProperties,
queriesInputCombines: false,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
repoPropertyQueries: {
combines: true,
input: [{ uses: "zzz" }, { uses: "aaa" }],
},
},
{
originalUserInput: {
queries: [{ uses: "uu" }, { uses: "vv" }],
},
},
{
queries: [
{ uses: "zzz" },
{ uses: "aaa" },
{ uses: "xxx" },
{ uses: "yyy" },
],
},
);
test(
"repo property queries combines everything else",
injectedConfigMacro,
{
...defaultAugmentationProperties,
queriesInputCombines: true,
queriesInput: [{ uses: "xxx" }, { uses: "yyy" }],
repoPropertyQueries: {
combines: true,
input: [{ uses: "zzz" }, { uses: "aaa" }],
},
},
{
originalUserInput: {
queries: [{ uses: "uu" }, { uses: "vv" }],
},
},
{
queries: [
{ uses: "zzz" },
{ uses: "aaa" },
{ uses: "xxx" },
{ uses: "yyy" },
{ uses: "uu" },
{ uses: "vv" },
],
},
);
test("passes a code scanning config AND qlconfig to the CLI", async (t: ExecutionContext<unknown>) => {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
+62 -2
View File
@@ -29,6 +29,7 @@ import {
getRecordingLogger,
LoggedMessage,
mockCodeQLVersion,
createTestConfig,
} from "./testing-utils";
import {
GitHubVariant,
@@ -82,11 +83,11 @@ function createTestInitConfigInputs(
externalRepoAuth: "token",
url: "https://github.example.com",
apiURL: undefined,
registriesAuthTokens: undefined,
},
features: createFeatures([]),
repositoryProperties: {},
logger: getRunnerLogger(true),
},
} satisfies configUtils.InitConfigInputs,
overrides,
);
}
@@ -223,12 +224,70 @@ test("load code quality config", async (t) => {
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
t.deepEqual(config, expectedConfig);
});
});
test("initActionState doesn't throw if there are queries configured in the repository properties", async (t) => {
return await withTmpDir(async (tempDir) => {
const logger = getRunnerLogger(true);
const languages = "javascript";
const codeql = createStubCodeQL({
async betterResolveLanguages() {
return {
extractors: {
javascript: [{ extractor_root: "" }],
},
};
},
});
// This should be ignored and no error should be thrown.
const repositoryProperties = {
"github-codeql-extra-queries": "+foo",
};
// Expected configuration for a CQ-only analysis.
const computedConfig: configUtils.UserConfig = {
"disable-default-queries": true,
queries: [{ uses: "code-quality" }],
"query-filters": [],
};
const expectedConfig = createTestConfig({
analysisKinds: [AnalysisKind.CodeQuality],
languages: [KnownLanguage.javascript],
codeQLCmd: codeql.getPath(),
computedConfig,
dbLocation: path.resolve(tempDir, "codeql_databases"),
debugArtifactName: "",
debugDatabaseName: "",
tempDir,
repositoryProperties,
});
await t.notThrowsAsync(async () => {
const config = await configUtils.initConfig(
createTestInitConfigInputs({
analysisKindsInput: "code-quality",
languagesInput: languages,
repository: { owner: "github", repo: "example" },
tempDir,
codeql,
repositoryProperties,
logger,
}),
);
t.deepEqual(config, expectedConfig);
});
});
});
test("loading a saved config produces the same config", async (t) => {
return await withTmpDir(async (tempDir) => {
const logger = getRunnerLogger(true);
@@ -461,6 +520,7 @@ test("load non-empty input", async (t) => {
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
const languagesInput = "javascript";
+29
View File
@@ -25,6 +25,7 @@ import {
import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import * as errorMessages from "./error-messages";
import { Feature, FeatureEnablement } from "./feature-flags";
import { RepositoryProperties } from "./feature-flags/properties";
import { getGitRoot, isAnalyzingDefaultBranch } from "./git-utils";
import { KnownLanguage, Language } from "./languages";
import { Logger } from "./logging";
@@ -167,6 +168,11 @@ export interface Config {
* `OverlayBase`.
*/
useOverlayDatabaseCaching: boolean;
/**
* A partial mapping from repository properties that affect us to their values.
*/
repositoryProperties: RepositoryProperties;
}
export async function getSupportedLanguageMap(
@@ -389,6 +395,7 @@ export interface InitConfigInputs {
githubVersion: GitHubVersion;
apiDetails: api.GitHubApiCombinedDetails;
features: FeatureEnablement;
repositoryProperties: RepositoryProperties;
logger: Logger;
}
@@ -416,6 +423,7 @@ export async function initActionState(
sourceRoot,
githubVersion,
features,
repositoryProperties,
logger,
}: InitConfigInputs,
userConfig: UserConfig,
@@ -451,9 +459,28 @@ export async function initActionState(
const augmentationProperties = await calculateAugmentation(
packsInput,
queriesInput,
repositoryProperties,
languages,
);
// If `code-quality` is the only enabled analysis kind, we don't support query customisation.
// It would be a problem if queries that are configured in repository properties cause `code-quality`-only
// analyses to break. We therefore ignore query customisations that are configured in repository properties
// if `code-quality` is the only enabled analysis kind.
if (
analysisKinds.length === 1 &&
analysisKinds.includes(AnalysisKind.CodeQuality) &&
augmentationProperties.repoPropertyQueries.input
) {
logger.info(
`Ignoring queries configured in the repository properties, because query customisations are not supported for Code Quality analyses.`,
);
augmentationProperties.repoPropertyQueries = {
combines: false,
input: undefined,
};
}
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
codeql,
@@ -464,6 +491,7 @@ export async function initActionState(
// Compute the full Code Scanning configuration that combines the configuration from the
// configuration file / `config` input with other inputs, such as `queries`.
const computedConfig = generateCodeScanningConfig(
logger,
userConfig,
augmentationProperties,
);
@@ -488,6 +516,7 @@ export async function initActionState(
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties,
};
}
+63
View File
@@ -1,5 +1,6 @@
import test, { ExecutionContext } from "ava";
import { RepositoryProperties } from "../feature-flags/properties";
import { KnownLanguage, Language } from "../languages";
import { prettyPrintPack } from "../util";
@@ -190,11 +191,13 @@ const calculateAugmentationMacro = test.macro({
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[],
repositoryProperties: RepositoryProperties,
expectedAugmentationProperties: dbConfig.AugmentationProperties,
) => {
const actualAugmentationProperties = await dbConfig.calculateAugmentation(
rawPacksInput,
rawQueriesInput,
repositoryProperties,
languages,
);
t.deepEqual(actualAugmentationProperties, expectedAugmentationProperties);
@@ -208,6 +211,7 @@ test(
undefined,
undefined,
[KnownLanguage.javascript],
{},
{
...dbConfig.defaultAugmentationProperties,
},
@@ -219,6 +223,7 @@ test(
undefined,
" a, b , c, d",
[KnownLanguage.javascript],
{},
{
...dbConfig.defaultAugmentationProperties,
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
@@ -231,6 +236,7 @@ test(
undefined,
" + a, b , c, d ",
[KnownLanguage.javascript],
{},
{
...dbConfig.defaultAugmentationProperties,
queriesInputCombines: true,
@@ -244,6 +250,7 @@ test(
" codeql/a , codeql/b , codeql/c , codeql/d ",
undefined,
[KnownLanguage.javascript],
{},
{
...dbConfig.defaultAugmentationProperties,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
@@ -256,6 +263,7 @@ test(
" + codeql/a, codeql/b, codeql/c, codeql/d",
undefined,
[KnownLanguage.javascript],
{},
{
...dbConfig.defaultAugmentationProperties,
packsInputCombines: true,
@@ -263,6 +271,42 @@ test(
},
);
test(
calculateAugmentationMacro,
"With repo property queries",
undefined,
undefined,
[KnownLanguage.javascript],
{
"github-codeql-extra-queries": "a, b, c, d",
},
{
...dbConfig.defaultAugmentationProperties,
repoPropertyQueries: {
combines: false,
input: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
},
},
);
test(
calculateAugmentationMacro,
"With repo property queries combining",
undefined,
undefined,
[KnownLanguage.javascript],
{
"github-codeql-extra-queries": "+ a, b, c, d",
},
{
...dbConfig.defaultAugmentationProperties,
repoPropertyQueries: {
combines: true,
input: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
},
},
);
const calculateAugmentationErrorMacro = test.macro({
exec: async (
t: ExecutionContext,
@@ -270,6 +314,7 @@ const calculateAugmentationErrorMacro = test.macro({
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[],
repositoryProperties: RepositoryProperties,
expectedError: RegExp | string,
) => {
await t.throwsAsync(
@@ -277,6 +322,7 @@ const calculateAugmentationErrorMacro = test.macro({
dbConfig.calculateAugmentation(
rawPacksInput,
rawQueriesInput,
repositoryProperties,
languages,
),
{ message: expectedError },
@@ -291,6 +337,7 @@ test(
undefined,
" + ",
[KnownLanguage.javascript],
{},
/The workflow property "queries" is invalid/,
);
@@ -300,15 +347,29 @@ test(
" + ",
undefined,
[KnownLanguage.javascript],
{},
/The workflow property "packs" is invalid/,
);
test(
calculateAugmentationErrorMacro,
"Plus (+) with nothing else (repo property queries)",
undefined,
undefined,
[KnownLanguage.javascript],
{
"github-codeql-extra-queries": " + ",
},
/The repository property "github-codeql-extra-queries" is invalid/,
);
test(
calculateAugmentationErrorMacro,
"Packs input with multiple languages",
" + a/b, c/d ",
undefined,
[KnownLanguage.javascript, KnownLanguage.java],
{},
/Cannot specify a 'packs' input in a multi-language analysis/,
);
@@ -318,6 +379,7 @@ test(
" + a/b, c/d ",
undefined,
[],
{},
/No languages specified/,
);
@@ -327,5 +389,6 @@ test(
" a-pack-without-a-scope ",
undefined,
[KnownLanguage.javascript],
{},
/"a-pack-without-a-scope" is not a valid pack/,
);
+128 -15
View File
@@ -3,7 +3,12 @@ import * as path from "path";
import * as semver from "semver";
import * as errorMessages from "../error-messages";
import {
RepositoryProperties,
RepositoryPropertyName,
} from "../feature-flags/properties";
import { Language } from "../languages";
import { Logger } from "../logging";
import { cloneObject, ConfigurationError, prettyPrintPack } from "../util";
export interface ExcludeQueryFilter {
@@ -16,16 +21,18 @@ export interface IncludeQueryFilter {
export type QueryFilter = ExcludeQueryFilter | IncludeQueryFilter;
export interface QuerySpec {
name?: string;
uses: string;
}
/**
* Format of the config file supplied by the user.
*/
export interface UserConfig {
name?: string;
"disable-default-queries"?: boolean;
queries?: Array<{
name?: string;
uses: string;
}>;
queries?: QuerySpec[];
"paths-ignore"?: string[];
paths?: string[];
@@ -39,6 +46,17 @@ export interface UserConfig {
"query-filters"?: QueryFilter[];
}
/**
* Represents additional configuration data from a source other than
* a configuration file.
*/
interface Augmentation<T> {
/** Whether or not the `input` combines with data in the base config. */
combines: boolean;
/** The additional input data. */
input?: T;
}
/**
* Describes how to augment the user config with inputs from the action.
*
@@ -58,7 +76,7 @@ export interface AugmentationProperties {
/**
* The queries input from the `with` block of the action declaration
*/
queriesInput?: Array<{ uses: string }>;
queriesInput?: QuerySpec[];
/**
* Whether or not the packs input combines with the packs in the config.
@@ -69,6 +87,11 @@ export interface AugmentationProperties {
* The packs input from the `with` block of the action declaration
*/
packsInput?: string[];
/**
* Extra queries from the corresponding repository property.
*/
repoPropertyQueries: Augmentation<QuerySpec[]>;
}
/**
@@ -80,6 +103,10 @@ export const defaultAugmentationProperties: AugmentationProperties = {
packsInputCombines: false,
packsInput: undefined,
queriesInput: undefined,
repoPropertyQueries: {
combines: false,
input: undefined,
},
};
/**
@@ -254,6 +281,7 @@ export function parsePacksFromInput(
*
* @param rawPacksInput The packs input from the action configuration.
* @param rawQueriesInput The queries input from the action configuration.
* @param repositoryProperties The dictionary of repository properties.
* @param languages The languages that the config file is for. If the packs input
* is non-empty, then there must be exactly one language. Otherwise, an
* error is thrown.
@@ -263,10 +291,10 @@ export function parsePacksFromInput(
* @throws An error if the packs input is non-empty and the languages input does
* not have exactly one language.
*/
// exported for testing.
export async function calculateAugmentation(
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
repositoryProperties: RepositoryProperties,
languages: Language[],
): Promise<AugmentationProperties> {
const packsInputCombines = shouldCombine(rawPacksInput);
@@ -281,17 +309,36 @@ export async function calculateAugmentation(
queriesInputCombines,
);
const repoExtraQueries =
repositoryProperties[RepositoryPropertyName.EXTRA_QUERIES];
const repoExtraQueriesCombines = shouldCombine(repoExtraQueries);
const repoPropertyQueries = {
combines: repoExtraQueriesCombines,
input: parseQueriesFromInput(
repoExtraQueries,
repoExtraQueriesCombines,
new ConfigurationError(
errorMessages.getRepoPropertyError(
RepositoryPropertyName.EXTRA_QUERIES,
errorMessages.getEmptyCombinesError(),
),
),
),
};
return {
packsInputCombines,
packsInput: packsInput?.[languages[0]],
queriesInput,
queriesInputCombines,
repoPropertyQueries,
};
}
function parseQueriesFromInput(
rawQueriesInput: string | undefined,
queriesInputCombines: boolean,
errorToThrow?: ConfigurationError,
) {
if (!rawQueriesInput) {
return undefined;
@@ -301,6 +348,9 @@ function parseQueriesFromInput(
? rawQueriesInput.trim().slice(1).trim()
: (rawQueriesInput?.trim() ?? "");
if (queriesInputCombines && trimmedInput.length === 0) {
if (errorToThrow) {
throw errorToThrow;
}
throw new ConfigurationError(
errorMessages.getConfigFilePropertyError(
undefined,
@@ -312,7 +362,71 @@ function parseQueriesFromInput(
return trimmedInput.split(",").map((query) => ({ uses: query.trim() }));
}
/**
* Combines queries from various configuration sources.
*
* @param logger The logger to use.
* @param config The loaded configuration file (either `config-file` or `config` input).
* @param augmentationProperties Additional configuration data from other sources.
* @returns Returns `augmentedConfig` with `queries` set to the computed array of queries.
*/
function combineQueries(
logger: Logger,
config: UserConfig,
augmentationProperties: AugmentationProperties,
): QuerySpec[] {
const result: QuerySpec[] = [];
// Query settings obtained from the repository properties have the highest precedence.
if (
augmentationProperties.repoPropertyQueries &&
augmentationProperties.repoPropertyQueries.input
) {
logger.info(
`Found query configuration in the repository properties (${RepositoryPropertyName.EXTRA_QUERIES}): ` +
`${augmentationProperties.repoPropertyQueries.input.map((q) => q.uses).join(", ")}`,
);
// If there are queries configured as a repository property, these may be organisational
// settings. If they don't allow combining with other query configurations, return just the
// ones configured in the repository properties.
if (!augmentationProperties.repoPropertyQueries.combines) {
logger.info(
`The queries configured in the repository properties don't allow combining with other query settings. ` +
`Any queries configured elsewhere will be ignored.`,
);
return augmentationProperties.repoPropertyQueries.input;
} else {
// Otherwise, add them to the query array and continue.
result.push(...augmentationProperties.repoPropertyQueries.input);
}
}
// If there is a `queries` input to the Action, it has the next highest precedence.
if (augmentationProperties.queriesInput) {
// If there is a `queries` input and `queriesInputCombines` is `false`, then we don't
// combine it with the queries configured in the configuration file (if any). That is the
// original behaviour of this property. However, we DO combine it with any queries that
// we obtained from the repository properties, since that may be enforced by the organisation.
if (!augmentationProperties.queriesInputCombines) {
return result.concat(augmentationProperties.queriesInput);
} else {
// If they combine, add them to the query array and continue.
result.push(...augmentationProperties.queriesInput);
}
}
// If we get to this point, we either don't have any extra configuration inputs or all of them
// allow themselves to be combined with the settings from the configuration file.
if (config.queries) {
result.push(...config.queries);
}
return result;
}
export function generateCodeScanningConfig(
logger: Logger,
originalUserInput: UserConfig,
augmentationProperties: AugmentationProperties,
): UserConfig {
@@ -320,15 +434,14 @@ export function generateCodeScanningConfig(
const augmentedConfig = cloneObject(originalUserInput);
// Inject the queries from the input
if (augmentationProperties.queriesInput) {
if (augmentationProperties.queriesInputCombines) {
augmentedConfig.queries = (augmentedConfig.queries || []).concat(
augmentationProperties.queriesInput,
);
} else {
augmentedConfig.queries = augmentationProperties.queriesInput;
}
}
augmentedConfig.queries = combineQueries(
logger,
augmentedConfig,
augmentationProperties,
);
logger.debug(
`Combined queries: ${augmentedConfig.queries?.map((q) => q.uses).join(",")}`,
);
if (augmentedConfig.queries?.length === 0) {
delete augmentedConfig.queries;
}
+13
View File
@@ -1,3 +1,5 @@
import { RepositoryPropertyName } from "./feature-flags/properties";
const PACKS_PROPERTY = "packs";
export function getConfigFileOutsideWorkspaceErrorMessage(
@@ -29,6 +31,10 @@ export function getConfigFileDirectoryGivenMessage(configFile: string): string {
return `The configuration file "${configFile}" looks like a directory, not a file`;
}
export function getEmptyCombinesError(): string {
return `A '+' was used to specify that you want to add extra arguments to the configuration, but no extra arguments were specified. Please either remove the '+' or specify some extra arguments.`;
}
export function getConfigFilePropertyError(
configFile: string | undefined,
property: string,
@@ -41,6 +47,13 @@ export function getConfigFilePropertyError(
}
}
export function getRepoPropertyError(
propertyName: RepositoryPropertyName,
error: string,
): string {
return `The repository property "${propertyName}" is invalid: ${error}`;
}
export function getPacksStrInvalid(
packStr: string,
configFile?: string,
+6
View File
@@ -73,6 +73,7 @@ export enum Feature {
OverlayAnalysisRust = "overlay_analysis_rust",
OverlayAnalysisSwift = "overlay_analysis_swift",
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
UseRepositoryProperties = "use_repository_properties",
QaTelemetryEnabled = "qa_telemetry_enabled",
ResolveSupportedLanguagesUsingCli = "resolve_supported_languages_using_cli",
}
@@ -264,6 +265,11 @@ export const featureConfig: Record<
minimumVersion: undefined,
toolsFeature: ToolsFeature.PythonDefaultIsToNotExtractStdlib,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.QaTelemetryEnabled]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
+97
View File
@@ -0,0 +1,97 @@
import test from "ava";
import * as sinon from "sinon";
import * as api from "../api-client";
import { getRunnerLogger } from "../logging";
import { parseRepositoryNwo } from "../repository";
import { setupTests } from "../testing-utils";
import * as util from "../util";
import * as properties from "./properties";
setupTests(test);
test("loadPropertiesFromApi throws if response data is not an array", async (t) => {
sinon.stub(api, "getRepositoryProperties").resolves({
headers: {},
status: 200,
url: "",
data: {},
});
const logger = getRunnerLogger(true);
const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
await t.throwsAsync(
properties.loadPropertiesFromApi(
{
type: util.GitHubVariant.DOTCOM,
},
logger,
mockRepositoryNwo,
),
);
});
test("loadPropertiesFromApi throws if response data contains unexpected objects", async (t) => {
sinon.stub(api, "getRepositoryProperties").resolves({
headers: {},
status: 200,
url: "",
data: [{}],
});
const logger = getRunnerLogger(true);
const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
await t.throwsAsync(
properties.loadPropertiesFromApi(
{
type: util.GitHubVariant.DOTCOM,
},
logger,
mockRepositoryNwo,
),
);
});
test("loadPropertiesFromApi returns empty object if on GHES", async (t) => {
sinon.stub(api, "getRepositoryProperties").resolves({
headers: {},
status: 200,
url: "",
data: [
{ property_name: "github-codeql-extra-queries", value: "+queries" },
{ property_name: "unknown-property", value: "something" },
] satisfies properties.RepositoryProperty[],
});
const logger = getRunnerLogger(true);
const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
const response = await properties.loadPropertiesFromApi(
{
type: util.GitHubVariant.GHES,
version: "",
},
logger,
mockRepositoryNwo,
);
t.deepEqual(response, {});
});
test("loadPropertiesFromApi loads known properties", async (t) => {
sinon.stub(api, "getRepositoryProperties").resolves({
headers: {},
status: 200,
url: "",
data: [
{ property_name: "github-codeql-extra-queries", value: "+queries" },
{ property_name: "unknown-property", value: "something" },
] satisfies properties.RepositoryProperty[],
});
const logger = getRunnerLogger(true);
const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
const response = await properties.loadPropertiesFromApi(
{
type: util.GitHubVariant.DOTCOM,
},
logger,
mockRepositoryNwo,
);
t.deepEqual(response, { "github-codeql-extra-queries": "+queries" });
});
+94
View File
@@ -0,0 +1,94 @@
import { getRepositoryProperties } from "../api-client";
import { Logger } from "../logging";
import { RepositoryNwo } from "../repository";
import { GitHubVariant, GitHubVersion } from "../util";
/**
* Enumerates repository property names that have some meaning to us.
*/
export enum RepositoryPropertyName {
EXTRA_QUERIES = "github-codeql-extra-queries",
}
/**
* A repository property has a name and a value.
*/
export interface RepositoryProperty {
property_name: string;
value: string;
}
/**
* The API returns a list of `RepositoryProperty` objects.
*/
type GitHubPropertiesResponse = RepositoryProperty[];
/**
* A partial mapping from `RepositoryPropertyName` to values.
*/
export type RepositoryProperties = Partial<
Record<RepositoryPropertyName, string>
>;
/**
* Retrieves all known repository properties from the API.
*
* @param logger The logger to use.
* @param repositoryNwo Information about the repository for which to load properties.
* @returns Returns a partial mapping from `RepositoryPropertyName` to values.
*/
export async function loadPropertiesFromApi(
gitHubVersion: GitHubVersion,
logger: Logger,
repositoryNwo: RepositoryNwo,
): Promise<RepositoryProperties> {
// TODO: To be safe for now; later we should replace this with a version check once we know
// which version of GHES we expect this to be supported by.
if (gitHubVersion.type === GitHubVariant.GHES) {
return {};
}
try {
const response = await getRepositoryProperties(repositoryNwo);
const remoteProperties = response.data as GitHubPropertiesResponse;
if (!Array.isArray(remoteProperties)) {
throw new Error(
`Expected repository properties API to return an array, but got: ${JSON.stringify(response.data)}`,
);
}
logger.debug(
`Retrieved ${remoteProperties.length} repository properties: ${remoteProperties.map((p) => p.property_name).join(", ")}`,
);
const knownProperties = new Set(Object.values(RepositoryPropertyName));
const properties: RepositoryProperties = {};
for (const property of remoteProperties) {
if (property.property_name === undefined) {
throw new Error(
`Expected property object to have a 'property_name', but got: ${JSON.stringify(property)}`,
);
}
if (
knownProperties.has(property.property_name as RepositoryPropertyName)
) {
properties[property.property_name] = property.value;
}
}
logger.debug("Loaded the following values for the repository properties:");
for (const [property, value] of Object.entries(properties).sort(
([nameA], [nameB]) => nameA.localeCompare(nameB),
)) {
logger.debug(` ${property}: ${value}`);
}
return properties;
} catch (e) {
throw new Error(
`Encountered an error while trying to determine repository properties: ${e}`,
);
}
}
+10
View File
@@ -32,6 +32,7 @@ import {
} from "./diagnostics";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { loadPropertiesFromApi } from "./feature-flags/properties";
import {
checkInstallPython311,
checkPacksForOverlayCompatibility,
@@ -196,6 +197,14 @@ async function run() {
logger,
);
// Fetch the values of known repository properties that affect us.
const enableRepoProps = await features.getValue(
Feature.UseRepositoryProperties,
);
const repositoryProperties = enableRepoProps
? await loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo)
: {};
const jobRunUuid = uuidV4();
logger.info(`Job run UUID is ${jobRunUuid}.`);
core.exportVariable(EnvVar.JOB_RUN_UUID, jobRunUuid);
@@ -317,6 +326,7 @@ async function run() {
githubVersion: gitHubVersion,
apiDetails,
features,
repositoryProperties,
logger,
});
+1
View File
@@ -378,6 +378,7 @@ export function createTestConfig(overrides: Partial<Config>): Config {
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
} satisfies Config,
overrides,
);