Merge pull request #3564 from github/henrymercer/fix-database-upload-retries

Fix retries when uploading databases

CHANGELOG.md

@@ -6,6 +6,7 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
 
 - Fixed [a bug](https://github.com/github/codeql-action/issues/3555) which caused the CodeQL Action to fail loading repository properties if a "Multi select" repository property was configured for the repository. [#3557](https://github.com/github/codeql-action/pull/3557)
 - The CodeQL Action now loads [custom repository properties](https://docs.github.com/en/organizations/managing-organization-settings/managing-custom-properties-for-repositories-in-your-organization) on GitHub Enterprise Server, enabling the customization of features such as `github-codeql-disable-overlay` that was previously only available on GitHub.com. [#3559](https://github.com/github/codeql-action/pull/3559)
+- Fixed the retry mechanism for database uploads. Previously this would fail with the error "Response body object should not be disturbed or locked". [#3564](https://github.com/github/codeql-action/pull/3564)
 
 ## 4.32.6 - 05 Mar 2026
 

lib/analyze-action-post.js

@@ -161404,6 +161404,7 @@ retry.VERSION = VERSION7;
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -161418,10 +161419,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/analyze-action.js

@@ -106782,6 +106782,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -106796,10 +106797,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );
@@ -110954,13 +110952,6 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
   await withGroupAsync("Cleaning up databases", async () => {
     await codeql.databaseCleanupCluster(config, cleanupLevel);
   });
-  const client = getApiClient();
-  const uploadsUrl = new URL(parseGitHubUrl(apiDetails.url));
-  uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;
-  let uploadsBaseUrl = uploadsUrl.toString();
-  if (uploadsBaseUrl.endsWith("/")) {
-    uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
-  }
   const reports = [];
   for (const language of config.languages) {
     let bundledDbSize = void 0;
@@ -110969,40 +110960,47 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
         includeDiagnostics: false
       });
       bundledDbSize = fs13.statSync(bundledDb).size;
-      const bundledDbReadStream = fs13.createReadStream(bundledDb);
       const commitOid = await getCommitOid(
         getRequiredInput("checkout_path")
       );
-      try {
-        const startTime = performance.now();
-        await client.request(
-          `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
-          {
-            baseUrl: uploadsBaseUrl,
-            owner: repositoryNwo.owner,
-            repo: repositoryNwo.repo,
+      const maxAttempts = 4;
+      let uploadDurationMs;
+      for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+          uploadDurationMs = await uploadBundledDatabase(
+            repositoryNwo,
             language,
-            name: `${language}-database`,
-            commit_oid: commitOid,
-            data: bundledDbReadStream,
-            headers: {
-              authorization: `token ${apiDetails.auth}`,
-              "Content-Type": "application/zip",
-              "Content-Length": bundledDbSize
-            }
+            commitOid,
+            bundledDb,
+            bundledDbSize,
+            apiDetails
+          );
+          break;
+        } catch (e) {
+          const httpError = asHTTPError(e);
+          const isRetryable = !httpError || !DO_NOT_RETRY_STATUSES.includes(httpError.status);
+          if (!isRetryable) {
+            throw e;
+          } else if (attempt === maxAttempts) {
+            logger.error(
+              `Maximum retry attempts exhausted (${attempt}), aborting database upload`
+            );
+            throw e;
           }
-        );
-        const endTime = performance.now();
-        reports.push({
-          language,
-          zipped_upload_size_bytes: bundledDbSize,
-          is_overlay_base: shouldUploadOverlayBase,
-          upload_duration_ms: endTime - startTime
-        });
-        logger.debug(`Successfully uploaded database for ${language}`);
-      } finally {
-        bundledDbReadStream.close();
+          const backoffMs = 15e3 * Math.pow(2, attempt - 1);
+          logger.debug(
+            `Database upload attempt ${attempt} of ${maxAttempts} failed for ${language}: ${getErrorMessage(e)}. Retrying in ${backoffMs / 1e3}s...`
+          );
+          await new Promise((resolve8) => setTimeout(resolve8, backoffMs));
+        }
       }
+      reports.push({
+        language,
+        zipped_upload_size_bytes: bundledDbSize,
+        is_overlay_base: shouldUploadOverlayBase,
+        upload_duration_ms: uploadDurationMs
+      });
+      logger.debug(`Successfully uploaded database for ${language}`);
     } catch (e) {
       logger.warning(
         `Failed to upload database for ${language}: ${getErrorMessage(e)}`
@@ -111016,6 +111014,43 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
   }
   return reports;
 }
+async function uploadBundledDatabase(repositoryNwo, language, commitOid, bundledDb, bundledDbSize, apiDetails) {
+  const client = getApiClient();
+  const uploadsUrl = new URL(parseGitHubUrl(apiDetails.url));
+  uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;
+  let uploadsBaseUrl = uploadsUrl.toString();
+  if (uploadsBaseUrl.endsWith("/")) {
+    uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
+  }
+  const bundledDbReadStream = fs13.createReadStream(bundledDb);
+  try {
+    const startTime = performance.now();
+    await client.request(
+      `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
+      {
+        baseUrl: uploadsBaseUrl,
+        owner: repositoryNwo.owner,
+        repo: repositoryNwo.repo,
+        language,
+        name: `${language}-database`,
+        commit_oid: commitOid,
+        data: bundledDbReadStream,
+        headers: {
+          authorization: `token ${apiDetails.auth}`,
+          "Content-Type": "application/zip",
+          "Content-Length": bundledDbSize
+        },
+        // Disable `octokit/plugin-retry.js`, since the request body is a ReadStream which can only be consumed once.
+        request: {
+          retries: 0
+        }
+      }
+    );
+    return performance.now() - startTime;
+  } finally {
+    bundledDbReadStream.close();
+  }
+}
 
 // src/status-report.ts
 var os4 = __toESM(require("os"));

lib/autobuild-action.js

@@ -103423,6 +103423,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -103437,10 +103438,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/init-action-post.js

@@ -164652,6 +164652,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -164666,10 +164667,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/init-action.js

@@ -104131,6 +104131,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -104145,10 +104146,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/resolve-environment-action.js

@@ -103431,6 +103431,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -103445,10 +103446,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/setup-codeql-action.js

@@ -103540,6 +103540,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -103554,10 +103555,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/start-proxy-action-post.js

@@ -161287,6 +161287,7 @@ retry.VERSION = VERSION7;
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -161301,10 +161302,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/start-proxy-action.js

@@ -120510,6 +120510,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -120524,10 +120525,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/upload-lib.js

@@ -106416,6 +106416,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -106430,10 +106431,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/upload-sarif-action-post.js

@@ -161287,6 +161287,7 @@ retry.VERSION = VERSION7;
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -161301,10 +161302,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/upload-sarif-action.js

@@ -106465,6 +106465,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -106479,10 +106480,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

src/api-client.test.ts

@@ -5,6 +5,7 @@ import * as sinon from "sinon";
 
 import * as actionsUtil from "./actions-util";
 import * as api from "./api-client";
+import { DO_NOT_RETRY_STATUSES } from "./api-client";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";
 
@@ -37,7 +38,7 @@ test.serial("getApiClient", async (t) => {
       log: sinon.match.any,
       userAgent: `CodeQL-Action/${actionsUtil.getActionVersion()}`,
       retry: {
-        doNotRetry: [400, 410, 422, 451],
+        doNotRetry: DO_NOT_RETRY_STATUSES,
       },
     }),
   );

src/api-client.ts

@@ -19,6 +19,15 @@ import {
 
 const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
 
+/**
+ * HTTP status codes that should not be retried.
+ *
+ * The default Octokit list is 400, 401, 403, 404, 410, 422, and 451. We have
+ * observed transient errors with authentication, so we remove 401, 403, and 404
+ * from the default list to ensure that these errors are retried.
+ */
+export const DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
+
 export type GitHubApiCombinedDetails = GitHubApiDetails &
   GitHubApiExternalRepoDetails;
 
@@ -52,10 +61,7 @@ function createApiClientWithDetails(
         error: core.error,
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451],
+        doNotRetry: DO_NOT_RETRY_STATUSES,
       },
     }),
   );

src/database-upload.test.ts

@@ -15,6 +15,7 @@ import * as gitUtils from "./git-utils";
 import { KnownLanguage } from "./languages";
 import { RepositoryNwo } from "./repository";
 import {
+  checkExpectedLogMessages,
   createFeatures,
   createTestConfig,
   getRecordingLogger,
@@ -93,7 +94,7 @@ test.serial(
         .returns("false");
       sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
 
-      const loggedMessages = [];
+      const loggedMessages: LoggedMessage[] = [];
       await cleanupAndUploadDatabases(
         testRepoName,
         getCodeQL(),
@@ -102,14 +103,9 @@ test.serial(
         createFeatures([]),
         getRecordingLogger(loggedMessages),
       );
-      t.assert(
-        loggedMessages.find(
-          (v: LoggedMessage) =>
-            v.type === "debug" &&
-            v.message ===
-              "Database upload disabled in workflow. Skipping upload.",
-        ) !== undefined,
-      );
+      checkExpectedLogMessages(t, loggedMessages, [
+        "Database upload disabled in workflow. Skipping upload.",
+      ]);
     });
   },
 );
@@ -127,7 +123,7 @@ test.serial(
 
       await mockHttpRequests(201);
 
-      const loggedMessages = [];
+      const loggedMessages: LoggedMessage[] = [];
       await cleanupAndUploadDatabases(
         testRepoName,
         getCodeQL(),
@@ -139,14 +135,9 @@ test.serial(
         createFeatures([]),
         getRecordingLogger(loggedMessages),
       );
-      t.assert(
-        loggedMessages.find(
-          (v: LoggedMessage) =>
-            v.type === "debug" &&
-            v.message ===
-              "Not uploading database because 'analysis-kinds: code-scanning' is not enabled.",
-        ) !== undefined,
-      );
+      checkExpectedLogMessages(t, loggedMessages, [
+        "Not uploading database because 'analysis-kinds: code-scanning' is not enabled.",
+      ]);
     });
   },
 );
@@ -163,7 +154,7 @@ test.serial("Abort database upload if running against GHES", async (t) => {
     const config = getTestConfig(tmpDir);
     config.gitHubVersion = { type: GitHubVariant.GHES, version: "3.0" };
 
-    const loggedMessages = [];
+    const loggedMessages: LoggedMessage[] = [];
     await cleanupAndUploadDatabases(
       testRepoName,
       getCodeQL(),
@@ -172,14 +163,9 @@ test.serial("Abort database upload if running against GHES", async (t) => {
       createFeatures([]),
       getRecordingLogger(loggedMessages),
     );
-    t.assert(
-      loggedMessages.find(
-        (v: LoggedMessage) =>
-          v.type === "debug" &&
-          v.message ===
-            "Not running against github.com or GHEC-DR. Skipping upload.",
-      ) !== undefined,
-    );
+    checkExpectedLogMessages(t, loggedMessages, [
+      "Not running against github.com or GHEC-DR. Skipping upload.",
+    ]);
   });
 });
 
@@ -194,7 +180,7 @@ test.serial(
         .returns("true");
       sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(false);
 
-      const loggedMessages = [];
+      const loggedMessages: LoggedMessage[] = [];
       await cleanupAndUploadDatabases(
         testRepoName,
         getCodeQL(),
@@ -203,48 +189,90 @@ test.serial(
         createFeatures([]),
         getRecordingLogger(loggedMessages),
       );
-      t.assert(
-        loggedMessages.find(
-          (v: LoggedMessage) =>
-            v.type === "debug" &&
-            v.message === "Not analyzing default branch. Skipping upload.",
-        ) !== undefined,
-      );
+      checkExpectedLogMessages(t, loggedMessages, [
+        "Not analyzing default branch. Skipping upload.",
+      ]);
     });
   },
 );
 
-test.serial("Don't crash if uploading a database fails", async (t) => {
-  await withTmpDir(async (tmpDir) => {
-    setupActionsVars(tmpDir, tmpDir);
-    sinon
-      .stub(actionsUtil, "getRequiredInput")
-      .withArgs("upload-database")
-      .returns("true");
-    sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
+test.serial(
+  "Don't crash if uploading a database fails with a non-retryable error",
+  async (t) => {
+    await withTmpDir(async (tmpDir) => {
+      setupActionsVars(tmpDir, tmpDir);
+      sinon
+        .stub(actionsUtil, "getRequiredInput")
+        .withArgs("upload-database")
+        .returns("true");
+      sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
 
-    await mockHttpRequests(500);
+      const databaseUploadSpy = await mockHttpRequests(422);
 
-    const loggedMessages = [] as LoggedMessage[];
-    await cleanupAndUploadDatabases(
-      testRepoName,
-      getCodeQL(),
-      getTestConfig(tmpDir),
-      testApiDetails,
-      createFeatures([]),
-      getRecordingLogger(loggedMessages),
-    );
+      const loggedMessages: LoggedMessage[] = [];
+      await cleanupAndUploadDatabases(
+        testRepoName,
+        getCodeQL(),
+        getTestConfig(tmpDir),
+        testApiDetails,
+        createFeatures([]),
+        getRecordingLogger(loggedMessages),
+      );
 
-    t.assert(
-      loggedMessages.find(
-        (v) =>
-          v.type === "warning" &&
-          v.message ===
-            "Failed to upload database for javascript: some error message",
-      ) !== undefined,
-    );
-  });
-});
+      checkExpectedLogMessages(t, loggedMessages, [
+        "Failed to upload database for javascript: some error message",
+      ]);
+
+      // Non-retryable errors should not be retried.
+      t.is(databaseUploadSpy.callCount, 1);
+    });
+  },
+);
+
+test.serial(
+  "Don't crash if uploading a database fails with a retryable error",
+  async (t) => {
+    await withTmpDir(async (tmpDir) => {
+      setupActionsVars(tmpDir, tmpDir);
+      sinon
+        .stub(actionsUtil, "getRequiredInput")
+        .withArgs("upload-database")
+        .returns("true");
+      sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
+
+      const databaseUploadSpy = await mockHttpRequests(500);
+
+      // Stub setTimeout to fire immediately to avoid real delays from retry backoff.
+      const originalSetTimeout = global.setTimeout;
+      const setTimeoutStub = sinon
+        .stub(global, "setTimeout")
+        .callsFake((fn: () => void) => originalSetTimeout(fn, 0));
+
+      const loggedMessages: LoggedMessage[] = [];
+      await cleanupAndUploadDatabases(
+        testRepoName,
+        getCodeQL(),
+        getTestConfig(tmpDir),
+        testApiDetails,
+        createFeatures([]),
+        getRecordingLogger(loggedMessages),
+      );
+
+      checkExpectedLogMessages(t, loggedMessages, [
+        "Failed to upload database for javascript: some error message",
+      ]);
+
+      // Retryable errors should be retried the expected number of times.
+      t.is(databaseUploadSpy.callCount, 4);
+
+      // setTimeout should have been called with the expected backoff delays.
+      const setTimeoutDelays = setTimeoutStub.args.map(
+        (args) => args[1] as number,
+      );
+      t.deepEqual(setTimeoutDelays, [15_000, 30_000, 60_000]);
+    });
+  },
+);
 
 test.serial("Successfully uploading a database to github.com", async (t) => {
   await withTmpDir(async (tmpDir) => {
@@ -257,7 +285,7 @@ test.serial("Successfully uploading a database to github.com", async (t) => {
 
     await mockHttpRequests(201);
 
-    const loggedMessages = [] as LoggedMessage[];
+    const loggedMessages: LoggedMessage[] = [];
     await cleanupAndUploadDatabases(
       testRepoName,
       getCodeQL(),
@@ -266,13 +294,9 @@ test.serial("Successfully uploading a database to github.com", async (t) => {
       createFeatures([]),
       getRecordingLogger(loggedMessages),
     );
-    t.assert(
-      loggedMessages.find(
-        (v) =>
-          v.type === "debug" &&
-          v.message === "Successfully uploaded database for javascript",
-      ) !== undefined,
-    );
+    checkExpectedLogMessages(t, loggedMessages, [
+      "Successfully uploaded database for javascript",
+    ]);
   });
 });
 
@@ -287,7 +311,7 @@ test.serial("Successfully uploading a database to GHEC-DR", async (t) => {
 
     const databaseUploadSpy = await mockHttpRequests(201);
 
-    const loggedMessages = [] as LoggedMessage[];
+    const loggedMessages: LoggedMessage[] = [];
     await cleanupAndUploadDatabases(
       testRepoName,
       getCodeQL(),
@@ -300,13 +324,9 @@ test.serial("Successfully uploading a database to GHEC-DR", async (t) => {
       createFeatures([]),
       getRecordingLogger(loggedMessages),
     );
-    t.assert(
-      loggedMessages.find(
-        (v) =>
-          v.type === "debug" &&
-          v.message === "Successfully uploaded database for javascript",
-      ) !== undefined,
-    );
+    checkExpectedLogMessages(t, loggedMessages, [
+      "Successfully uploaded database for javascript",
+    ]);
     t.assert(
       databaseUploadSpy.calledOnceWith(
         sinon.match.string,

src/database-upload.ts

@@ -2,7 +2,11 @@ import * as fs from "fs";
 
 import * as actionsUtil from "./actions-util";
 import { AnalysisKind } from "./analyses";
-import { getApiClient, GitHubApiDetails } from "./api-client";
+import {
+  DO_NOT_RETRY_STATUSES,
+  getApiClient,
+  GitHubApiDetails,
+} from "./api-client";
 import { type CodeQL } from "./codeql";
 import { Config } from "./config-utils";
 import { Feature, FeatureEnablement } from "./feature-flags";
@@ -11,7 +15,7 @@ import { Logger, withGroupAsync } from "./logging";
 import { OverlayDatabaseMode } from "./overlay";
 import { RepositoryNwo } from "./repository";
 import * as util from "./util";
-import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
+import { asHTTPError, bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
 
 /** Information about a database upload. */
 export interface DatabaseUploadResult {
@@ -81,18 +85,6 @@ export async function cleanupAndUploadDatabases(
     await codeql.databaseCleanupCluster(config, cleanupLevel);
   });
 
-  const client = getApiClient();
-
-  const uploadsUrl = new URL(parseGitHubUrl(apiDetails.url));
-  uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;
-
-  // Octokit expects the baseUrl to not have a trailing slash,
-  // but it is included by default in a URL.
-  let uploadsBaseUrl = uploadsUrl.toString();
-  if (uploadsBaseUrl.endsWith("/")) {
-    uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
-  }
-
   const reports: DatabaseUploadResult[] = [];
   for (const language of config.languages) {
     let bundledDbSize: number | undefined = undefined;
@@ -105,40 +97,51 @@ export async function cleanupAndUploadDatabases(
         includeDiagnostics: false,
       });
       bundledDbSize = fs.statSync(bundledDb).size;
-      const bundledDbReadStream = fs.createReadStream(bundledDb);
       const commitOid = await gitUtils.getCommitOid(
         actionsUtil.getRequiredInput("checkout_path"),
       );
-      try {
-        const startTime = performance.now();
-        await client.request(
-          `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
-          {
-            baseUrl: uploadsBaseUrl,
-            owner: repositoryNwo.owner,
-            repo: repositoryNwo.repo,
+      // Upload with manual retry logic. We disable Octokit's built-in retries
+      // because the request body is a ReadStream, which can only be consumed
+      // once.
+      const maxAttempts = 4; // 1 initial attempt + 3 retries, identical to the default retry behavior of Octokit
+      let uploadDurationMs: number | undefined;
+      for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+          uploadDurationMs = await uploadBundledDatabase(
+            repositoryNwo,
             language,
-            name: `${language}-database`,
-            commit_oid: commitOid,
-            data: bundledDbReadStream,
-            headers: {
-              authorization: `token ${apiDetails.auth}`,
-              "Content-Type": "application/zip",
-              "Content-Length": bundledDbSize,
-            },
-          },
-        );
-        const endTime = performance.now();
-        reports.push({
-          language,
-          zipped_upload_size_bytes: bundledDbSize,
-          is_overlay_base: shouldUploadOverlayBase,
-          upload_duration_ms: endTime - startTime,
-        });
-        logger.debug(`Successfully uploaded database for ${language}`);
-      } finally {
-        bundledDbReadStream.close();
+            commitOid,
+            bundledDb,
+            bundledDbSize,
+            apiDetails,
+          );
+          break;
+        } catch (e) {
+          const httpError = asHTTPError(e);
+          const isRetryable =
+            !httpError || !DO_NOT_RETRY_STATUSES.includes(httpError.status);
+          if (!isRetryable) {
+            throw e;
+          } else if (attempt === maxAttempts) {
+            logger.error(
+              `Maximum retry attempts exhausted (${attempt}), aborting database upload`,
+            );
+            throw e;
+          }
+          const backoffMs = 15_000 * Math.pow(2, attempt - 1); // 15s, 30s, 60s
+          logger.debug(
+            `Database upload attempt ${attempt} of ${maxAttempts} failed for ${language}: ${util.getErrorMessage(e)}. Retrying in ${backoffMs / 1000}s...`,
+          );
+          await new Promise((resolve) => setTimeout(resolve, backoffMs));
+        }
       }
+      reports.push({
+        language,
+        zipped_upload_size_bytes: bundledDbSize,
+        is_overlay_base: shouldUploadOverlayBase,
+        upload_duration_ms: uploadDurationMs,
+      });
+      logger.debug(`Successfully uploaded database for ${language}`);
     } catch (e) {
       // Log a warning but don't fail the workflow
       logger.warning(
@@ -155,3 +158,58 @@ export async function cleanupAndUploadDatabases(
   }
   return reports;
 }
+
+/**
+ * Uploads a bundled database to the GitHub API.
+ *
+ * @returns the duration of the upload in milliseconds
+ */
+async function uploadBundledDatabase(
+  repositoryNwo: RepositoryNwo,
+  language: string,
+  commitOid: string,
+  bundledDb: string,
+  bundledDbSize: number,
+  apiDetails: GitHubApiDetails,
+): Promise<number> {
+  const client = getApiClient();
+
+  const uploadsUrl = new URL(parseGitHubUrl(apiDetails.url));
+  uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;
+
+  // Octokit expects the baseUrl to not have a trailing slash,
+  // but it is included by default in a URL.
+  let uploadsBaseUrl = uploadsUrl.toString();
+  if (uploadsBaseUrl.endsWith("/")) {
+    uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
+  }
+
+  const bundledDbReadStream = fs.createReadStream(bundledDb);
+  try {
+    const startTime = performance.now();
+    await client.request(
+      `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
+      {
+        baseUrl: uploadsBaseUrl,
+        owner: repositoryNwo.owner,
+        repo: repositoryNwo.repo,
+        language,
+        name: `${language}-database`,
+        commit_oid: commitOid,
+        data: bundledDbReadStream,
+        headers: {
+          authorization: `token ${apiDetails.auth}`,
+          "Content-Type": "application/zip",
+          "Content-Length": bundledDbSize,
+        },
+        // Disable `octokit/plugin-retry.js`, since the request body is a ReadStream which can only be consumed once.
+        request: {
+          retries: 0,
+        },
+      },
+    );
+    return performance.now() - startTime;
+  } finally {
+    bundledDbReadStream.close();
+  }
+}