Compare commits

...

17 Commits

14 changed files with 794 additions and 171 deletions

View File

@ -4,51 +4,130 @@ on:
pull_request:
branches:
- master
- releases/**
paths-ignore:
- '**.md'
push:
branches:
- master
- releases/**
paths-ignore:
- '**.md'
jobs:
test:
name: Test on ${{ matrix.os }}
# Build and unit test
build:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- uses: actions/setup-node@v1
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: '12.x'
- name: Get npm cache directory
- name: Determine npm cache directory
id: npm-cache
run: |
echo "::set-output name=dir::$(npm config get cache)"
- uses: actions/cache@v1
- name: Restore npm cache
uses: actions/cache@v1
with:
path: ${{ steps.npm-cache.outputs.dir }}
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- run: npm ci
- name: Prettier Format Check
run: npm run format-check
- name: ESLint Check
run: npm run lint
- name: Build & Test
run: npm run test
# End to end save and restore
test-save:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Generate files
shell: bash
run: __tests__/create-cache-files.sh ${{ runner.os }}
- name: Save cache
uses: ./
with:
key: test-${{ runner.os }}-${{ github.run_id }}
path: test-cache
test-restore:
needs: test-save
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Restore cache
uses: ./
with:
key: test-${{ runner.os }}-${{ github.run_id }}
path: test-cache
- name: Verify cache
shell: bash
run: __tests__/verify-cache-files.sh ${{ runner.os }}
# End to end with proxy
test-proxy-save:
runs-on: ubuntu-latest
container:
image: ubuntu:latest
options: --dns 127.0.0.1
services:
squid-proxy:
image: datadog/squid:latest
ports:
- 3128:3128
env:
https_proxy: http://squid-proxy:3128
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Generate files
run: __tests__/create-cache-files.sh proxy
- name: Save cache
uses: ./
with:
key: test-proxy-${{ github.run_id }}
path: test-cache
test-proxy-restore:
needs: test-proxy-save
runs-on: ubuntu-latest
container:
image: ubuntu:latest
options: --dns 127.0.0.1
services:
squid-proxy:
image: datadog/squid:latest
ports:
- 3128:3128
env:
https_proxy: http://squid-proxy:3128
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Restore cache
uses: ./
with:
key: test-proxy-${{ github.run_id }}
path: test-cache
- name: Verify cache
run: __tests__/verify-cache-files.sh proxy

View File

@ -80,7 +80,7 @@ See [Examples](examples.md) for a list of `actions/cache` implementations for us
## Cache Limits
A repository can have up to 2GB of caches. Once the 2GB limit is reached, older caches will be evicted based on when the cache was last accessed. Caches that are not accessed within the last week will also be evicted.
A repository can have up to 5GB of caches. Once the 5GB limit is reached, older caches will be evicted based on when the cache was last accessed. Caches that are not accessed within the last week will also be evicted.
## Skipping steps based on cache-hit

View File

@ -0,0 +1,144 @@
import { retry } from "../src/cacheHttpClient";
import * as testUtils from "../src/utils/testUtils";
afterEach(() => {
testUtils.clearInputs();
});
interface TestResponse {
statusCode: number;
result: string | null;
}
function handleResponse(
response: TestResponse | undefined
): Promise<TestResponse> {
if (!response) {
fail("Retry method called too many times");
}
if (response.statusCode === 999) {
throw Error("Test Error");
} else {
return Promise.resolve(response);
}
}
async function testRetryExpectingResult(
responses: Array<TestResponse>,
expectedResult: string | null
): Promise<void> {
responses = responses.reverse(); // Reverse responses since we pop from end
const actualResult = await retry(
"test",
() => handleResponse(responses.pop()),
(response: TestResponse) => response.statusCode
);
expect(actualResult.result).toEqual(expectedResult);
}
async function testRetryExpectingError(
responses: Array<TestResponse>
): Promise<void> {
responses = responses.reverse(); // Reverse responses since we pop from end
expect(
retry(
"test",
() => handleResponse(responses.pop()),
(response: TestResponse) => response.statusCode
)
).rejects.toBeInstanceOf(Error);
}
test("retry works on successful response", async () => {
await testRetryExpectingResult(
[
{
statusCode: 200,
result: "Ok"
}
],
"Ok"
);
});
test("retry works after retryable status code", async () => {
await testRetryExpectingResult(
[
{
statusCode: 503,
result: null
},
{
statusCode: 200,
result: "Ok"
}
],
"Ok"
);
});
test("retry fails after exhausting retries", async () => {
await testRetryExpectingError([
{
statusCode: 503,
result: null
},
{
statusCode: 503,
result: null
},
{
statusCode: 200,
result: "Ok"
}
]);
});
test("retry fails after non-retryable status code", async () => {
await testRetryExpectingError([
{
statusCode: 500,
result: null
},
{
statusCode: 200,
result: "Ok"
}
]);
});
test("retry works after error", async () => {
await testRetryExpectingResult(
[
{
statusCode: 999,
result: null
},
{
statusCode: 200,
result: "Ok"
}
],
"Ok"
);
});
test("retry returns after client error", async () => {
await testRetryExpectingResult(
[
{
statusCode: 400,
result: null
},
{
statusCode: 200,
result: "Ok"
}
],
null
);
});

11
__tests__/create-cache-files.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/sh
# Validate args
prefix="$1"
if [ -z "$prefix" ]; then
echo "Must supply prefix argument"
exit 1
fi
mkdir test-cache
echo "$prefix $GITHUB_RUN_ID" > test-cache/test-file.txt

View File

@ -2,6 +2,8 @@ import * as exec from "@actions/exec";
import * as io from "@actions/io";
import * as tar from "../src/tar";
import fs = require("fs");
jest.mock("@actions/exec");
jest.mock("@actions/io");
@ -11,17 +13,19 @@ beforeAll(() => {
});
});
test("extract tar", async () => {
test("extract BSD tar", async () => {
const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const archivePath = "cache.tar";
const IS_WINDOWS = process.platform === "win32";
const archivePath = IS_WINDOWS
? `${process.env["windir"]}\\fakepath\\cache.tar`
: "cache.tar";
const targetDirectory = "~/.npm/cache";
await tar.extractTar(archivePath, targetDirectory);
expect(mkdirMock).toHaveBeenCalledWith(targetDirectory);
const IS_WINDOWS = process.platform === "win32";
const tarPath = IS_WINDOWS
? `${process.env["windir"]}\\System32\\tar.exe`
: "tar";
@ -29,13 +33,37 @@ test("extract tar", async () => {
expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [
"-xz",
"-f",
archivePath,
IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath,
"-C",
targetDirectory
IS_WINDOWS ? targetDirectory?.replace(/\\/g, "/") : targetDirectory
]);
});
test("create tar", async () => {
test("extract GNU tar", async () => {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
jest.spyOn(fs, "existsSync").mockReturnValueOnce(false);
jest.spyOn(tar, "isGnuTar").mockReturnValue(Promise.resolve(true));
const execMock = jest.spyOn(exec, "exec");
const archivePath = `${process.env["windir"]}\\fakepath\\cache.tar`;
const targetDirectory = "~/.npm/cache";
await tar.extractTar(archivePath, targetDirectory);
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenLastCalledWith(`"tar"`, [
"-xz",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
targetDirectory?.replace(/\\/g, "/"),
"--force-local"
]);
}
});
test("create BSD tar", async () => {
const execMock = jest.spyOn(exec, "exec");
const archivePath = "cache.tar";
@ -50,9 +78,9 @@ test("create tar", async () => {
expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [
"-cz",
"-f",
archivePath,
IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath,
"-C",
sourceDirectory,
IS_WINDOWS ? sourceDirectory?.replace(/\\/g, "/") : sourceDirectory,
"."
]);
});

30
__tests__/verify-cache-files.sh Executable file
View File

@ -0,0 +1,30 @@
#!/bin/sh
# Validate args
prefix="$1"
if [ -z "$prefix" ]; then
echo "Must supply prefix argument"
exit 1
fi
# Sanity check GITHUB_RUN_ID defined
if [ -z "$GITHUB_RUN_ID" ]; then
echo "GITHUB_RUN_ID not defined"
exit 1
fi
# Verify file exists
file="test-cache/test-file.txt"
echo "Checking for $file"
if [ ! -e $file ]; then
echo "File does not exist"
exit 1
fi
# Verify file content
content="$(cat $file)"
echo "File content:\n$content"
if [ -z "$(echo $content | grep --fixed-strings "$prefix $GITHUB_RUN_ID")" ]; then
echo "Unexpected file content"
exit 1
fi

View File

@ -1,4 +1,4 @@
name: 'Cache Artifacts'
name: 'Cache'
description: 'Cache artifacts like dependencies and build outputs to improve workflow execution time'
author: 'GitHub'
inputs:

181
dist/restore/index.js vendored
View File

@ -1252,9 +1252,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const fs = __importStar(__webpack_require__(747));
const auth_1 = __webpack_require__(226);
const http_client_1 = __webpack_require__(539);
const auth_1 = __webpack_require__(226);
const fs = __importStar(__webpack_require__(747));
const stream = __importStar(__webpack_require__(794));
const util = __importStar(__webpack_require__(669));
const constants_1 = __webpack_require__(694);
const utils = __importStar(__webpack_require__(443));
function isSuccessStatusCode(statusCode) {
if (!statusCode) {
@ -1262,6 +1265,12 @@ function isSuccessStatusCode(statusCode) {
}
return statusCode >= 200 && statusCode < 300;
}
function isServerErrorStatusCode(statusCode) {
if (!statusCode) {
return true;
}
return statusCode >= 500;
}
function isRetryableStatusCode(statusCode) {
if (!statusCode) {
return false;
@ -1301,12 +1310,56 @@ function createHttpClient() {
const bearerCredentialHandler = new auth_1.BearerCredentialHandler(token);
return new http_client_1.HttpClient("actions/cache", [bearerCredentialHandler], getRequestOptions());
}
function retry(name, method, getStatusCode, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
let response = undefined;
let statusCode = undefined;
let isRetryable = false;
let errorMessage = "";
let attempt = 1;
while (attempt <= maxAttempts) {
try {
response = yield method();
statusCode = getStatusCode(response);
if (!isServerErrorStatusCode(statusCode)) {
return response;
}
isRetryable = isRetryableStatusCode(statusCode);
errorMessage = `Cache service responded with ${statusCode}`;
}
catch (error) {
isRetryable = true;
errorMessage = error.message;
}
core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`);
if (!isRetryable) {
core.debug(`${name} - Error is not retryable`);
break;
}
attempt++;
}
throw Error(`${name} failed: ${errorMessage}`);
});
}
exports.retry = retry;
function retryTypedResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.statusCode, maxAttempts);
});
}
exports.retryTypedResponse = retryTypedResponse;
function retryHttpClientResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.message.statusCode, maxAttempts);
});
}
exports.retryHttpClientResponse = retryHttpClientResponse;
function getCacheEntry(keys) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const httpClient = createHttpClient();
const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
const response = yield httpClient.getJson(getCacheApiUrl(resource));
const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource)));
if (response.statusCode === 204) {
return null;
}
@ -1325,21 +1378,35 @@ function getCacheEntry(keys) {
});
}
exports.getCacheEntry = getCacheEntry;
function pipeResponseToStream(response, stream) {
function pipeResponseToStream(response, output) {
return __awaiter(this, void 0, void 0, function* () {
return new Promise(resolve => {
response.message.pipe(stream).on("close", () => {
resolve();
});
});
const pipeline = util.promisify(stream.pipeline);
yield pipeline(response.message, output);
});
}
function downloadCache(archiveLocation, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createWriteStream(archivePath);
const httpClient = new http_client_1.HttpClient("actions/cache");
const downloadResponse = yield httpClient.get(archiveLocation);
const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation));
// Abort download if no traffic received over the socket.
downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => {
downloadResponse.message.destroy();
core.debug(`Aborting download, socket timed out after ${constants_1.SocketTimeout} ms`);
});
yield pipeResponseToStream(downloadResponse, stream);
// Validate download size.
const contentLengthHeader = downloadResponse.message.headers["content-length"];
if (contentLengthHeader) {
const expectedLength = parseInt(contentLengthHeader);
const actualLength = utils.getArchiveFileSize(archivePath);
if (actualLength != expectedLength) {
throw new Error(`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`);
}
}
else {
core.debug("Unable to validate download, no Content-Length header");
}
});
}
exports.downloadCache = downloadCache;
@ -1351,7 +1418,7 @@ function reserveCache(key) {
const reserveCacheRequest = {
key
};
const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest);
const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest));
return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1);
});
}
@ -1364,7 +1431,7 @@ function getContentRange(start, end) {
// Content-Range: bytes 0-199/*
return `bytes ${start}-${end}/*`;
}
function uploadChunk(httpClient, resourceUrl, data, start, end) {
function uploadChunk(httpClient, resourceUrl, openStream, start, end) {
return __awaiter(this, void 0, void 0, function* () {
core.debug(`Uploading chunk of size ${end -
start +
@ -1373,21 +1440,7 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) {
"Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end)
};
const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () {
return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders);
});
const response = yield uploadChunkRequest();
if (isSuccessStatusCode(response.message.statusCode)) {
return;
}
if (isRetryableStatusCode(response.message.statusCode)) {
core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`);
const retryResponse = yield uploadChunkRequest();
if (isSuccessStatusCode(retryResponse.message.statusCode)) {
return;
}
}
throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`);
yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, () => httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders));
});
}
function parseEnvNumber(key) {
@ -1417,13 +1470,16 @@ function uploadFile(httpClient, cacheId, archivePath) {
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
yield uploadChunk(httpClient, resourceUrl, () => fs
.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
yield uploadChunk(httpClient, resourceUrl, chunk, start, end);
})
.on("error", error => {
throw new Error(`Cache upload failed because file read failed with ${error.Message}`);
}), start, end);
}
})));
}
@ -1436,7 +1492,7 @@ function uploadFile(httpClient, cacheId, archivePath) {
function commitCache(httpClient, cacheId, filesize) {
return __awaiter(this, void 0, void 0, function* () {
const commitCacheRequest = { size: filesize };
return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest);
return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest));
});
}
function saveCache(cacheId, archivePath) {
@ -2721,6 +2777,10 @@ var Events;
Events["Push"] = "push";
Events["PullRequest"] = "pull_request";
})(Events = exports.Events || (exports.Events = {}));
// Socket timeout in milliseconds during download. If no traffic is received
// over the socket during this period, the socket is destroyed and the download
// is aborted.
exports.SocketTimeout = 5000;
/***/ }),
@ -2861,6 +2921,13 @@ run();
exports.default = run;
/***/ }),
/***/ 794:
/***/ (function(module) {
module.exports = require("stream");
/***/ }),
/***/ 826:
@ -2928,10 +2995,30 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const exec_1 = __webpack_require__(986);
const io = __importStar(__webpack_require__(1));
const fs_1 = __webpack_require__(747);
function getTarPath() {
const path = __importStar(__webpack_require__(622));
const tar = __importStar(__webpack_require__(943));
function isGnuTar() {
return __awaiter(this, void 0, void 0, function* () {
core.debug("Checking tar --version");
let versionOutput = "";
yield exec_1.exec("tar --version", [], {
ignoreReturnCode: true,
silent: true,
listeners: {
stdout: (data) => (versionOutput += data.toString()),
stderr: (data) => (versionOutput += data.toString())
}
});
core.debug(versionOutput.trim());
return versionOutput.toUpperCase().includes("GNU TAR");
});
}
exports.isGnuTar = isGnuTar;
function getTarPath(args) {
return __awaiter(this, void 0, void 0, function* () {
// Explicitly use BSD Tar on Windows
const IS_WINDOWS = process.platform === "win32";
@ -2940,22 +3027,21 @@ function getTarPath() {
if (fs_1.existsSync(systemTar)) {
return systemTar;
}
else if (yield tar.isGnuTar()) {
args.push("--force-local");
}
}
return yield io.which("tar", true);
});
}
function execTar(args) {
var _a, _b;
var _a;
return __awaiter(this, void 0, void 0, function* () {
try {
yield exec_1.exec(`"${yield getTarPath()}"`, args);
yield exec_1.exec(`"${yield getTarPath(args)}"`, args);
}
catch (error) {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`);
}
throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`);
throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`);
}
});
}
@ -2963,14 +3049,27 @@ function extractTar(archivePath, targetDirectory) {
return __awaiter(this, void 0, void 0, function* () {
// Create directory to extract tar into
yield io.mkdirP(targetDirectory);
const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
const args = [
"-xz",
"-f",
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
"-C",
targetDirectory.replace(new RegExp("\\" + path.sep, "g"), "/")
];
yield execTar(args);
});
}
exports.extractTar = extractTar;
function createTar(archivePath, sourceDirectory) {
return __awaiter(this, void 0, void 0, function* () {
const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
const args = [
"-cz",
"-f",
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
"-C",
sourceDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"),
"."
];
yield execTar(args);
});
}

181
dist/save/index.js vendored
View File

@ -1252,9 +1252,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const fs = __importStar(__webpack_require__(747));
const auth_1 = __webpack_require__(226);
const http_client_1 = __webpack_require__(539);
const auth_1 = __webpack_require__(226);
const fs = __importStar(__webpack_require__(747));
const stream = __importStar(__webpack_require__(794));
const util = __importStar(__webpack_require__(669));
const constants_1 = __webpack_require__(694);
const utils = __importStar(__webpack_require__(443));
function isSuccessStatusCode(statusCode) {
if (!statusCode) {
@ -1262,6 +1265,12 @@ function isSuccessStatusCode(statusCode) {
}
return statusCode >= 200 && statusCode < 300;
}
function isServerErrorStatusCode(statusCode) {
if (!statusCode) {
return true;
}
return statusCode >= 500;
}
function isRetryableStatusCode(statusCode) {
if (!statusCode) {
return false;
@ -1301,12 +1310,56 @@ function createHttpClient() {
const bearerCredentialHandler = new auth_1.BearerCredentialHandler(token);
return new http_client_1.HttpClient("actions/cache", [bearerCredentialHandler], getRequestOptions());
}
function retry(name, method, getStatusCode, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
let response = undefined;
let statusCode = undefined;
let isRetryable = false;
let errorMessage = "";
let attempt = 1;
while (attempt <= maxAttempts) {
try {
response = yield method();
statusCode = getStatusCode(response);
if (!isServerErrorStatusCode(statusCode)) {
return response;
}
isRetryable = isRetryableStatusCode(statusCode);
errorMessage = `Cache service responded with ${statusCode}`;
}
catch (error) {
isRetryable = true;
errorMessage = error.message;
}
core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`);
if (!isRetryable) {
core.debug(`${name} - Error is not retryable`);
break;
}
attempt++;
}
throw Error(`${name} failed: ${errorMessage}`);
});
}
exports.retry = retry;
function retryTypedResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.statusCode, maxAttempts);
});
}
exports.retryTypedResponse = retryTypedResponse;
function retryHttpClientResponse(name, method, maxAttempts = 2) {
return __awaiter(this, void 0, void 0, function* () {
return yield retry(name, method, (response) => response.message.statusCode, maxAttempts);
});
}
exports.retryHttpClientResponse = retryHttpClientResponse;
function getCacheEntry(keys) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const httpClient = createHttpClient();
const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
const response = yield httpClient.getJson(getCacheApiUrl(resource));
const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource)));
if (response.statusCode === 204) {
return null;
}
@ -1325,21 +1378,35 @@ function getCacheEntry(keys) {
});
}
exports.getCacheEntry = getCacheEntry;
function pipeResponseToStream(response, stream) {
function pipeResponseToStream(response, output) {
return __awaiter(this, void 0, void 0, function* () {
return new Promise(resolve => {
response.message.pipe(stream).on("close", () => {
resolve();
});
});
const pipeline = util.promisify(stream.pipeline);
yield pipeline(response.message, output);
});
}
function downloadCache(archiveLocation, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createWriteStream(archivePath);
const httpClient = new http_client_1.HttpClient("actions/cache");
const downloadResponse = yield httpClient.get(archiveLocation);
const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation));
// Abort download if no traffic received over the socket.
downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => {
downloadResponse.message.destroy();
core.debug(`Aborting download, socket timed out after ${constants_1.SocketTimeout} ms`);
});
yield pipeResponseToStream(downloadResponse, stream);
// Validate download size.
const contentLengthHeader = downloadResponse.message.headers["content-length"];
if (contentLengthHeader) {
const expectedLength = parseInt(contentLengthHeader);
const actualLength = utils.getArchiveFileSize(archivePath);
if (actualLength != expectedLength) {
throw new Error(`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`);
}
}
else {
core.debug("Unable to validate download, no Content-Length header");
}
});
}
exports.downloadCache = downloadCache;
@ -1351,7 +1418,7 @@ function reserveCache(key) {
const reserveCacheRequest = {
key
};
const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest);
const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest));
return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1);
});
}
@ -1364,7 +1431,7 @@ function getContentRange(start, end) {
// Content-Range: bytes 0-199/*
return `bytes ${start}-${end}/*`;
}
function uploadChunk(httpClient, resourceUrl, data, start, end) {
function uploadChunk(httpClient, resourceUrl, openStream, start, end) {
return __awaiter(this, void 0, void 0, function* () {
core.debug(`Uploading chunk of size ${end -
start +
@ -1373,21 +1440,7 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) {
"Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end)
};
const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () {
return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders);
});
const response = yield uploadChunkRequest();
if (isSuccessStatusCode(response.message.statusCode)) {
return;
}
if (isRetryableStatusCode(response.message.statusCode)) {
core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`);
const retryResponse = yield uploadChunkRequest();
if (isSuccessStatusCode(retryResponse.message.statusCode)) {
return;
}
}
throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`);
yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, () => httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders));
});
}
function parseEnvNumber(key) {
@ -1417,13 +1470,16 @@ function uploadFile(httpClient, cacheId, archivePath) {
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
yield uploadChunk(httpClient, resourceUrl, () => fs
.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
yield uploadChunk(httpClient, resourceUrl, chunk, start, end);
})
.on("error", error => {
throw new Error(`Cache upload failed because file read failed with ${error.Message}`);
}), start, end);
}
})));
}
@ -1436,7 +1492,7 @@ function uploadFile(httpClient, cacheId, archivePath) {
function commitCache(httpClient, cacheId, filesize) {
return __awaiter(this, void 0, void 0, function* () {
const commitCacheRequest = { size: filesize };
return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest);
return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest));
});
}
function saveCache(cacheId, archivePath) {
@ -2802,6 +2858,10 @@ var Events;
Events["Push"] = "push";
Events["PullRequest"] = "pull_request";
})(Events = exports.Events || (exports.Events = {}));
// Socket timeout in milliseconds during download. If no traffic is received
// over the socket during this period, the socket is destroyed and the download
// is aborted.
exports.SocketTimeout = 5000;
/***/ }),
@ -2844,6 +2904,13 @@ module.exports = require("fs");
/***/ }),
/***/ 794:
/***/ (function(module) {
module.exports = require("stream");
/***/ }),
/***/ 826:
/***/ (function(module, __unusedexports, __webpack_require__) {
@ -2909,10 +2976,30 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const exec_1 = __webpack_require__(986);
const io = __importStar(__webpack_require__(1));
const fs_1 = __webpack_require__(747);
function getTarPath() {
const path = __importStar(__webpack_require__(622));
const tar = __importStar(__webpack_require__(943));
function isGnuTar() {
return __awaiter(this, void 0, void 0, function* () {
core.debug("Checking tar --version");
let versionOutput = "";
yield exec_1.exec("tar --version", [], {
ignoreReturnCode: true,
silent: true,
listeners: {
stdout: (data) => (versionOutput += data.toString()),
stderr: (data) => (versionOutput += data.toString())
}
});
core.debug(versionOutput.trim());
return versionOutput.toUpperCase().includes("GNU TAR");
});
}
exports.isGnuTar = isGnuTar;
function getTarPath(args) {
return __awaiter(this, void 0, void 0, function* () {
// Explicitly use BSD Tar on Windows
const IS_WINDOWS = process.platform === "win32";
@ -2921,22 +3008,21 @@ function getTarPath() {
if (fs_1.existsSync(systemTar)) {
return systemTar;
}
else if (yield tar.isGnuTar()) {
args.push("--force-local");
}
}
return yield io.which("tar", true);
});
}
function execTar(args) {
var _a, _b;
var _a;
return __awaiter(this, void 0, void 0, function* () {
try {
yield exec_1.exec(`"${yield getTarPath()}"`, args);
yield exec_1.exec(`"${yield getTarPath(args)}"`, args);
}
catch (error) {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`);
}
throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`);
throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`);
}
});
}
@ -2944,14 +3030,27 @@ function extractTar(archivePath, targetDirectory) {
return __awaiter(this, void 0, void 0, function* () {
// Create directory to extract tar into
yield io.mkdirP(targetDirectory);
const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
const args = [
"-xz",
"-f",
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
"-C",
targetDirectory.replace(new RegExp("\\" + path.sep, "g"), "/")
];
yield execTar(args);
});
}
exports.extractTar = extractTar;
function createTar(archivePath, sourceDirectory) {
return __awaiter(this, void 0, void 0, function* () {
const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
const args = [
"-cz",
"-f",
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
"-C",
sourceDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"),
"."
];
yield execTar(args);
});
}

2
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "cache",
"version": "1.1.1",
"version": "1.1.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

View File

@ -1,6 +1,6 @@
{
"name": "cache",
"version": "1.1.1",
"version": "1.1.2",
"private": true,
"description": "Cache dependencies and build outputs",
"main": "dist/restore/index.js",

View File

@ -1,12 +1,16 @@
import * as core from "@actions/core";
import * as fs from "fs";
import { BearerCredentialHandler } from "@actions/http-client/auth";
import { HttpClient, HttpCodes } from "@actions/http-client";
import { BearerCredentialHandler } from "@actions/http-client/auth";
import {
IHttpClientResponse,
IRequestOptions,
ITypedResponse
} from "@actions/http-client/interfaces";
import * as fs from "fs";
import * as stream from "stream";
import * as util from "util";
import { SocketTimeout } from "./constants";
import {
ArtifactCacheEntry,
CommitCacheRequest,
@ -22,6 +26,13 @@ function isSuccessStatusCode(statusCode?: number): boolean {
return statusCode >= 200 && statusCode < 300;
}
function isServerErrorStatusCode(statusCode?: number): boolean {
if (!statusCode) {
return true;
}
return statusCode >= 500;
}
function isRetryableStatusCode(statusCode?: number): boolean {
if (!statusCode) {
return false;
@ -77,14 +88,83 @@ function createHttpClient(): HttpClient {
);
}
export async function retry<T>(
name: string,
method: () => Promise<T>,
getStatusCode: (T) => number | undefined,
maxAttempts = 2
): Promise<T> {
let response: T | undefined = undefined;
let statusCode: number | undefined = undefined;
let isRetryable = false;
let errorMessage = "";
let attempt = 1;
while (attempt <= maxAttempts) {
try {
response = await method();
statusCode = getStatusCode(response);
if (!isServerErrorStatusCode(statusCode)) {
return response;
}
isRetryable = isRetryableStatusCode(statusCode);
errorMessage = `Cache service responded with ${statusCode}`;
} catch (error) {
isRetryable = true;
errorMessage = error.message;
}
core.debug(
`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`
);
if (!isRetryable) {
core.debug(`${name} - Error is not retryable`);
break;
}
attempt++;
}
throw Error(`${name} failed: ${errorMessage}`);
}
export async function retryTypedResponse<T>(
name: string,
method: () => Promise<ITypedResponse<T>>,
maxAttempts = 2
): Promise<ITypedResponse<T>> {
return await retry(
name,
method,
(response: ITypedResponse<T>) => response.statusCode,
maxAttempts
);
}
export async function retryHttpClientResponse<T>(
name: string,
method: () => Promise<IHttpClientResponse>,
maxAttempts = 2
): Promise<IHttpClientResponse> {
return await retry(
name,
method,
(response: IHttpClientResponse) => response.message.statusCode,
maxAttempts
);
}
export async function getCacheEntry(
keys: string[]
): Promise<ArtifactCacheEntry | null> {
const httpClient = createHttpClient();
const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
const response = await httpClient.getJson<ArtifactCacheEntry>(
getCacheApiUrl(resource)
const response = await retryTypedResponse("getCacheEntry", () =>
httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
);
if (response.statusCode === 204) {
return null;
@ -107,13 +187,10 @@ export async function getCacheEntry(
async function pipeResponseToStream(
response: IHttpClientResponse,
stream: NodeJS.WritableStream
output: NodeJS.WritableStream
): Promise<void> {
return new Promise(resolve => {
response.message.pipe(stream).on("close", () => {
resolve();
});
});
const pipeline = util.promisify(stream.pipeline);
await pipeline(response.message, output);
}
export async function downloadCache(
@ -122,8 +199,37 @@ export async function downloadCache(
): Promise<void> {
const stream = fs.createWriteStream(archivePath);
const httpClient = new HttpClient("actions/cache");
const downloadResponse = await httpClient.get(archiveLocation);
const downloadResponse = await retryHttpClientResponse(
"downloadCache",
() => httpClient.get(archiveLocation)
);
// Abort download if no traffic received over the socket.
downloadResponse.message.socket.setTimeout(SocketTimeout, () => {
downloadResponse.message.destroy();
core.debug(
`Aborting download, socket timed out after ${SocketTimeout} ms`
);
});
await pipeResponseToStream(downloadResponse, stream);
// Validate download size.
const contentLengthHeader =
downloadResponse.message.headers["content-length"];
if (contentLengthHeader) {
const expectedLength = parseInt(contentLengthHeader);
const actualLength = utils.getArchiveFileSize(archivePath);
if (actualLength != expectedLength) {
throw new Error(
`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`
);
}
} else {
core.debug("Unable to validate download, no Content-Length header");
}
}
// Reserve Cache
@ -133,9 +239,11 @@ export async function reserveCache(key: string): Promise<number> {
const reserveCacheRequest: ReserveCacheRequest = {
key
};
const response = await httpClient.postJson<ReserveCacheResponse>(
getCacheApiUrl("caches"),
reserveCacheRequest
const response = await retryTypedResponse("reserveCache", () =>
httpClient.postJson<ReserveCacheResponse>(
getCacheApiUrl("caches"),
reserveCacheRequest
)
);
return response?.result?.cacheId ?? -1;
}
@ -152,7 +260,7 @@ function getContentRange(start: number, end: number): string {
async function uploadChunk(
httpClient: HttpClient,
resourceUrl: string,
data: NodeJS.ReadableStream,
openStream: () => NodeJS.ReadableStream,
start: number,
end: number
): Promise<void> {
@ -169,32 +277,15 @@ async function uploadChunk(
"Content-Range": getContentRange(start, end)
};
const uploadChunkRequest = async (): Promise<IHttpClientResponse> => {
return await httpClient.sendStream(
"PATCH",
resourceUrl,
data,
additionalHeaders
);
};
const response = await uploadChunkRequest();
if (isSuccessStatusCode(response.message.statusCode)) {
return;
}
if (isRetryableStatusCode(response.message.statusCode)) {
core.debug(
`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`
);
const retryResponse = await uploadChunkRequest();
if (isSuccessStatusCode(retryResponse.message.statusCode)) {
return;
}
}
throw new Error(
`Cache service responded with ${response.message.statusCode} during chunk upload.`
await retryHttpClientResponse(
`uploadChunk (start: ${start}, end: ${end})`,
() =>
httpClient.sendStream(
"PATCH",
resourceUrl,
openStream(),
additionalHeaders
)
);
}
@ -236,17 +327,23 @@ async function uploadFile(
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
await uploadChunk(
httpClient,
resourceUrl,
chunk,
() =>
fs
.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
})
.on("error", error => {
throw new Error(
`Cache upload failed because file read failed with ${error.Message}`
);
}),
start,
end
);
@ -265,9 +362,11 @@ async function commitCache(
filesize: number
): Promise<ITypedResponse<null>> {
const commitCacheRequest: CommitCacheRequest = { size: filesize };
return await httpClient.postJson<null>(
getCacheApiUrl(`caches/${cacheId.toString()}`),
commitCacheRequest
return await retryTypedResponse("commitCache", () =>
httpClient.postJson<null>(
getCacheApiUrl(`caches/${cacheId.toString()}`),
commitCacheRequest
)
);
}

View File

@ -18,3 +18,8 @@ export enum Events {
Push = "push",
PullRequest = "pull_request"
}
// Socket timeout in milliseconds during download. If no traffic is received
// over the socket during this period, the socket is destroyed and the download
// is aborted.
export const SocketTimeout = 5000;

View File

@ -1,14 +1,36 @@
import * as core from "@actions/core";
import { exec } from "@actions/exec";
import * as io from "@actions/io";
import { existsSync } from "fs";
import * as path from "path";
import * as tar from "./tar";
async function getTarPath(): Promise<string> {
export async function isGnuTar(): Promise<boolean> {
core.debug("Checking tar --version");
let versionOutput = "";
await exec("tar --version", [], {
ignoreReturnCode: true,
silent: true,
listeners: {
stdout: (data: Buffer): string =>
(versionOutput += data.toString()),
stderr: (data: Buffer): string => (versionOutput += data.toString())
}
});
core.debug(versionOutput.trim());
return versionOutput.toUpperCase().includes("GNU TAR");
}
async function getTarPath(args: string[]): Promise<string> {
// Explicitly use BSD Tar on Windows
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
if (existsSync(systemTar)) {
return systemTar;
} else if (await tar.isGnuTar()) {
args.push("--force-local");
}
}
return await io.which("tar", true);
@ -16,14 +38,8 @@ async function getTarPath(): Promise<string> {
async function execTar(args: string[]): Promise<void> {
try {
await exec(`"${await getTarPath()}"`, args);
await exec(`"${await getTarPath(args)}"`, args);
} catch (error) {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
throw new Error(
`Tar failed with error: ${error?.message}. Ensure BSD tar is installed and on the PATH.`
);
}
throw new Error(`Tar failed with error: ${error?.message}`);
}
}
@ -34,7 +50,13 @@ export async function extractTar(
): Promise<void> {
// Create directory to extract tar into
await io.mkdirP(targetDirectory);
const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
const args = [
"-xz",
"-f",
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
"-C",
targetDirectory.replace(new RegExp("\\" + path.sep, "g"), "/")
];
await execTar(args);
}
@ -42,6 +64,13 @@ export async function createTar(
archivePath: string,
sourceDirectory: string
): Promise<void> {
const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
const args = [
"-cz",
"-f",
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
"-C",
sourceDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"),
"."
];
await execTar(args);
}