actions/cache
1
0
Fork 0
mirror of https://github.com/actions/cache.git synced 2024-12-22 22:11:46 +01:00
Code Issues Projects Releases Packages Wiki Activity

Release v1.1

Browse source
This commit is contained in:
Josh Gross 2020-01-06 13:36:33 -05:00
parent cffae9552b
commit 4c4974aff1
14 changed files with 959 additions and 373 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
README.md
View file

@ -35,7 +35,7 @@ on: push
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
@ -49,14 +49,31 @@ jobs:
- name: Generate Prime Numbers
if: steps.cache-primes.outputs.cache-hit != 'true'
run: /generate-primes.sh -d prime-numbers
- name: Use Prime Numbers
run: /primes.sh -d prime-numbers
```
## Ecosystem Examples
## Implementation Examples
Every programming language and framework has its own way of caching.
See [Examples](examples.md) for a list of `actions/cache` implementations for use with:
- [C# - Nuget](./examples.md#c---nuget)
- [Elixir - Mix](./examples.md#elixir---mix)
- [Go - Modules](./examples.md#go---modules)
- [Java - Gradle](./examples.md#java---gradle)
- [Java - Maven](./examples.md#java---maven)
- [Node - npm](./examples.md#node---npm)
- [Node - Yarn](./examples.md#node---yarn)
- [PHP - Composer](./examples.md#php---composer)
- [Python - pip](./examples.md#python---pip)
- [Ruby - Gem](./examples.md#ruby---gem)
- [Rust - Cargo](./examples.md#rust---cargo)
- [Swift, Objective-C - Carthage](./examples.md#swift-objective-c---carthage)
- [Swift, Objective-C - CocoaPods](./examples.md#swift-objective-c---cocoapods)
See [Examples](examples.md)
## Cache Limits
@ -76,7 +93,7 @@ steps:
with:
path: path/to/dependencies
key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }}
- name: Install Dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: /install.sh

87
__tests__/restore.test.ts
View file

@ -1,18 +1,16 @@
import * as core from "@actions/core";
import * as exec from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path";
import * as cacheHttpClient from "../src/cacheHttpClient";
import { Events, Inputs } from "../src/constants";
import { ArtifactCacheEntry } from "../src/contracts";
import run from "../src/restore";
import * as tar from "../src/tar";
import * as actionUtils from "../src/utils/actionUtils";
import * as testUtils from "../src/utils/testUtils";
jest.mock("@actions/exec");
jest.mock("@actions/io");
jest.mock("../src/utils/actionUtils");
jest.mock("../src/cacheHttpClient");
jest.mock("../src/tar");
jest.mock("../src/utils/actionUtils");
beforeAll(() => {
jest.spyOn(actionUtils, "resolvePath").mockImplementation(filePath => {
@ -35,10 +33,6 @@ beforeAll(() => {
const actualUtils = jest.requireActual("../src/utils/actionUtils");
return actualUtils.getSupportedEvents();
});
jest.spyOn(io, "which").mockImplementation(tool => {
return Promise.resolve(tool);
});
});
beforeEach(() => {
@ -245,8 +239,7 @@ test("restore with cache found", async () => {
.spyOn(actionUtils, "getArchiveFileSize")
.mockReturnValue(fileSize);
const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const extractTarMock = jest.spyOn(tar, "extractTar");
const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput");
await run();
@ -255,24 +248,14 @@ test("restore with cache found", async () => {
expect(getCacheMock).toHaveBeenCalledWith([key]);
expect(setCacheStateMock).toHaveBeenCalledWith(cacheEntry);
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1);
expect(downloadCacheMock).toHaveBeenCalledWith(cacheEntry, archivePath);
expect(downloadCacheMock).toHaveBeenCalledWith(
cacheEntry.archiveLocation,
archivePath
);
expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath);
expect(mkdirMock).toHaveBeenCalledWith(cachePath);
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(extractTarMock).toHaveBeenCalledTimes(1);
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(true);
@ -323,8 +306,7 @@ test("restore with a pull request event and cache found", async () => {
.spyOn(actionUtils, "getArchiveFileSize")
.mockReturnValue(fileSize);
const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const extractTarMock = jest.spyOn(tar, "extractTar");
const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput");
await run();
@ -333,25 +315,15 @@ test("restore with a pull request event and cache found", async () => {
expect(getCacheMock).toHaveBeenCalledWith([key]);
expect(setCacheStateMock).toHaveBeenCalledWith(cacheEntry);
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1);
expect(downloadCacheMock).toHaveBeenCalledWith(cacheEntry, archivePath);
expect(downloadCacheMock).toHaveBeenCalledWith(
cacheEntry.archiveLocation,
archivePath
);
expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath);
expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`);
expect(mkdirMock).toHaveBeenCalledWith(cachePath);
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(extractTarMock).toHaveBeenCalledTimes(1);
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(true);
@ -402,8 +374,7 @@ test("restore with cache found for restore key", async () => {
.spyOn(actionUtils, "getArchiveFileSize")
.mockReturnValue(fileSize);
const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const extractTarMock = jest.spyOn(tar, "extractTar");
const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput");
await run();
@ -412,25 +383,15 @@ test("restore with cache found for restore key", async () => {
expect(getCacheMock).toHaveBeenCalledWith([key, restoreKey]);
expect(setCacheStateMock).toHaveBeenCalledWith(cacheEntry);
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1);
expect(downloadCacheMock).toHaveBeenCalledWith(cacheEntry, archivePath);
expect(downloadCacheMock).toHaveBeenCalledWith(
cacheEntry.archiveLocation,
archivePath
);
expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath);
expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`);
expect(mkdirMock).toHaveBeenCalledWith(cachePath);
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(extractTarMock).toHaveBeenCalledTimes(1);
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(false);

152
__tests__/save.test.ts
View file

@ -1,19 +1,17 @@
import * as core from "@actions/core";
import * as exec from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path";
import * as cacheHttpClient from "../src/cacheHttpClient";
import { Events, Inputs } from "../src/constants";
import { ArtifactCacheEntry } from "../src/contracts";
import run from "../src/save";
import * as tar from "../src/tar";
import * as actionUtils from "../src/utils/actionUtils";
import * as testUtils from "../src/utils/testUtils";
jest.mock("@actions/core");
jest.mock("@actions/exec");
jest.mock("@actions/io");
jest.mock("../src/utils/actionUtils");
jest.mock("../src/cacheHttpClient");
jest.mock("../src/tar");
jest.mock("../src/utils/actionUtils");
beforeAll(() => {
jest.spyOn(core, "getInput").mockImplementation((name, options) => {
@ -49,10 +47,6 @@ beforeAll(() => {
jest.spyOn(actionUtils, "createTempDirectory").mockImplementation(() => {
return Promise.resolve("/foo/bar");
});
jest.spyOn(io, "which").mockImplementation(tool => {
return Promise.resolve(tool);
});
});
beforeEach(() => {
@ -128,7 +122,7 @@ test("save with exact match returns early", async () => {
return primaryKey;
});
const execMock = jest.spyOn(exec, "exec");
const createTarMock = jest.spyOn(tar, "createTar");
await run();
@ -136,7 +130,7 @@ test("save with exact match returns early", async () => {
`Cache hit occurred on the primary key ${primaryKey}, not saving cache.`
);
expect(execMock).toHaveBeenCalledTimes(0);
expect(createTarMock).toHaveBeenCalledTimes(0);
expect(failedMock).toHaveBeenCalledTimes(0);
});
@ -198,9 +192,9 @@ test("save with large cache outputs warning", async () => {
const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath);
const execMock = jest.spyOn(exec, "exec");
const createTarMock = jest.spyOn(tar, "createTar");
const cacheSize = 1024 * 1024 * 1024; //~1GB, over the 400MB limit
const cacheSize = 4 * 1024 * 1024 * 1024; //~4GB, over the 2GB limit
jest.spyOn(actionUtils, "getArchiveFileSize").mockImplementationOnce(() => {
return cacheSize;
});
@ -209,30 +203,68 @@ test("save with large cache outputs warning", async () => {
const archivePath = path.join("/foo/bar", "cache.tgz");
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(createTarMock).toHaveBeenCalledTimes(1);
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith(
"Cache size of ~1024 MB (1073741824 B) is over the 400MB limit, not saving cache."
"Cache size of ~4096 MB (4294967296 B) is over the 2GB limit, not saving cache."
);
expect(failedMock).toHaveBeenCalledTimes(0);
});
test("save with reserve cache failure outputs warning", async () => {
const infoMock = jest.spyOn(core, "info");
const logWarningMock = jest.spyOn(actionUtils, "logWarning");
const failedMock = jest.spyOn(core, "setFailed");
const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43";
const cacheEntry: ArtifactCacheEntry = {
cacheKey: "Linux-node-",
scope: "refs/heads/master",
creationTime: "2019-11-13T19:18:02+00:00",
archiveLocation: "www.actionscache.test/download"
};
jest.spyOn(core, "getState")
// Cache Entry State
.mockImplementationOnce(() => {
return JSON.stringify(cacheEntry);
})
// Cache Key State
.mockImplementationOnce(() => {
return primaryKey;
});
const inputPath = "node_modules";
testUtils.setInput(Inputs.Path, inputPath);
const reserveCacheMock = jest
.spyOn(cacheHttpClient, "reserveCache")
.mockImplementationOnce(() => {
return Promise.resolve(-1);
});
const createTarMock = jest.spyOn(tar, "createTar");
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
expect(infoMock).toHaveBeenCalledWith(
`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
);
expect(createTarMock).toHaveBeenCalledTimes(0);
expect(saveCacheMock).toHaveBeenCalledTimes(0);
expect(logWarningMock).toHaveBeenCalledTimes(0);
expect(failedMock).toHaveBeenCalledTimes(0);
});
test("save with server error outputs warning", async () => {
const logWarningMock = jest.spyOn(actionUtils, "logWarning");
const failedMock = jest.spyOn(core, "setFailed");
@ -259,7 +291,14 @@ test("save with server error outputs warning", async () => {
const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath);
const execMock = jest.spyOn(exec, "exec");
const cacheId = 4;
const reserveCacheMock = jest
.spyOn(cacheHttpClient, "reserveCache")
.mockImplementationOnce(() => {
return Promise.resolve(cacheId);
});
const createTarMock = jest.spyOn(tar, "createTar");
const saveCacheMock = jest
.spyOn(cacheHttpClient, "saveCache")
@ -269,26 +308,16 @@ test("save with server error outputs warning", async () => {
await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
const archivePath = path.join("/foo/bar", "cache.tgz");
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(createTarMock).toHaveBeenCalledTimes(1);
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred");
@ -321,32 +350,29 @@ test("save with valid inputs uploads a cache", async () => {
const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath);
const execMock = jest.spyOn(exec, "exec");
const cacheId = 4;
const reserveCacheMock = jest
.spyOn(cacheHttpClient, "reserveCache")
.mockImplementationOnce(() => {
return Promise.resolve(cacheId);
});
const createTarMock = jest.spyOn(tar, "createTar");
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
const archivePath = path.join("/foo/bar", "cache.tgz");
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(createTarMock).toHaveBeenCalledTimes(1);
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath);
expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(failedMock).toHaveBeenCalledTimes(0);
});

58
__tests__/tar.test.ts Normal file
View file

@ -0,0 +1,58 @@
import * as exec from "@actions/exec";
import * as io from "@actions/io";
import * as tar from "../src/tar";
jest.mock("@actions/exec");
jest.mock("@actions/io");
beforeAll(() => {
jest.spyOn(io, "which").mockImplementation(tool => {
return Promise.resolve(tool);
});
});
test("extract tar", async () => {
const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const archivePath = "cache.tar";
const targetDirectory = "~/.npm/cache";
await tar.extractTar(archivePath, targetDirectory);
expect(mkdirMock).toHaveBeenCalledWith(targetDirectory);
const IS_WINDOWS = process.platform === "win32";
const tarPath = IS_WINDOWS
? `${process.env["windir"]}\\System32\\tar.exe`
: "tar";
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [
"-xz",
"-f",
archivePath,
"-C",
targetDirectory
]);
});
test("create tar", async () => {
const execMock = jest.spyOn(exec, "exec");
const archivePath = "cache.tar";
const sourceDirectory = "~/.npm/cache";
await tar.createTar(archivePath, sourceDirectory);
const IS_WINDOWS = process.platform === "win32";
const tarPath = IS_WINDOWS
? `${process.env["windir"]}\\System32\\tar.exe`
: "tar";
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [
"-cz",
"-f",
archivePath,
"-C",
sourceDirectory,
"."
]);
});

265
dist/restore/index.js vendored
View file

@ -1496,47 +1496,63 @@ const fs = __importStar(__webpack_require__(747));
const Handlers_1 = __webpack_require__(941);
const HttpClient_1 = __webpack_require__(874);
const RestClient_1 = __webpack_require__(105);
function getCacheUrl() {
const utils = __importStar(__webpack_require__(443));
function isSuccessStatusCode(statusCode) {
return statusCode >= 200 && statusCode < 300;
}
function isRetryableStatusCode(statusCode) {
const retryableStatusCodes = [
HttpClient_1.HttpCodes.BadGateway,
HttpClient_1.HttpCodes.ServiceUnavailable,
HttpClient_1.HttpCodes.GatewayTimeout
];
return retryableStatusCodes.includes(statusCode);
}
function getCacheApiUrl() {
// Ideally we just use ACTIONS_CACHE_URL
const cacheUrl = (process.env["ACTIONS_CACHE_URL"] ||
const baseUrl = (process.env["ACTIONS_CACHE_URL"] ||
process.env["ACTIONS_RUNTIME_URL"] ||
"").replace("pipelines", "artifactcache");
if (!cacheUrl) {
if (!baseUrl) {
throw new Error("Cache Service Url not found, unable to restore cache.");
}
core.debug(`Cache Url: ${cacheUrl}`);
return cacheUrl;
core.debug(`Cache Url: ${baseUrl}`);
return `${baseUrl}_apis/artifactcache/`;
}
function createAcceptHeader(type, apiVersion) {
return `${type};api-version=${apiVersion}`;
}
function getRequestOptions() {
const requestOptions = {
acceptHeader: createAcceptHeader("application/json", "5.2-preview.1")
acceptHeader: createAcceptHeader("application/json", "6.0-preview.1")
};
return requestOptions;
}
function createRestClient() {
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token);
return new RestClient_1.RestClient("actions/cache", getCacheApiUrl(), [
bearerCredentialHandler
]);
}
function getCacheEntry(keys) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const cacheUrl = getCacheUrl();
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent(keys.join(","))}`;
const restClient = new RestClient_1.RestClient("actions/cache", cacheUrl, [
bearerCredentialHandler
]);
const restClient = createRestClient();
const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
const response = yield restClient.get(resource, getRequestOptions());
if (response.statusCode === 204) {
return null;
}
if (response.statusCode !== 200) {
if (!isSuccessStatusCode(response.statusCode)) {
throw new Error(`Cache service responded with ${response.statusCode}`);
}
const cacheResult = response.result;
if (!cacheResult || !cacheResult.archiveLocation) {
const cacheDownloadUrl = (_a = cacheResult) === null || _a === void 0 ? void 0 : _a.archiveLocation;
if (!cacheDownloadUrl) {
throw new Error("Cache not found.");
}
core.setSecret(cacheResult.archiveLocation);
core.setSecret(cacheDownloadUrl);
core.debug(`Cache Result:`);
core.debug(JSON.stringify(cacheResult));
return cacheResult;
@ -1552,34 +1568,117 @@ function pipeResponseToStream(response, stream) {
});
});
}
function downloadCache(cacheEntry, archivePath) {
function downloadCache(archiveLocation, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createWriteStream(archivePath);
const httpClient = new HttpClient_1.HttpClient("actions/cache");
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const downloadResponse = yield httpClient.get(cacheEntry.archiveLocation);
const downloadResponse = yield httpClient.get(archiveLocation);
yield pipeResponseToStream(downloadResponse, stream);
});
}
exports.downloadCache = downloadCache;
function saveCache(key, archivePath) {
// Reserve Cache
function reserveCache(key) {
var _a, _b, _c;
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createReadStream(archivePath);
const cacheUrl = getCacheUrl();
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`;
const postUrl = cacheUrl + resource;
const restClient = new RestClient_1.RestClient("actions/cache", undefined, [
bearerCredentialHandler
]);
const restClient = createRestClient();
const reserveCacheRequest = {
key
};
const response = yield restClient.create("caches", reserveCacheRequest, getRequestOptions());
return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1);
});
}
exports.reserveCache = reserveCache;
function getContentRange(start, end) {
// Format: `bytes start-end/filesize
// start and end are inclusive
// filesize can be *
// For a 200 byte chunk starting at byte 0:
// Content-Range: bytes 0-199/*
return `bytes ${start}-${end}/*`;
}
function uploadChunk(restClient, resourceUrl, data, start, end) {
return __awaiter(this, void 0, void 0, function* () {
core.debug(`Uploading chunk of size ${end -
start +
1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`);
const requestOptions = getRequestOptions();
requestOptions.additionalHeaders = {
"Content-Type": "application/octet-stream"
"Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end)
};
const response = yield restClient.uploadStream("POST", postUrl, stream, requestOptions);
if (response.statusCode !== 200) {
throw new Error(`Cache service responded with ${response.statusCode}`);
const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () {
return yield restClient.uploadStream("PATCH", resourceUrl, data, requestOptions);
});
const response = yield uploadChunkRequest();
if (isSuccessStatusCode(response.statusCode)) {
return;
}
if (isRetryableStatusCode(response.statusCode)) {
core.debug(`Received ${response.statusCode}, retrying chunk at offset ${start}.`);
const retryResponse = yield uploadChunkRequest();
if (isSuccessStatusCode(retryResponse.statusCode)) {
return;
}
}
throw new Error(`Cache service responded with ${response.statusCode} during chunk upload.`);
});
}
function uploadFile(restClient, cacheId, archivePath) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
// Upload Chunks
const fileSize = fs.statSync(archivePath).size;
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
const fd = fs.openSync(archivePath, "r");
const concurrency = (_a = Number(process.env["CACHE_UPLOAD_CONCURRENCY"]), (_a !== null && _a !== void 0 ? _a : 4)); // # of HTTP requests in parallel
const MAX_CHUNK_SIZE = (_b = Number(process.env["CACHE_UPLOAD_CHUNK_SIZE"]), (_b !== null && _b !== void 0 ? _b : 32 * 1024 * 1024)); // 32 MB Chunks
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parallelUploads = [...new Array(concurrency).keys()];
core.debug("Awaiting all uploads");
let offset = 0;
try {
yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () {
while (offset < fileSize) {
const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE);
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
yield uploadChunk(restClient, resourceUrl, chunk, start, end);
}
})));
}
finally {
fs.closeSync(fd);
}
return;
});
}
function commitCache(restClient, cacheId, filesize) {
return __awaiter(this, void 0, void 0, function* () {
const requestOptions = getRequestOptions();
const commitCacheRequest = { size: filesize };
return yield restClient.create(`caches/${cacheId.toString()}`, commitCacheRequest, requestOptions);
});
}
function saveCache(cacheId, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const restClient = createRestClient();
core.debug("Upload cache");
yield uploadFile(restClient, cacheId, archivePath);
// Commit Cache
core.debug("Commiting cache");
const cacheSize = utils.getArchiveFileSize(archivePath);
const commitCacheResponse = yield commitCache(restClient, cacheId, cacheSize);
if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
throw new Error(`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`);
}
core.info("Cache saved successfully");
});
@ -2991,13 +3090,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const exec_1 = __webpack_require__(986);
const io = __importStar(__webpack_require__(1));
const path = __importStar(__webpack_require__(622));
const cacheHttpClient = __importStar(__webpack_require__(154));
const constants_1 = __webpack_require__(694);
const tar_1 = __webpack_require__(943);
const utils = __importStar(__webpack_require__(443));
function run() {
var _a;
return __awaiter(this, void 0, void 0, function* () {
try {
// Validate inputs, this can cause task failure
@ -3035,7 +3134,7 @@ function run() {
}
try {
const cacheEntry = yield cacheHttpClient.getCacheEntry(keys);
if (!cacheEntry) {
if (!((_a = cacheEntry) === null || _a === void 0 ? void 0 : _a.archiveLocation)) {
core.info(`Cache not found for input keys: ${keys.join(", ")}.`);
return;
}
@ -3044,27 +3143,10 @@ function run() {
// Store the cache result
utils.setCacheState(cacheEntry);
// Download the cache from the cache entry
yield cacheHttpClient.downloadCache(cacheEntry, archivePath);
yield cacheHttpClient.downloadCache(cacheEntry.archiveLocation, archivePath);
const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.info(`Cache Size: ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B)`);
// Create directory to extract tar into
yield io.mkdirP(cachePath);
// http://man7.org/linux/man-pages/man1/tar.1.html
// tar [-options] <name of the tar archive> [files or directories which to add into archive]
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
const tarPath = yield io.which("tar", true);
core.debug(`Tar Path: ${tarPath}`);
yield exec_1.exec(`"${tarPath}"`, args);
yield tar_1.extractTar(archivePath, cachePath);
const isExactKeyMatch = utils.isExactKeyMatch(primaryKey, cacheEntry);
utils.setCacheHitOutput(isExactKeyMatch);
core.info(`Cache restored from key: ${cacheEntry && cacheEntry.cacheKey}`);
@ -5160,6 +5242,79 @@ var personalaccesstoken_1 = __webpack_require__(327);
exports.PersonalAccessTokenCredentialHandler = personalaccesstoken_1.PersonalAccessTokenCredentialHandler;
/***/ }),
/***/ 943:
/***/ (function(__unusedmodule, exports, __webpack_require__) {
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const exec_1 = __webpack_require__(986);
const io = __importStar(__webpack_require__(1));
const fs_1 = __webpack_require__(747);
function getTarPath() {
return __awaiter(this, void 0, void 0, function* () {
// Explicitly use BSD Tar on Windows
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
if (fs_1.existsSync(systemTar)) {
return systemTar;
}
}
return yield io.which("tar", true);
});
}
function execTar(args) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
try {
yield exec_1.exec(`"${yield getTarPath()}"`, args);
}
catch (error) {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`);
}
throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`);
}
});
}
function extractTar(archivePath, targetDirectory) {
return __awaiter(this, void 0, void 0, function* () {
// Create directory to extract tar into
yield io.mkdirP(targetDirectory);
const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
yield execTar(args);
});
}
exports.extractTar = extractTar;
function createTar(archivePath, sourceDirectory) {
return __awaiter(this, void 0, void 0, function* () {
const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
yield execTar(args);
});
}
exports.createTar = createTar;
/***/ }),
/***/ 986:

273
dist/save/index.js vendored
View file

@ -1496,47 +1496,63 @@ const fs = __importStar(__webpack_require__(747));
const Handlers_1 = __webpack_require__(941);
const HttpClient_1 = __webpack_require__(874);
const RestClient_1 = __webpack_require__(105);
function getCacheUrl() {
const utils = __importStar(__webpack_require__(443));
function isSuccessStatusCode(statusCode) {
return statusCode >= 200 && statusCode < 300;
}
function isRetryableStatusCode(statusCode) {
const retryableStatusCodes = [
HttpClient_1.HttpCodes.BadGateway,
HttpClient_1.HttpCodes.ServiceUnavailable,
HttpClient_1.HttpCodes.GatewayTimeout
];
return retryableStatusCodes.includes(statusCode);
}
function getCacheApiUrl() {
// Ideally we just use ACTIONS_CACHE_URL
const cacheUrl = (process.env["ACTIONS_CACHE_URL"] ||
const baseUrl = (process.env["ACTIONS_CACHE_URL"] ||
process.env["ACTIONS_RUNTIME_URL"] ||
"").replace("pipelines", "artifactcache");
if (!cacheUrl) {
if (!baseUrl) {
throw new Error("Cache Service Url not found, unable to restore cache.");
}
core.debug(`Cache Url: ${cacheUrl}`);
return cacheUrl;
core.debug(`Cache Url: ${baseUrl}`);
return `${baseUrl}_apis/artifactcache/`;
}
function createAcceptHeader(type, apiVersion) {
return `${type};api-version=${apiVersion}`;
}
function getRequestOptions() {
const requestOptions = {
acceptHeader: createAcceptHeader("application/json", "5.2-preview.1")
acceptHeader: createAcceptHeader("application/json", "6.0-preview.1")
};
return requestOptions;
}
function createRestClient() {
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token);
return new RestClient_1.RestClient("actions/cache", getCacheApiUrl(), [
bearerCredentialHandler
]);
}
function getCacheEntry(keys) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const cacheUrl = getCacheUrl();
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent(keys.join(","))}`;
const restClient = new RestClient_1.RestClient("actions/cache", cacheUrl, [
bearerCredentialHandler
]);
const restClient = createRestClient();
const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
const response = yield restClient.get(resource, getRequestOptions());
if (response.statusCode === 204) {
return null;
}
if (response.statusCode !== 200) {
if (!isSuccessStatusCode(response.statusCode)) {
throw new Error(`Cache service responded with ${response.statusCode}`);
}
const cacheResult = response.result;
if (!cacheResult || !cacheResult.archiveLocation) {
const cacheDownloadUrl = (_a = cacheResult) === null || _a === void 0 ? void 0 : _a.archiveLocation;
if (!cacheDownloadUrl) {
throw new Error("Cache not found.");
}
core.setSecret(cacheResult.archiveLocation);
core.setSecret(cacheDownloadUrl);
core.debug(`Cache Result:`);
core.debug(JSON.stringify(cacheResult));
return cacheResult;
@ -1552,34 +1568,117 @@ function pipeResponseToStream(response, stream) {
});
});
}
function downloadCache(cacheEntry, archivePath) {
function downloadCache(archiveLocation, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createWriteStream(archivePath);
const httpClient = new HttpClient_1.HttpClient("actions/cache");
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const downloadResponse = yield httpClient.get(cacheEntry.archiveLocation);
const downloadResponse = yield httpClient.get(archiveLocation);
yield pipeResponseToStream(downloadResponse, stream);
});
}
exports.downloadCache = downloadCache;
function saveCache(key, archivePath) {
// Reserve Cache
function reserveCache(key) {
var _a, _b, _c;
return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createReadStream(archivePath);
const cacheUrl = getCacheUrl();
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new Handlers_1.BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`;
const postUrl = cacheUrl + resource;
const restClient = new RestClient_1.RestClient("actions/cache", undefined, [
bearerCredentialHandler
]);
const restClient = createRestClient();
const reserveCacheRequest = {
key
};
const response = yield restClient.create("caches", reserveCacheRequest, getRequestOptions());
return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1);
});
}
exports.reserveCache = reserveCache;
function getContentRange(start, end) {
// Format: `bytes start-end/filesize
// start and end are inclusive
// filesize can be *
// For a 200 byte chunk starting at byte 0:
// Content-Range: bytes 0-199/*
return `bytes ${start}-${end}/*`;
}
function uploadChunk(restClient, resourceUrl, data, start, end) {
return __awaiter(this, void 0, void 0, function* () {
core.debug(`Uploading chunk of size ${end -
start +
1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`);
const requestOptions = getRequestOptions();
requestOptions.additionalHeaders = {
"Content-Type": "application/octet-stream"
"Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end)
};
const response = yield restClient.uploadStream("POST", postUrl, stream, requestOptions);
if (response.statusCode !== 200) {
throw new Error(`Cache service responded with ${response.statusCode}`);
const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () {
return yield restClient.uploadStream("PATCH", resourceUrl, data, requestOptions);
});
const response = yield uploadChunkRequest();
if (isSuccessStatusCode(response.statusCode)) {
return;
}
if (isRetryableStatusCode(response.statusCode)) {
core.debug(`Received ${response.statusCode}, retrying chunk at offset ${start}.`);
const retryResponse = yield uploadChunkRequest();
if (isSuccessStatusCode(retryResponse.statusCode)) {
return;
}
}
throw new Error(`Cache service responded with ${response.statusCode} during chunk upload.`);
});
}
function uploadFile(restClient, cacheId, archivePath) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
// Upload Chunks
const fileSize = fs.statSync(archivePath).size;
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
const fd = fs.openSync(archivePath, "r");
const concurrency = (_a = Number(process.env["CACHE_UPLOAD_CONCURRENCY"]), (_a !== null && _a !== void 0 ? _a : 4)); // # of HTTP requests in parallel
const MAX_CHUNK_SIZE = (_b = Number(process.env["CACHE_UPLOAD_CHUNK_SIZE"]), (_b !== null && _b !== void 0 ? _b : 32 * 1024 * 1024)); // 32 MB Chunks
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parallelUploads = [...new Array(concurrency).keys()];
core.debug("Awaiting all uploads");
let offset = 0;
try {
yield Promise.all(parallelUploads.map(() => __awaiter(this, void 0, void 0, function* () {
while (offset < fileSize) {
const chunkSize = Math.min(fileSize - offset, MAX_CHUNK_SIZE);
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
yield uploadChunk(restClient, resourceUrl, chunk, start, end);
}
})));
}
finally {
fs.closeSync(fd);
}
return;
});
}
function commitCache(restClient, cacheId, filesize) {
return __awaiter(this, void 0, void 0, function* () {
const requestOptions = getRequestOptions();
const commitCacheRequest = { size: filesize };
return yield restClient.create(`caches/${cacheId.toString()}`, commitCacheRequest, requestOptions);
});
}
function saveCache(cacheId, archivePath) {
return __awaiter(this, void 0, void 0, function* () {
const restClient = createRestClient();
core.debug("Upload cache");
yield uploadFile(restClient, cacheId, archivePath);
// Commit Cache
core.debug("Commiting cache");
const cacheSize = utils.getArchiveFileSize(archivePath);
const commitCacheResponse = yield commitCache(restClient, cacheId, cacheSize);
if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
throw new Error(`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`);
}
core.info("Cache saved successfully");
});
@ -2879,11 +2978,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
const core = __importStar(__webpack_require__(470));
const exec_1 = __webpack_require__(986);
const io = __importStar(__webpack_require__(1));
const path = __importStar(__webpack_require__(622));
const cacheHttpClient = __importStar(__webpack_require__(154));
const constants_1 = __webpack_require__(694);
const tar_1 = __webpack_require__(943);
const utils = __importStar(__webpack_require__(443));
function run() {
return __awaiter(this, void 0, void 0, function* () {
@ -2905,35 +3003,27 @@ function run() {
core.info(`Cache hit occurred on the primary key ${primaryKey}, not saving cache.`);
return;
}
core.debug("Reserving Cache");
const cacheId = yield cacheHttpClient.reserveCache(primaryKey);
if (cacheId == -1) {
core.info(`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`);
return;
}
core.debug(`Cache ID: ${cacheId}`);
const cachePath = utils.resolvePath(core.getInput(constants_1.Inputs.Path, { required: true }));
core.debug(`Cache Path: ${cachePath}`);
const archivePath = path.join(yield utils.createTempDirectory(), "cache.tgz");
core.debug(`Archive Path: ${archivePath}`);
// http://man7.org/linux/man-pages/man1/tar.1.html
// tar [-options] <name of the tar archive> [files or directories which to add into archive]
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
const tarPath = yield io.which("tar", true);
core.debug(`Tar Path: ${tarPath}`);
yield exec_1.exec(`"${tarPath}"`, args);
const fileSizeLimit = 400 * 1024 * 1024; // 400MB
yield tar_1.createTar(archivePath, cachePath);
const fileSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB per repo limit
const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.debug(`File Size: ${archiveFileSize}`);
if (archiveFileSize > fileSizeLimit) {
utils.logWarning(`Cache size of ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B) is over the 400MB limit, not saving cache.`);
utils.logWarning(`Cache size of ~${Math.round(archiveFileSize / (1024 * 1024))} MB (${archiveFileSize} B) is over the 2GB limit, not saving cache.`);
return;
}
yield cacheHttpClient.saveCache(primaryKey, archivePath);
core.debug(`Saving Cache (ID: ${cacheId})`);
yield cacheHttpClient.saveCache(cacheId, archivePath);
}
catch (error) {
utils.logWarning(error.message);
@ -5133,6 +5223,79 @@ var personalaccesstoken_1 = __webpack_require__(327);
exports.PersonalAccessTokenCredentialHandler = personalaccesstoken_1.PersonalAccessTokenCredentialHandler;
/***/ }),
/***/ 943:
/***/ (function(__unusedmodule, exports, __webpack_require__) {
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
result["default"] = mod;
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
const exec_1 = __webpack_require__(986);
const io = __importStar(__webpack_require__(1));
const fs_1 = __webpack_require__(747);
function getTarPath() {
return __awaiter(this, void 0, void 0, function* () {
// Explicitly use BSD Tar on Windows
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
if (fs_1.existsSync(systemTar)) {
return systemTar;
}
}
return yield io.which("tar", true);
});
}
function execTar(args) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
try {
yield exec_1.exec(`"${yield getTarPath()}"`, args);
}
catch (error) {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`);
}
throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`);
}
});
}
function extractTar(archivePath, targetDirectory) {
return __awaiter(this, void 0, void 0, function* () {
// Create directory to extract tar into
yield io.mkdirP(targetDirectory);
const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
yield execTar(args);
});
}
exports.extractTar = extractTar;
function createTar(archivePath, sourceDirectory) {
return __awaiter(this, void 0, void 0, function* () {
const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
yield execTar(args);
});
}
exports.createTar = createTar;
/***/ }),
/***/ 986:

58
examples.md
View file

@ -1,27 +1,20 @@
# Examples
- [Examples](#examples)
- [C# - Nuget](#c---nuget)
- [Elixir - Mix](#elixir---mix)
- [Go - Modules](#go---modules)
- [Java - Gradle](#java---gradle)
- [Java - Maven](#java---maven)
- [Node - npm](#node---npm)
- [macOS and Ubuntu](#macos-and-ubuntu)
- [Windows](#windows)
- [Using multiple systems and `npm config`](#using-multiple-systems-and-npm-config)
- [Node - Yarn](#node---yarn)
- [PHP - Composer](#php---composer)
- [Python - pip](#python---pip)
- [Simple example](#simple-example)
- [Multiple OS's in a workflow](#multiple-oss-in-a-workflow)
- [Using a script to get cache location](#using-a-script-to-get-cache-location)
- [Ruby - Gem](#ruby---gem)
- [Rust - Cargo](#rust---cargo)
- [Swift, Objective-C - Carthage](#swift-objective-c---carthage)
- [Swift, Objective-C - CocoaPods](#swift-objective-c---cocoapods)
- [C# - NuGet](#c---nuget)
- [Elixir - Mix](#elixir---mix)
- [Go - Modules](#go---modules)
- [Java - Gradle](#java---gradle)
- [Java - Maven](#java---maven)
- [Node - npm](#node---npm)
- [Node - Yarn](#node---yarn)
- [PHP - Composer](#php---composer)
- [Python - pip](#python---pip)
- [Ruby - Gem](#ruby---gem)
- [Rust - Cargo](#rust---cargo)
- [Swift, Objective-C - Carthage](#swift-objective-c---carthage)
- [Swift, Objective-C - CocoaPods](#swift-objective-c---cocoapods)
## C# - Nuget
## C# - NuGet
Using [NuGet lock files](https://docs.microsoft.com/nuget/consume-packages/package-references-in-project-files#locking-dependencies):
```yaml
@ -33,6 +26,21 @@ Using [NuGet lock files](https://docs.microsoft.com/nuget/consume-packages/packa
${{ runner.os }}-nuget-
```
Depending on the environment, huge packages might be pre-installed in the global cache folder.
If you do not want to include them, consider to move the cache folder like below.
>Note: This workflow does not work for projects that require files to be placed in user profile package folder
```yaml
env:
NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
steps:
- uses: actions/cache@v1
with:
path: ${{ github.workspace }}/.nuget/packages
key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
restore-keys: |
${{ runner.os }}-nuget-
```
## Elixir - Mix
```yaml
- uses: actions/cache@v1
@ -226,6 +234,14 @@ Replace `~/.cache/pip` with the correct `path` if not using Ubuntu.
restore-keys: |
${{ runner.os }}-gem-
```
When dependencies are installed later in the workflow, we must specify the same path for the bundler.
```yaml
- name: Bundle install
run: |
bundle config path vendor/bundle
bundle install --jobs 4 --retry 3
```
## Rust - Cargo

26
package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "cache",
"version": "1.0.3",
"version": "1.1.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -2854,9 +2854,9 @@
"dev": true
},
"handlebars": {
"version": "4.5.1",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.1.tgz",
"integrity": "sha512-C29UoFzHe9yM61lOsIlCE5/mQVGrnIOrOq7maQl76L7tYPCgC1og0Ajt6uWnX4ZTxBPnjw+CUvawphwCfJgUnA==",
"version": "4.5.3",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.3.tgz",
"integrity": "sha512-3yPecJoJHK/4c6aZhSvxOyG4vJKDshV36VHp0iVCDVh7o9w2vwi3NSnL2MMPj3YdduqaBcu7cGbggJQM0br9xA==",
"dev": true,
"requires": {
"neo-async": "^2.6.0",
@ -4859,9 +4859,9 @@
"dev": true
},
"prettier": {
"version": "1.18.2",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-1.18.2.tgz",
"integrity": "sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw==",
"version": "1.19.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz",
"integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==",
"dev": true
},
"prettier-linter-helpers": {
@ -5983,15 +5983,15 @@
}
},
"typescript": {
"version": "3.6.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.4.tgz",
"integrity": "sha512-unoCll1+l+YK4i4F8f22TaNVPRHcD9PA3yCuZ8g5e0qGqlVlJ/8FSateOLLSagn+Yg5+ZwuPkL8LFUc0Jcvksg==",
"version": "3.7.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.3.tgz",
"integrity": "sha512-Mcr/Qk7hXqFBXMN7p7Lusj1ktCBydylfQM/FZCk5glCNQJrCUKPkMHdo9R0MTFWsC/4kPFvDS0fDPvukfCkFsw==",
"dev": true
},
"uglify-js": {
"version": "3.6.7",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.7.tgz",
"integrity": "sha512-4sXQDzmdnoXiO+xvmTzQsfIiwrjUCSA95rSP4SEd8tDb51W2TiDOlL76Hl+Kw0Ie42PSItCW8/t6pBNCF2R48A==",
"version": "3.7.3",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.3.tgz",
"integrity": "sha512-7tINm46/3puUA4hCkKYo4Xdts+JDaVC9ZPRcG8Xw9R4nhO/gZgUM3TENq8IF4Vatk8qCig4MzP/c8G4u2BkVQg==",
"dev": true,
"optional": true,
"requires": {

6
package.json
View file

@ -1,6 +1,6 @@
{
"name": "cache",
"version": "1.0.3",
"version": "1.1.0",
"private": true,
"description": "Cache dependencies and build outputs",
"main": "dist/restore/index.js",
@ -46,8 +46,8 @@
"jest": "^24.8.0",
"jest-circus": "^24.7.1",
"nock": "^11.7.0",
"prettier": "1.18.2",
"prettier": "^1.19.1",
"ts-jest": "^24.0.2",
"typescript": "^3.6.4"
"typescript": "^3.7.3"
}
}

250
src/cacheHttpClient.ts
View file

@ -1,26 +1,49 @@
import * as core from "@actions/core";
import * as fs from "fs";
import { BearerCredentialHandler } from "typed-rest-client/Handlers";
import { HttpClient } from "typed-rest-client/HttpClient";
import { HttpClient, HttpCodes } from "typed-rest-client/HttpClient";
import { IHttpClientResponse } from "typed-rest-client/Interfaces";
import { IRequestOptions, RestClient } from "typed-rest-client/RestClient";
import { ArtifactCacheEntry } from "./contracts";
import {
IRequestOptions,
RestClient,
IRestResponse
} from "typed-rest-client/RestClient";
import {
ArtifactCacheEntry,
CommitCacheRequest,
ReserveCacheRequest,
ReserveCacheResponse
} from "./contracts";
import * as utils from "./utils/actionUtils";
function getCacheUrl(): string {
function isSuccessStatusCode(statusCode: number): boolean {
return statusCode >= 200 && statusCode < 300;
}
function isRetryableStatusCode(statusCode: number): boolean {
const retryableStatusCodes = [
HttpCodes.BadGateway,
HttpCodes.ServiceUnavailable,
HttpCodes.GatewayTimeout
];
return retryableStatusCodes.includes(statusCode);
}
function getCacheApiUrl(): string {
// Ideally we just use ACTIONS_CACHE_URL
const cacheUrl: string = (
const baseUrl: string = (
process.env["ACTIONS_CACHE_URL"] ||
process.env["ACTIONS_RUNTIME_URL"] ||
""
).replace("pipelines", "artifactcache");
if (!cacheUrl) {
if (!baseUrl) {
throw new Error(
"Cache Service Url not found, unable to restore cache."
);
}
core.debug(`Cache Url: ${cacheUrl}`);
return cacheUrl;
core.debug(`Cache Url: ${baseUrl}`);
return `${baseUrl}_apis/artifactcache/`;
}
function createAcceptHeader(type: string, apiVersion: string): string {
@ -29,26 +52,26 @@ function createAcceptHeader(type: string, apiVersion: string): string {
function getRequestOptions(): IRequestOptions {
const requestOptions: IRequestOptions = {
acceptHeader: createAcceptHeader("application/json", "5.2-preview.1")
acceptHeader: createAcceptHeader("application/json", "6.0-preview.1")
};
return requestOptions;
}
export async function getCacheEntry(
keys: string[]
): Promise<ArtifactCacheEntry | null> {
const cacheUrl = getCacheUrl();
function createRestClient(): RestClient {
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent(
keys.join(",")
)}`;
const restClient = new RestClient("actions/cache", cacheUrl, [
return new RestClient("actions/cache", getCacheApiUrl(), [
bearerCredentialHandler
]);
}
export async function getCacheEntry(
keys: string[]
): Promise<ArtifactCacheEntry | null> {
const restClient = createRestClient();
const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
const response = await restClient.get<ArtifactCacheEntry>(
resource,
@ -57,14 +80,15 @@ export async function getCacheEntry(
if (response.statusCode === 204) {
return null;
}
if (response.statusCode !== 200) {
if (!isSuccessStatusCode(response.statusCode)) {
throw new Error(`Cache service responded with ${response.statusCode}`);
}
const cacheResult = response.result;
if (!cacheResult || !cacheResult.archiveLocation) {
const cacheDownloadUrl = cacheResult?.archiveLocation;
if (!cacheDownloadUrl) {
throw new Error("Cache not found.");
}
core.setSecret(cacheResult.archiveLocation);
core.setSecret(cacheDownloadUrl);
core.debug(`Cache Result:`);
core.debug(JSON.stringify(cacheResult));
@ -83,46 +107,178 @@ async function pipeResponseToStream(
}
export async function downloadCache(
cacheEntry: ArtifactCacheEntry,
archiveLocation: string,
archivePath: string
): Promise<void> {
const stream = fs.createWriteStream(archivePath);
const httpClient = new HttpClient("actions/cache");
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const downloadResponse = await httpClient.get(cacheEntry.archiveLocation!);
const downloadResponse = await httpClient.get(archiveLocation);
await pipeResponseToStream(downloadResponse, stream);
}
export async function saveCache(
key: string,
archivePath: string
// Reserve Cache
export async function reserveCache(key: string): Promise<number> {
const restClient = createRestClient();
const reserveCacheRequest: ReserveCacheRequest = {
key
};
const response = await restClient.create<ReserveCacheResponse>(
"caches",
reserveCacheRequest,
getRequestOptions()
);
return response?.result?.cacheId ?? -1;
}
function getContentRange(start: number, end: number): string {
// Format: `bytes start-end/filesize
// start and end are inclusive
// filesize can be *
// For a 200 byte chunk starting at byte 0:
// Content-Range: bytes 0-199/*
return `bytes ${start}-${end}/*`;
}
async function uploadChunk(
restClient: RestClient,
resourceUrl: string,
data: NodeJS.ReadableStream,
start: number,
end: number
): Promise<void> {
const stream = fs.createReadStream(archivePath);
const cacheUrl = getCacheUrl();
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`;
const postUrl = cacheUrl + resource;
const restClient = new RestClient("actions/cache", undefined, [
bearerCredentialHandler
]);
core.debug(
`Uploading chunk of size ${end -
start +
1} bytes at offset ${start} with content range: ${getContentRange(
start,
end
)}`
);
const requestOptions = getRequestOptions();
requestOptions.additionalHeaders = {
"Content-Type": "application/octet-stream"
"Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end)
};
const response = await restClient.uploadStream<void>(
"POST",
postUrl,
stream,
const uploadChunkRequest = async (): Promise<IRestResponse<void>> => {
return await restClient.uploadStream<void>(
"PATCH",
resourceUrl,
data,
requestOptions
);
};
const response = await uploadChunkRequest();
if (isSuccessStatusCode(response.statusCode)) {
return;
}
if (isRetryableStatusCode(response.statusCode)) {
core.debug(
`Received ${response.statusCode}, retrying chunk at offset ${start}.`
);
const retryResponse = await uploadChunkRequest();
if (isSuccessStatusCode(retryResponse.statusCode)) {
return;
}
}
throw new Error(
`Cache service responded with ${response.statusCode} during chunk upload.`
);
}
async function uploadFile(
restClient: RestClient,
cacheId: number,
archivePath: string
): Promise<void> {
// Upload Chunks
const fileSize = fs.statSync(archivePath).size;
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
const fd = fs.openSync(archivePath, "r");
const concurrency = Number(process.env["CACHE_UPLOAD_CONCURRENCY"]) ?? 4; // # of HTTP requests in parallel
const MAX_CHUNK_SIZE =
Number(process.env["CACHE_UPLOAD_CHUNK_SIZE"]) ?? 32 * 1024 * 1024; // 32 MB Chunks
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parallelUploads = [...new Array(concurrency).keys()];
core.debug("Awaiting all uploads");
let offset = 0;
try {
await Promise.all(
parallelUploads.map(async () => {
while (offset < fileSize) {
const chunkSize = Math.min(
fileSize - offset,
MAX_CHUNK_SIZE
);
const start = offset;
const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
await uploadChunk(
restClient,
resourceUrl,
chunk,
start,
end
);
}
})
);
} finally {
fs.closeSync(fd);
}
return;
}
async function commitCache(
restClient: RestClient,
cacheId: number,
filesize: number
): Promise<IRestResponse<void>> {
const requestOptions = getRequestOptions();
const commitCacheRequest: CommitCacheRequest = { size: filesize };
return await restClient.create(
`caches/${cacheId.toString()}`,
commitCacheRequest,
requestOptions
);
if (response.statusCode !== 200) {
throw new Error(`Cache service responded with ${response.statusCode}`);
}
export async function saveCache(
cacheId: number,
archivePath: string
): Promise<void> {
const restClient = createRestClient();
core.debug("Upload cache");
await uploadFile(restClient, cacheId, archivePath);
// Commit Cache
core.debug("Commiting cache");
const cacheSize = utils.getArchiveFileSize(archivePath);
const commitCacheResponse = await commitCache(
restClient,
cacheId,
cacheSize
);
if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
throw new Error(
`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`
);
}
core.info("Cache saved successfully");

13
src/contracts.d.ts vendored
View file

@ -4,3 +4,16 @@ export interface ArtifactCacheEntry {
creationTime?: string;
archiveLocation?: string;
}
export interface CommitCacheRequest {
size: number;
}
export interface ReserveCacheRequest {
key: string;
version?: string;
}
export interface ReserveCacheResponse {
cacheId: number;
}

32
src/restore.ts
View file

@ -1,9 +1,8 @@
import * as core from "@actions/core";
import { exec } from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path";
import * as cacheHttpClient from "./cacheHttpClient";
import { Events, Inputs, State } from "./constants";
import { extractTar } from "./tar";
import * as utils from "./utils/actionUtils";
async function run(): Promise<void> {
@ -61,7 +60,7 @@ async function run(): Promise<void> {
try {
const cacheEntry = await cacheHttpClient.getCacheEntry(keys);
if (!cacheEntry) {
if (!cacheEntry?.archiveLocation) {
core.info(
`Cache not found for input keys: ${keys.join(", ")}.`
);
@ -78,7 +77,10 @@ async function run(): Promise<void> {
utils.setCacheState(cacheEntry);
// Download the cache from the cache entry
await cacheHttpClient.downloadCache(cacheEntry, archivePath);
await cacheHttpClient.downloadCache(
cacheEntry.archiveLocation,
archivePath
);
const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.info(
@ -87,27 +89,7 @@ async function run(): Promise<void> {
)} MB (${archiveFileSize} B)`
);
// Create directory to extract tar into
await io.mkdirP(cachePath);
// http://man7.org/linux/man-pages/man1/tar.1.html
// tar [-options] <name of the tar archive> [files or directories which to add into archive]
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
const tarPath = await io.which("tar", true);
core.debug(`Tar Path: ${tarPath}`);
await exec(`"${tarPath}"`, args);
await extractTar(archivePath, cachePath);
const isExactKeyMatch = utils.isExactKeyMatch(
primaryKey,

38
src/save.ts
View file

@ -1,9 +1,8 @@
import * as core from "@actions/core";
import { exec } from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path";
import * as cacheHttpClient from "./cacheHttpClient";
import { Events, Inputs, State } from "./constants";
import { createTar } from "./tar";
import * as utils from "./utils/actionUtils";
async function run(): Promise<void> {
@ -35,6 +34,15 @@ async function run(): Promise<void> {
return;
}
core.debug("Reserving Cache");
const cacheId = await cacheHttpClient.reserveCache(primaryKey);
if (cacheId == -1) {
core.info(
`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
);
return;
}
core.debug(`Cache ID: ${cacheId}`);
const cachePath = utils.resolvePath(
core.getInput(Inputs.Path, { required: true })
);
@ -46,38 +54,22 @@ async function run(): Promise<void> {
);
core.debug(`Archive Path: ${archivePath}`);
// http://man7.org/linux/man-pages/man1/tar.1.html
// tar [-options] <name of the tar archive> [files or directories which to add into archive]
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
await createTar(archivePath, cachePath);
const tarPath = await io.which("tar", true);
core.debug(`Tar Path: ${tarPath}`);
await exec(`"${tarPath}"`, args);
const fileSizeLimit = 400 * 1024 * 1024; // 400MB
const fileSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB per repo limit
const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.debug(`File Size: ${archiveFileSize}`);
if (archiveFileSize > fileSizeLimit) {
utils.logWarning(
`Cache size of ~${Math.round(
archiveFileSize / (1024 * 1024)
)} MB (${archiveFileSize} B) is over the 400MB limit, not saving cache.`
)} MB (${archiveFileSize} B) is over the 2GB limit, not saving cache.`
);
return;
}
await cacheHttpClient.saveCache(primaryKey, archivePath);
core.debug(`Saving Cache (ID: ${cacheId})`);
await cacheHttpClient.saveCache(cacheId, archivePath);
} catch (error) {
utils.logWarning(error.message);
}

47
src/tar.ts Normal file
View file

@ -0,0 +1,47 @@
import { exec } from "@actions/exec";
import * as io from "@actions/io";
import { existsSync } from "fs";
async function getTarPath(): Promise<string> {
// Explicitly use BSD Tar on Windows
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
if (existsSync(systemTar)) {
return systemTar;
}
}
return await io.which("tar", true);
}
async function execTar(args: string[]): Promise<void> {
try {
await exec(`"${await getTarPath()}"`, args);
} catch (error) {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
throw new Error(
`Tar failed with error: ${error?.message}. Ensure BSD tar is installed and on the PATH.`
);
}
throw new Error(`Tar failed with error: ${error?.message}`);
}
}
export async function extractTar(
archivePath: string,
targetDirectory: string
): Promise<void> {
// Create directory to extract tar into
await io.mkdirP(targetDirectory);
const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
await execTar(args);
}
export async function createTar(
archivePath: string,
sourceDirectory: string
): Promise<void> {
const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
await execTar(args);
}