Initial pass at chunked upload apis

This commit is contained in:
Josh Gross 2019-12-13 15:19:25 -05:00
parent 4809f4ada4
commit bad827c28e
6 changed files with 158 additions and 50 deletions

View File

@ -200,7 +200,7 @@ test("save with large cache outputs warning", async () => {
const execMock = jest.spyOn(exec, "exec"); const execMock = jest.spyOn(exec, "exec");
const cacheSize = 1024 * 1024 * 1024; //~1GB, over the 400MB limit const cacheSize = 4 * 1024 * 1024 * 1024; //~4GB, over the 2GB limit
jest.spyOn(actionUtils, "getArchiveFileSize").mockImplementationOnce(() => { jest.spyOn(actionUtils, "getArchiveFileSize").mockImplementationOnce(() => {
return cacheSize; return cacheSize;
}); });
@ -227,7 +227,7 @@ test("save with large cache outputs warning", async () => {
expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith( expect(logWarningMock).toHaveBeenCalledWith(
"Cache size of ~1024 MB (1073741824 B) is over the 400MB limit, not saving cache." "Cache size of ~4 GB (4294967296 B) is over the 2GB limit, not saving cache."
); );
expect(failedMock).toHaveBeenCalledTimes(0); expect(failedMock).toHaveBeenCalledTimes(0);

12
package-lock.json generated
View File

@ -4859,9 +4859,9 @@
"dev": true "dev": true
}, },
"prettier": { "prettier": {
"version": "1.18.2", "version": "1.19.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-1.18.2.tgz", "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz",
"integrity": "sha512-OeHeMc0JhFE9idD4ZdtNibzY0+TPHSpSSb9h8FqtP+YnoZZ1sl8Vc9b1sasjfymH3SonAF4QcA2+mzHPhMvIiw==", "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==",
"dev": true "dev": true
}, },
"prettier-linter-helpers": { "prettier-linter-helpers": {
@ -5983,9 +5983,9 @@
} }
}, },
"typescript": { "typescript": {
"version": "3.6.4", "version": "3.7.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.4.tgz", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.3.tgz",
"integrity": "sha512-unoCll1+l+YK4i4F8f22TaNVPRHcD9PA3yCuZ8g5e0qGqlVlJ/8FSateOLLSagn+Yg5+ZwuPkL8LFUc0Jcvksg==", "integrity": "sha512-Mcr/Qk7hXqFBXMN7p7Lusj1ktCBydylfQM/FZCk5glCNQJrCUKPkMHdo9R0MTFWsC/4kPFvDS0fDPvukfCkFsw==",
"dev": true "dev": true
}, },
"uglify-js": { "uglify-js": {

View File

@ -1,6 +1,6 @@
{ {
"name": "cache", "name": "cache",
"version": "1.0.3", "version": "1.1.0",
"private": true, "private": true,
"description": "Cache dependencies and build outputs", "description": "Cache dependencies and build outputs",
"main": "dist/restore/index.js", "main": "dist/restore/index.js",
@ -46,8 +46,8 @@
"jest": "^24.8.0", "jest": "^24.8.0",
"jest-circus": "^24.7.1", "jest-circus": "^24.7.1",
"nock": "^11.7.0", "nock": "^11.7.0",
"prettier": "1.18.2", "prettier": "^1.19.1",
"ts-jest": "^24.0.2", "ts-jest": "^24.0.2",
"typescript": "^3.6.4" "typescript": "^3.7.3"
} }
} }

View File

@ -3,24 +3,39 @@ import * as fs from "fs";
import { BearerCredentialHandler } from "typed-rest-client/Handlers"; import { BearerCredentialHandler } from "typed-rest-client/Handlers";
import { HttpClient } from "typed-rest-client/HttpClient"; import { HttpClient } from "typed-rest-client/HttpClient";
import { IHttpClientResponse } from "typed-rest-client/Interfaces"; import { IHttpClientResponse } from "typed-rest-client/Interfaces";
import { IRequestOptions, RestClient } from "typed-rest-client/RestClient"; import {
import { ArtifactCacheEntry } from "./contracts"; IRequestOptions,
RestClient,
IRestResponse
} from "typed-rest-client/RestClient";
import {
ArtifactCacheEntry,
CommitCacheRequest,
ReserveCacheRequest,
ReserverCacheResponse
} from "./contracts";
import * as utils from "./utils/actionUtils";
function getCacheUrl(): string { const MAX_CHUNK_SIZE = 4000000; // 4 MB Chunks
function isSuccessStatusCode(statusCode: number): boolean {
return statusCode >= 200 && statusCode < 300;
}
function getCacheApiUrl(): string {
// Ideally we just use ACTIONS_CACHE_URL // Ideally we just use ACTIONS_CACHE_URL
const cacheUrl: string = ( const baseUrl: string = (
process.env["ACTIONS_CACHE_URL"] || process.env["ACTIONS_CACHE_URL"] ||
process.env["ACTIONS_RUNTIME_URL"] || process.env["ACTIONS_RUNTIME_URL"] ||
"" ""
).replace("pipelines", "artifactcache"); ).replace("pipelines", "artifactcache");
if (!cacheUrl) { if (!baseUrl) {
throw new Error( throw new Error(
"Cache Service Url not found, unable to restore cache." "Cache Service Url not found, unable to restore cache."
); );
} }
core.debug(`Cache Url: ${cacheUrl}`); core.debug(`Cache Url: ${baseUrl}`);
return cacheUrl; return `${baseUrl}_apis/artifactcache/`;
} }
function createAcceptHeader(type: string, apiVersion: string): string { function createAcceptHeader(type: string, apiVersion: string): string {
@ -29,7 +44,7 @@ function createAcceptHeader(type: string, apiVersion: string): string {
function getRequestOptions(): IRequestOptions { function getRequestOptions(): IRequestOptions {
const requestOptions: IRequestOptions = { const requestOptions: IRequestOptions = {
acceptHeader: createAcceptHeader("application/json", "5.2-preview.1") acceptHeader: createAcceptHeader("application/json", "6.0-preview.1")
}; };
return requestOptions; return requestOptions;
@ -38,13 +53,11 @@ function getRequestOptions(): IRequestOptions {
export async function getCacheEntry( export async function getCacheEntry(
keys: string[] keys: string[]
): Promise<ArtifactCacheEntry | null> { ): Promise<ArtifactCacheEntry | null> {
const cacheUrl = getCacheUrl(); const cacheUrl = getCacheApiUrl();
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new BearerCredentialHandler(token); const bearerCredentialHandler = new BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache?keys=${encodeURIComponent( const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
keys.join(",")
)}`;
const restClient = new RestClient("actions/cache", cacheUrl, [ const restClient = new RestClient("actions/cache", cacheUrl, [
bearerCredentialHandler bearerCredentialHandler
@ -57,14 +70,15 @@ export async function getCacheEntry(
if (response.statusCode === 204) { if (response.statusCode === 204) {
return null; return null;
} }
if (response.statusCode !== 200) { if (!isSuccessStatusCode(response.statusCode)) {
throw new Error(`Cache service responded with ${response.statusCode}`); throw new Error(`Cache service responded with ${response.statusCode}`);
} }
const cacheResult = response.result; const cacheResult = response.result;
if (!cacheResult || !cacheResult.archiveLocation) { const cacheDownloadUrl = cacheResult?.archiveLocation;
if (!cacheDownloadUrl) {
throw new Error("Cache not found."); throw new Error("Cache not found.");
} }
core.setSecret(cacheResult.archiveLocation); core.setSecret(cacheDownloadUrl);
core.debug(`Cache Result:`); core.debug(`Cache Result:`);
core.debug(JSON.stringify(cacheResult)); core.debug(JSON.stringify(cacheResult));
@ -83,46 +97,127 @@ async function pipeResponseToStream(
} }
export async function downloadCache( export async function downloadCache(
cacheEntry: ArtifactCacheEntry, archiveLocation: string,
archivePath: string archivePath: string
): Promise<void> { ): Promise<void> {
const stream = fs.createWriteStream(archivePath); const stream = fs.createWriteStream(archivePath);
const httpClient = new HttpClient("actions/cache"); const httpClient = new HttpClient("actions/cache");
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion const downloadResponse = await httpClient.get(archiveLocation);
const downloadResponse = await httpClient.get(cacheEntry.archiveLocation!);
await pipeResponseToStream(downloadResponse, stream); await pipeResponseToStream(downloadResponse, stream);
} }
// Returns Cache ID
async function reserveCache(
restClient: RestClient,
key: string
): Promise<number> {
const reserveCacheRequest: ReserveCacheRequest = {
key
};
const response = await restClient.create<ReserverCacheResponse>(
"caches",
reserveCacheRequest
);
return response?.result?.cacheId || -1;
}
function getContentRange(start: number, length: number): string {
// Format: `bytes start-end/filesize
// start and end are inclusive
// filesize can be *
// For a 200 byte chunk starting at byte 0:
// Content-Range: bytes 0-199/*
return `bytes ${start}-${start + length - 1}/*`;
}
async function uploadChunk(
restClient: RestClient,
cacheId: number,
data: Buffer,
offset: number
): Promise<IRestResponse<void>> {
const requestOptions = getRequestOptions();
requestOptions.additionalHeaders = {
"Content-Type": "application/octet-stream",
"Content-Range": getContentRange(offset, data.byteLength)
};
return await restClient.update(
cacheId.toString(),
data.toString("utf8"),
requestOptions
);
}
async function commitCache(
restClient: RestClient,
cacheId: number,
filesize: number
): Promise<IRestResponse<void>> {
const requestOptions = getRequestOptions();
const commitCacheRequest: CommitCacheRequest = { size: filesize };
return await restClient.create(
cacheId.toString(),
commitCacheRequest,
requestOptions
);
}
export async function saveCache( export async function saveCache(
key: string, key: string,
archivePath: string archivePath: string
): Promise<void> { ): Promise<void> {
const stream = fs.createReadStream(archivePath);
const cacheUrl = getCacheUrl();
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new BearerCredentialHandler(token); const bearerCredentialHandler = new BearerCredentialHandler(token);
const resource = `_apis/artifactcache/cache/${encodeURIComponent(key)}`; const restClient = new RestClient("actions/cache", getCacheApiUrl(), [
const postUrl = cacheUrl + resource;
const restClient = new RestClient("actions/cache", undefined, [
bearerCredentialHandler bearerCredentialHandler
]); ]);
const requestOptions = getRequestOptions(); // Reserve Cache
requestOptions.additionalHeaders = { const cacheId = await reserveCache(restClient, key);
"Content-Type": "application/octet-stream" if (cacheId < 0) {
}; throw new Error(`Unable to reserve cache.`);
}
const response = await restClient.uploadStream<void>( // Upload Chunks
"POST", const stream = fs.createReadStream(archivePath);
postUrl, let streamIsClosed = false;
stream, stream.on("close", () => {
requestOptions streamIsClosed = true;
});
const uploads: Promise<IRestResponse<void>>[] = [];
let offset = 0;
while (!streamIsClosed) {
const chunk: Buffer = stream.read(MAX_CHUNK_SIZE);
uploads.push(uploadChunk(restClient, cacheId, chunk, offset));
offset += MAX_CHUNK_SIZE;
}
const responses = await Promise.all(uploads);
const failedResponse = responses.find(
x => !isSuccessStatusCode(x.statusCode)
);
if (failedResponse) {
throw new Error(
`Cache service responded with ${failedResponse.statusCode} during chunk upload.`
);
}
// Commit Cache
const cacheSize = utils.getArchiveFileSize(archivePath);
const commitCacheResponse = await commitCache(
restClient,
cacheId,
cacheSize
);
if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
throw new Error(
`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`
); );
if (response.statusCode !== 200) {
throw new Error(`Cache service responded with ${response.statusCode}`);
} }
core.info("Cache saved successfully"); core.info("Cache saved successfully");

13
src/contracts.d.ts vendored
View File

@ -4,3 +4,16 @@ export interface ArtifactCacheEntry {
creationTime?: string; creationTime?: string;
archiveLocation?: string; archiveLocation?: string;
} }
export interface CommitCacheRequest {
size: number;
}
export interface ReserveCacheRequest {
key: string;
version?: string;
}
export interface ReserverCacheResponse {
cacheId: number;
}

View File

@ -65,14 +65,14 @@ async function run(): Promise<void> {
core.debug(`Tar Path: ${tarPath}`); core.debug(`Tar Path: ${tarPath}`);
await exec(`"${tarPath}"`, args); await exec(`"${tarPath}"`, args);
const fileSizeLimit = 400 * 1024 * 1024; // 400MB const fileSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB per repo limit
const archiveFileSize = utils.getArchiveFileSize(archivePath); const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.debug(`File Size: ${archiveFileSize}`); core.debug(`File Size: ${archiveFileSize}`);
if (archiveFileSize > fileSizeLimit) { if (archiveFileSize > fileSizeLimit) {
utils.logWarning( utils.logWarning(
`Cache size of ~${Math.round( `Cache size of ~${Math.round(
archiveFileSize / (1024 * 1024) archiveFileSize / (1024 * 1024 * 1024)
)} MB (${archiveFileSize} B) is over the 400MB limit, not saving cache.` )} GB (${archiveFileSize} B) is over the 2GB limit, not saving cache.`
); );
return; return;
} }