diff --git a/src/commands/cloud.ts b/src/commands/cloud.ts index 8eb2106..bedb952 100644 --- a/src/commands/cloud.ts +++ b/src/commands/cloud.ts @@ -35,7 +35,11 @@ import { isIosMatrixConfig, } from '../types/domain/device.types.js'; import { resolveAuth } from '../utils/auth.js'; -import { matrixIsIos, parseDeviceMatrix } from '../utils/device-matrix.js'; +import { + assertMatrixSupported, + matrixIsIos, + parseDeviceMatrix, +} from '../utils/device-matrix.js'; import { detectCiContext, isCI } from '../utils/ci.js'; import { CliError, @@ -822,6 +826,10 @@ export const cloudCommand = defineCommand({ // there is no matrix, and tolerant of older APIs that lack the endpoint. if (deviceMatrix.length > 0) { const estimate = await ApiGateway.estimateMatrix(apiUrl, auth, fields); + // A null estimate means the API predates the matrix (404/405). It would + // silently strip deviceMatrix and run one device — refuse rather than + // hand back a green single-device run the user reads as a matrix. + assertMatrixSupported(deviceMatrix, estimate); if (estimate) { const osPrefix = matrixIsIos(deviceMatrix) ? 'iOS' : 'API'; const rows = ui.fields([ diff --git a/src/utils/device-matrix.ts b/src/utils/device-matrix.ts index 8e9e5f9..7f171ae 100644 --- a/src/utils/device-matrix.ts +++ b/src/utils/device-matrix.ts @@ -4,6 +4,34 @@ import { } from '../types/domain/device.types.js'; import { CliError } from './cli.js'; +/** + * Refuse to submit a device matrix to an API that cannot honour it. + * + * The estimate endpoint is only called when a matrix was actually requested, so + * a null estimate (the gateway maps 404/405 to null) means the API predates the + * feature. That API would **silently strip** the unknown `deviceMatrix` field — + * its ValidationPipe runs `whitelist: true, forbidNonWhitelisted: false` — and + * run every flow on a single default device, exiting 0. The user would believe + * they had tested N devices when they tested one: the exact silent + * under-testing the device matrix exists to prevent. Fail loudly instead. + * + * @throws CliError when a matrix was requested but the API does not support it. + */ +export function assertMatrixSupported( + deviceMatrix: DeviceMatrixConfig[], + estimate: unknown | null, +): void { + if (deviceMatrix.length === 0 || estimate) return; + + throw new CliError( + 'This DeviceCloud API does not support device matrices, so ' + + '--ios-device-matrix / --android-device-matrix cannot be honoured. ' + + 'Submitting anyway would silently run every flow on a single default ' + + 'device and report success. Upgrade the API, or drop the matrix flags ' + + 'and use --ios-device / --android-device for a single-device run.', + ); +} + /** * Parse repeated `--ios-device-matrix :` and * `--android-device-matrix :[:play]` flags into an explicit device diff --git a/test/unit/device-matrix.test.ts b/test/unit/device-matrix.test.ts index 8a4cb11..20476a1 100644 --- a/test/unit/device-matrix.test.ts +++ b/test/unit/device-matrix.test.ts @@ -2,6 +2,7 @@ import { expect } from 'chai'; import { CliError } from '../../src/utils/cli.js'; import { + assertMatrixSupported, matrixIsIos, parseDeviceMatrix, } from '../../src/utils/device-matrix.js'; @@ -59,3 +60,36 @@ describe('parseDeviceMatrix', () => { ); }); }); + +/** + * An API that predates the matrix silently STRIPS the unknown deviceMatrix + * field (its ValidationPipe is whitelist:true / forbidNonWhitelisted:false) and + * runs every flow on one default device, exiting 0. Submitting into that is the + * worst outcome the feature can produce, so it must be refused, not tolerated. + */ +describe('assertMatrixSupported', () => { + const matrix = [{ iOSDevice: 'iphone-16', iOSVersion: '18' }]; + + it('throws when a matrix was requested but the API has no estimate endpoint', () => { + expect(() => assertMatrixSupported(matrix, null)).to.throw( + CliError, + /does not support device matrices/i, + ); + }); + + it('explains that submitting anyway would silently run a single device', () => { + expect(() => assertMatrixSupported(matrix, null)).to.throw( + /silently run every flow on a single default device/i, + ); + }); + + it('passes when the API returned an estimate', () => { + expect(() => + assertMatrixSupported(matrix, { cellCount: 2, totalCost: 0.16 }), + ).to.not.throw(); + }); + + it('is a no-op when no matrix was requested (legacy single-device runs)', () => { + expect(() => assertMatrixSupported([], null)).to.not.throw(); + }); +});