From aa9b3bd693b717590528389baa0da6379c0d200c Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Tue, 17 Mar 2026 14:16:22 -0600 Subject: [PATCH 1/8] code to test Timezone Offset --- .../schema/cql-test-configuration.schema.json | 78 +++--- conf/localhost.json | 18 +- src/bin/cql-tests.ts | 11 +- src/commands/run-tests-command.ts | 6 +- src/conf/config-loader.ts | 14 +- src/cql-engine/cql-engine.ts | 116 ++++++--- src/jobs/job-processor.ts | 2 +- src/models/config-types.ts | 1 + src/models/results-types.ts | 1 + src/models/test-types.ts | 2 + src/server/config-utils.ts | 18 +- src/services/test-runner.ts | 246 ------------------ src/shared/results-shared.ts | 16 +- src/test-results/cql-test-results.ts | 2 +- test/server-command.test.ts | 33 +-- 15 files changed, 196 insertions(+), 368 deletions(-) delete mode 100644 src/services/test-runner.ts diff --git a/assets/schema/cql-test-configuration.schema.json b/assets/schema/cql-test-configuration.schema.json index 51ea5dc..121416b 100644 --- a/assets/schema/cql-test-configuration.schema.json +++ b/assets/schema/cql-test-configuration.schema.json @@ -31,41 +31,49 @@ "type": "object", "description": "Build configuration settings", "properties": { - "CqlFileVersion": { - "type": "string", - "pattern": "^\\d+\\.\\d+\\.\\d+$", - "description": "CQL file version in semantic versioning format" - }, - "CqlOutputPath": { - "type": "string", - "description": "Path where CQL files are output" - }, - "CqlVersion": { - "type": "string", - "pattern": "^\\d+\\.\\d+(\\.\\d+)?$", - "description": "CQL engine version for version-based test filtering (optional)" - }, - "testsRunDescription":{ - "type": "string", - "description": "Information about this test run" - }, - "cqlTranslator": { - "type": "string", - "description": "Which CQL Translator is used in this test run." - }, - "cqlTranslatorVersion": { - "type": "string", - "description": "Which CQL translator version is used in this test run" - }, - "cqlEngine": { - "type": "string", - "description": "Which CQL engine is used in this test run" - }, - "cqlEngineVersion": { - "type": "string", - "description": "Which version of the CQL engine is used in this test run" - } - }, + "CqlFileVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "description": "CQL file version in semantic versioning format" + }, + "CqlOutputPath": { + "type": "string", + "description": "Path where CQL files are output" + }, + "CqlVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+(\\.\\d+)?$", + "description": "CQL engine version for version-based test filtering (optional)" + }, + "testsRunDescription": { + "type": "string", + "description": "Information about this test run" + }, + "cqlTranslator": { + "type": "string", + "description": "Which CQL Translator is used in this test run." + }, + "cqlTranslatorVersion": { + "type": "string", + "description": "Which CQL translator version is used in this test run" + }, + "cqlEngine": { + "type": "string", + "description": "Which CQL engine is used in this test run" + }, + "cqlEngineVersion": { + "type": "string", + "description": "Which version of the CQL engine is used in this test run" + }, + "SERVER_OFFSET_ISO": { + "type": "string", + "description": "ISO 8601 formatted date string to offset the server start time" + }, + "TimeZoneOffsetPolicy": { + "type": "string", + "description": "Which timezone offset policy is used in this test run." + } + }, "required": ["CqlFileVersion", "CqlOutputPath"], "additionalProperties": false }, diff --git a/conf/localhost.json b/conf/localhost.json index 1b711a0..2b54a2e 100644 --- a/conf/localhost.json +++ b/conf/localhost.json @@ -4,14 +4,16 @@ "CqlOperation": "$cql" }, "Build": { - "CqlFileVersion": "1.0.000", - "CqlOutputPath": "./cql", - "CqlVersion": "1.5", - "testsRunDescription": "Local host test run", - "cqlTranslator": "Java CQFramework Translator", - "cqlTranslatorVersion": "Unknown", - "cqlEngine": "Java CQFramework Engine", - "cqlEngineVersion": "4.1.0" + "CqlFileVersion": "1.0.000", + "CqlOutputPath": "./cql", + "CqlVersion": "1.5", + "testsRunDescription": "Local host test run", + "cqlTranslator": "Java CQFramework Translator", + "cqlTranslatorVersion": "Unknown", + "cqlEngine": "Java CQFramework Engine", + "cqlEngineVersion": "4.1.0", + "SERVER_OFFSET_ISO": "-06:00", + "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" }, "Debug": { "QuickTest": false diff --git a/src/bin/cql-tests.ts b/src/bin/cql-tests.ts index db035a5..542d6d0 100644 --- a/src/bin/cql-tests.ts +++ b/src/bin/cql-tests.ts @@ -43,7 +43,8 @@ program ); process.exit(1); } - throw error; + console.error(error); + process.exit(1); } }); @@ -73,7 +74,8 @@ program ); process.exit(1); } - throw error; + console.error(error); + process.exit(1); } }); @@ -92,4 +94,7 @@ program await serverCommand.start(); }); -program.parse(); +program.parseAsync().catch((error: any) => { + console.error(error); + process.exit(1); +}); diff --git a/src/commands/run-tests-command.ts b/src/commands/run-tests-command.ts index 803e57c..3b601f4 100644 --- a/src/commands/run-tests-command.ts +++ b/src/commands/run-tests-command.ts @@ -1,4 +1,4 @@ -import { TestRunner } from '../services/test-runner.js'; +import { TestRunner } from '../services/test-runner'; import { ConfigLoader } from '../conf/config-loader.js'; // Type declaration for CVL loader @@ -54,7 +54,9 @@ export class RunCommand { cqlTranslator: config.Build?.cqlTranslator, cqlTranslatorVersion: config.Build?.cqlTranslatorVersion, cqlEngine: config.Build?.cqlEngine, - cqlEngineVersion: config.Build?.cqlEngineVersion + cqlEngineVersion: config.Build?.cqlEngineVersion, + SERVER_OFFSET_ISO: config.Build?.SERVER_OFFSET_ISO, + TimeZoneOffsetPolicy: config.Build?.TimeZoneOffsetPolicy, }, Tests: { ResultsPath: config.Tests.ResultsPath, diff --git a/src/conf/config-loader.ts b/src/conf/config-loader.ts index f53ca3b..ff7bcbe 100644 --- a/src/conf/config-loader.ts +++ b/src/conf/config-loader.ts @@ -17,6 +17,8 @@ export class ConfigLoader implements Config { cqlTranslatorVersion: string; cqlEngine: string; cqlEngineVersion: string; + SERVER_OFFSET_ISO: string; + TimeZoneOffsetPolicy: string; }; Tests: { ResultsPath: string; @@ -55,14 +57,18 @@ export class ConfigLoader implements Config { process.env.CQL_VERSION || configData.Build?.CqlVersion, testsRunDescription: process.env.TESTS_RUN_DESCRIPTION || configData.Build?.testsRunDescription, - cqlTranslator: + cqlTranslator: process.env.CQL_TRANSLATOR || configData.Build?.cqlTranslator || 'Unknown', - cqlTranslatorVersion: + cqlTranslatorVersion: process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'Unknown', cqlEngine: process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'Unknown', - cqlEngineVersion: - process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'Unknown' + cqlEngineVersion: + process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'Unknown', + SERVER_OFFSET_ISO: + process.env.SERVER_OFFSET_ISO || configData.Build?.SERVER_OFFSET_ISO || '+00:00', + TimeZoneOffsetPolicy: + process.env.TIME_ZONE_OFFSET_POLICY || configData.Build?.TimeZoneOffsetPolicy || '', }; this.Tests = { diff --git a/src/cql-engine/cql-engine.ts b/src/cql-engine/cql-engine.ts index 7754784..e2730d5 100644 --- a/src/cql-engine/cql-engine.ts +++ b/src/cql-engine/cql-engine.ts @@ -1,5 +1,6 @@ import axios, { AxiosResponse } from 'axios'; import { CQLEngineInfo } from '../models/results-types.js'; +import { response } from 'express'; /** * Represents a CQL Engine. @@ -39,19 +40,36 @@ export class CQLEngine { private baseURL?: string; private metadata?: any; private description?: string; + private _SERVER_OFFSET_ISO?: string; - /** - * Creates an instance of CQLEngine. - * @param baseURL - The base URL for the CQL engine. - * @param cqlPath - The path for the CQL engine (optional). - */ - constructor(baseURL: string, cqlPath: string | null = null, - cqlTranslator: string, cqlTranslatorVersion: string, - cqlEngine: string, cqlEngineVersion: string) { - this._prepareBaseURL(baseURL, cqlPath); - this._setInformationFields(cqlTranslator, cqlTranslatorVersion, - cqlEngine, cqlEngineVersion); - } + /** + * Creates a new CQLEngine instance. + * + * @param baseURL Base URL of the FHIR server (e.g. http://localhost:8080/fhir/$cql) + * @param cqlPath Optional path to local CQL files + * @param cqlTranslator Name of the CQL translator + * @param cqlTranslatorVersion Version of the CQL translator + * @param cqlEngine Name of the CQL engine + * @param cqlEngineVersion Version of the CQL engine + * @param SERVER_OFFSET_ISO Timezone offset used by the server (e.g. "+00:00", "-06:00") + */ constructor( + baseURL: string, + cqlPath: string | null = null, + cqlTranslator: string = '', + cqlTranslatorVersion: string = '', + cqlEngine: string = '', + cqlEngineVersion: string = '', + SERVER_OFFSET_ISO: string = '' + ) { + this._prepareBaseURL(baseURL, cqlPath); + this._setInformationFields( + cqlTranslator, + cqlTranslatorVersion, + cqlEngine, + cqlEngineVersion + ); + this.SERVER_OFFSET_ISO = SERVER_OFFSET_ISO; + } /** * Prepares the base URL. @@ -86,31 +104,31 @@ export class CQLEngine { private _setInformationFields(cqlTranslator: string, cqlTranslatorVersion: string, cqlEngine: string, cqlEngineVersion: string) { - this.info.cqlTranslator = cqlTranslator; - this.info.cqlTranslatorVersion = cqlTranslatorVersion; - this.info.cqlEngine = cqlEngine; - this.info.cqlEngineVersion = cqlEngineVersion; - } - - /** - * Fetches metadata from the CQL engine. - * @param force - Whether to force fetching metadata. - * @returns A Promise that resolves when metadata is fetched. - */ - async fetch(force: boolean = false): Promise { - if (this.baseURL) { - if (!this.metadata || force) { - try { - const response: AxiosResponse = await axios.get(`${this.baseURL}/metadata`); - if (response?.data) { - this.metadata = response.data; - } - } catch (e) { - console.error(e); - } - } - } - } + this.info.cqlTranslator = cqlTranslator; + this.info.cqlTranslatorVersion = cqlTranslatorVersion; + this.info.cqlEngine = cqlEngine; + this.info.cqlEngineVersion = cqlEngineVersion; + } + + /** + * Fetches metadata from the CQL engine. + * @param force - Whether to force fetching metadata. + * @returns A Promise that resolves when metadata is fetched. + */ + async fetch(force: boolean = false): Promise { + if (this.baseURL) { + if (!this.metadata || force) { + try { + const response: AxiosResponse = await axios.get(`${this.baseURL}/metadata`); + if (response?.data) { + this.metadata = response.data; + } + } catch (e) { + console.error(e); + } + } + } + } /** * Sets the API URL. @@ -208,6 +226,14 @@ export class CQLEngine { return this.info?.cqlEngineVersion ?? null; } + get SERVER_OFFSET_ISO(): string { + return this._SERVER_OFFSET_ISO; + } + + set SERVER_OFFSET_ISO(value: string) { + this._SERVER_OFFSET_ISO = value; + } + /** * Converts the CQLEngine object to JSON. * @returns The JSON representation of the CQLEngine object. @@ -222,6 +248,7 @@ export class CQLEngine { cqlTranslatorVersion: this.info.cqlTranslatorVersion || '', cqlEngine: this.info.cqlEngine || '', cqlEngineVersion: this.info.cqlEngineVersion || '', + SERVER_OFFSET_ISO: this.SERVER_OFFSET_ISO || '', }; } @@ -231,9 +258,15 @@ export class CQLEngine { * @returns The CQLEngine instance. */ static fromJSON(cqlInfo: CQLEngineInfo): CQLEngine { - const engine = new CQLEngine(cqlInfo.apiUrl || '', null, - cqlInfo.cqlTranslator, cqlInfo.cqlTranslatorVersion, - cqlInfo.cqlEngine, cqlInfo.cqlEngineVersion); + const engine = new CQLEngine( + cqlInfo.apiUrl || '', + null, + cqlInfo.cqlTranslator, + cqlInfo.cqlTranslatorVersion, + cqlInfo.cqlEngine, + cqlInfo.cqlEngineVersion, + cqlInfo.SERVER_OFFSET_ISO + ); if (cqlInfo?.cqlVersion) { engine.cqlVersion = cqlInfo.cqlVersion; } @@ -249,6 +282,9 @@ export class CQLEngine { if (cqlInfo?.cqlEngineVersion) { engine.cqlEngineVersion = cqlInfo.cqlEngineVersion; } + if (cqlInfo?.SERVER_OFFSET_ISO) { + engine._SERVER_OFFSET_ISO = cqlInfo.SERVER_OFFSET_ISO; + } return engine; } } diff --git a/src/jobs/job-processor.ts b/src/jobs/job-processor.ts index e56198a..9ce636f 100644 --- a/src/jobs/job-processor.ts +++ b/src/jobs/job-processor.ts @@ -30,7 +30,7 @@ export class JobProcessor { // Process the tests using the shared TestRunner const results = await this.testRunner.runTests(jobRequest.config, { - onProgress: async (current, total, message) => { + onProgress: async (current: number, total: number, message?: string) => { await this.jobManager.updateJobProgress(jobId, current, total, message); }, }); diff --git a/src/models/config-types.ts b/src/models/config-types.ts index 6cf9b3c..396c294 100644 --- a/src/models/config-types.ts +++ b/src/models/config-types.ts @@ -17,6 +17,7 @@ export interface Config { CqlOutputPath: string; CqlVersion?: string; testsRunDescription?: string; // Note: schema has this misplaced but it's used in code + SERVER_OFFSET_ISO: string; }; Tests: { ResultsPath: string; diff --git a/src/models/results-types.ts b/src/models/results-types.ts index c1a5740..c263929 100644 --- a/src/models/results-types.ts +++ b/src/models/results-types.ts @@ -28,4 +28,5 @@ export interface CQLEngineInfo { cqlTranslatorVersion: string; cqlEngine: string; cqlEngineVersion: string; + SERVER_OFFSET_ISO: string; } diff --git a/src/models/test-types.ts b/src/models/test-types.ts index 2ec86c3..d0e1964 100644 --- a/src/models/test-types.ts +++ b/src/models/test-types.ts @@ -40,6 +40,7 @@ export interface TestGroup { description?: string; reference?: string; notes?: string; + capability?: CapabilityKV[]; test: Test[]; } @@ -79,6 +80,7 @@ export interface InternalTestResult { invalid?: 'false' | 'true' | 'semantic' | 'undefined'; expression: string; capability?: CapabilityKV[]; + groupCapability?: CapabilityKV[]; SkipMessage?: string; } diff --git a/src/server/config-utils.ts b/src/server/config-utils.ts index 6f686dd..b5626b3 100644 --- a/src/server/config-utils.ts +++ b/src/server/config-utils.ts @@ -50,14 +50,16 @@ export function createConfigFromData(configData: any): ConfigLoader { }; config.Build = { - CqlFileVersion: process.env.CQL_FILE_VERSION || configData.Build?.CqlFileVersion || '1.0.000', - CqlOutputPath: process.env.CQL_OUTPUT_PATH || configData.Build?.CqlOutputPath || './cql', - CqlVersion: process.env.CQL_VERSION || configData.Build?.CqlVersion, - testsRunDescription: process.env.TESTS_RUN_DESCRIPTION || configData.Build?.testsRunDescription || 'No configuration provided', - cqlTranslator: process.env.CQL_TRANSLATOR || configData.Build?.cqlTranslator || 'No configuration provided', - cqlTranslatorVersion: process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'No configuration provided', - cqlEngine: process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'No configuration provided', - cqlEngineVersion: process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'No configuration provided' + CqlFileVersion: process.env.CQL_FILE_VERSION || configData.Build?.CqlFileVersion || '1.0.000', + CqlOutputPath: process.env.CQL_OUTPUT_PATH || configData.Build?.CqlOutputPath || './cql', + CqlVersion: process.env.CQL_VERSION || configData.Build?.CqlVersion, + testsRunDescription: process.env.TESTS_RUN_DESCRIPTION || configData.Build?.testsRunDescription || 'No configuration provided', + cqlTranslator: process.env.CQL_TRANSLATOR || configData.Build?.cqlTranslator || 'No configuration provided', + cqlTranslatorVersion: process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'No configuration provided', + cqlEngine: process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'No configuration provided', + cqlEngineVersion: process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'No configuration provided', + SERVER_OFFSET_ISO: process.env.SERVER_OFFSET_ISO || configData.Build?.SERVER_OFFSET_ISO || '+00:00', + TimeZoneOffsetPolicy: process.env.TIME_ZONE_OFFSET_POLICY || configData.Build?.TimeZoneOffsetPolicy || '', }; config.Tests = { diff --git a/src/services/test-runner.ts b/src/services/test-runner.ts deleted file mode 100644 index 2d6727a..0000000 --- a/src/services/test-runner.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { ConfigLoader } from '../conf/config-loader.js'; -import { CQLEngine } from '../cql-engine/cql-engine.js'; -import { TestLoader } from '../loaders/test-loader.js'; -import { CQLTestResults } from '../test-results/cql-test-results.js'; -import { generateEmptyResults, generateParametersResource } from '../shared/results-shared.js'; -import { InternalTestResult } from '../models/test-types.js'; -import { ResultExtractor } from '../extractors/result-extractor.js'; -import { ServerConnectivity } from '../shared/server-connectivity.js'; -import { buildExtractor } from '../server/extractor-builder.js'; -import { createConfigFromData } from '../server/config-utils.js'; -import { resultsEqual } from '../shared/results-utils.js'; - -export interface TestRunnerOptions { - onProgress?: (current: number, total: number, message?: string) => Promise; - useAxios?: boolean; // For backward compatibility with run-tests-command -} - -export class TestRunner { - public async runTests( - configData: any, - options: TestRunnerOptions = {} - ): Promise { - // Create a temporary config loader from the provided data - const config = createConfigFromData(configData); - const serverBaseUrl = config.FhirServer.BaseUrl; - const cqlEndpoint = config.CqlEndpoint; - - // Verify server connectivity before proceeding - await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); - - const cqlEngine = new CQLEngine( - serverBaseUrl, - cqlEndpoint, - configData.Build.cqlTranslator, - configData.Build.cqlTranslatorVersion, - configData.Build.cqlEngine, - configData.Build.cqlEngineVersion - ); - cqlEngine.cqlVersion = '1.5'; //default value - const cqlVersion = config.Build?.CqlVersion; - if (typeof cqlVersion === 'string' && cqlVersion.trim() !== '') { - cqlEngine.cqlVersion = cqlVersion; - } - - // Load CVL using dynamic import - // @ts-ignore - const cvlModule = await import('../../cvl/cvl.mjs'); - const cvl = cvlModule.default; - - const tests = TestLoader.load(); - const quickTest = config.Debug?.QuickTest || false; - const resultExtractor = buildExtractor(); - const emptyResults = await generateEmptyResults(tests, quickTest); - const skipMap = config.skipListMap(); - - const results = new CQLTestResults(cqlEngine); - - const totalTests = emptyResults.reduce((sum, testFile) => sum + testFile.length, 0); - let completedTests = 0; - - for (const testFile of emptyResults) { - for (const result of testFile) { - if (this.shouldSkipVersionTest(cqlEngine, result)) { - //add to skipMap - const skipReason = - 'test version ' + - result.testVersion + - ' not applicable to engine version ' + - cqlEngine.cqlVersion; - this.addToSkipList( - skipMap, - result.testsName, - result.groupName, - result.testName, - skipReason - ); - } - await this.runTest( - result, - cqlEngine.apiUrl!, - cvl, - resultExtractor, - skipMap, - config, - options.useAxios - ); - results.add(result); - - completedTests++; - if (options.onProgress) { - await options.onProgress( - completedTests, - totalTests, - `Running test ${result.testsName}:${result.groupName}:${result.testName}` - ); - } - } - } - // Return the CQLTestResults instance - return results; - } - - private async runTest( - result: InternalTestResult, - apiUrl: string, - cvl: any, - resultExtractor: ResultExtractor, - skipMap: Map, - config: ConfigLoader, - useAxios: boolean = false - ): Promise { - const key = `${result.testsName}-${result.groupName}-${result.testName}`; - - if (result.testStatus === 'skip') { - result.SkipMessage = 'Skipped by cql-tests-runner'; - return result; - } else if (skipMap.has(key)) { - const reason = skipMap.get(key) || ''; - result.SkipMessage = `Skipped by config: ${reason}`; - result.testStatus = 'skip'; - return result; - } - - const data = generateParametersResource(result, config.FhirServer.CqlOperation); - - try { - console.log( - 'Running test %s:%s:%s', - result.testsName, - result.groupName, - result.testName - ); - - let response: any; - if (useAxios) { - // Use axios for backward compatibility - const axios = await import('axios'); - const axiosResponse = await axios.default.post(apiUrl, data, { - headers: { - 'Content-Type': 'application/json', - }, - }); - response = { - status: axiosResponse.status, - data: axiosResponse.data, - }; - } else { - // Use fetch (default for new code) - const fetchResponse = await fetch(apiUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }); - response = { - status: fetchResponse.status, - data: await fetchResponse.json(), - }; - } - - result.responseStatus = response.status; - const responseBody = response.data; - result.actual = resultExtractor.extract(responseBody); - const invalid = result.invalid; - - if (invalid === 'true' || invalid === 'semantic') { - result.testStatus = response.status === 200 ? 'fail' : 'pass'; - } else { - if (response.status === 200) { - result.testStatus = resultsEqual(cvl.parse(result.expected), result.actual) - ? 'pass' - : 'fail'; - } else { - result.testStatus = 'fail'; - } - } - } catch (error: any) { - result.testStatus = 'error'; - result.error = { - message: error.message, - name: error.name || 'Error', - stack: error.stack, - }; - } - - console.log( - 'Test %s:%s:%s status: %s expected: %s actual: %s', - result.testsName, - result.groupName, - result.testName, - result.testStatus, - result.expected, - result.actual - ); - - return result; - } - - private compareVersions(versionA: string | undefined, versionB: string | undefined): number { - // Split into numeric parts (e.g., "1.5.2" β†’ [1,5,2]) - const partsA = String(versionA ?? '') - .trim() - .split('.') - .map(n => parseInt(n, 10) || 0); - const partsB = String(versionB ?? '') - .trim() - .split('.') - .map(n => parseInt(n, 10) || 0); - - const maxLength = Math.max(partsA.length, partsB.length); - - for (let i = 0; i < maxLength; i++) { - const numA = partsA[i] ?? 0; - const numB = partsB[i] ?? 0; - if (numA !== numB) { - return numA < numB ? -1 : 1; // -1 if A < B, 1 if A > B - } - } - return 0; // versions are equal - } - - private shouldSkipVersionTest(cqlEngine: CQLEngine, result: InternalTestResult): boolean { - const engineVersion = cqlEngine?.cqlVersion; - if (!engineVersion) return false; // no version to compare against - // Rule 1: if test.version is set, engine must be >= test.version - if (result.testVersion && this.compareVersions(engineVersion, result.testVersion) < 0) { - return true; - } - // Rule 2: if test.versionTo is set, engine must be <= test.versionTo - if (result.testVersionTo && this.compareVersions(engineVersion, result.testVersionTo) > 0) { - return true; - } - return false; // passes all checks - } - - private addToSkipList( - skipMap: Map, - testsName: string, - groupName: string, - testName: string, - reason: string - ): void { - skipMap.set(`${testsName}-${groupName}-${testName}`, reason); - } -} diff --git a/src/shared/results-shared.ts b/src/shared/results-shared.ts index 50b77a4..afb75c2 100644 --- a/src/shared/results-shared.ts +++ b/src/shared/results-shared.ts @@ -19,8 +19,14 @@ export class Result implements InternalTestResult { invalid: 'false' | 'true' | 'semantic' | 'undefined'; expression: string; capability: CapabilityKV[] = []; - - constructor(testsName: string, groupName: string, test: Test) { + groupCapability: CapabilityKV[] = []; + + constructor( + testsName: string, + groupName: string, + test: Test, + groupCapability: CapabilityKV[] = [] + ) { this.testsName = testsName; this.groupName = groupName; this.testName = test.name; @@ -54,7 +60,9 @@ export class Result implements InternalTestResult { this.capability = Array.isArray(test.capability) ? test.capability.map(({ code, value }) => ({ code, value })) : []; - } + this.groupCapability = Array.isArray(groupCapability) + ? groupCapability.map(({ code, value }) => ({ code, value })) + : []; } } export async function generateEmptyResults( @@ -77,7 +85,7 @@ export async function generateEmptyResults( if (test != undefined) { for (const t of test) { console.log(' Test: ' + t.name); - const r = new Result(ts.name, group.name, t); + const r = new Result(ts.name, group.name, t, group.capability || []); results.push(r); groupTests.push(r); } diff --git a/src/test-results/cql-test-results.ts b/src/test-results/cql-test-results.ts index 4a6df87..5b3de47 100644 --- a/src/test-results/cql-test-results.ts +++ b/src/test-results/cql-test-results.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { CQLEngine } from '../cql-engine/cql-engine.js'; -import { TestResult, InternalTestResult } from '../models/test-types.js'; +import { TestResult, InternalTestResult } from '../models/test-types.orig'; import { TestResultsSummary, CQLTestResultsData } from '../models/results-types.js'; import { ResultsValidator } from '../conf/results-validator.js'; diff --git a/test/server-command.test.ts b/test/server-command.test.ts index 392e436..7a12fda 100644 --- a/test/server-command.test.ts +++ b/test/server-command.test.ts @@ -6,22 +6,23 @@ import { ServerCommand } from '../src/commands/server-command.js'; // Test data and mock helpers const createMockConfig = (overrides = {}) => ({ - FhirServer: { - BaseUrl: 'http://localhost:8080/fhir/', - CqlOperation: '$cql', - }, - Build: { - CqlFileVersion: '1.0.000', - CqlOutputPath: './cql', - }, - Debug: { - QuickTest: false, - }, - Tests: { - ResultsPath: './results', - SkipList: [], - }, - ...overrides, + FhirServer: { + BaseUrl: 'http://localhost:8080/fhir/', + CqlOperation: '$cql', + }, + Build: { + CqlFileVersion: '1.0.000', + CqlOutputPath: './cql', + SERVER_OFFSET_ISO: '+00:00' + }, + Debug: { + QuickTest: false, + }, + Tests: { + ResultsPath: './results', + SkipList: [], + }, + ...overrides, }); const createMockResults = () => ({ From 4266546a85cfec05108c9f36caeb8f9362e9e697 Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Tue, 17 Mar 2026 14:21:01 -0600 Subject: [PATCH 2/8] updated documentation about configuration settings concerning timezone offset --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0db30d6..422b11e 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,17 @@ Configuration settings are set in a JSON configuration file. The file `conf/loca "CqlOperation": "$cql" }, "Build": { - "CqlFileVersion": "1.0.000", - "CqlOutputPath": "./cql", - "testsRunDescription": '', - "testsRunDescription": "Local host test run", - "cqlTranslator": "Java CQFramework Translator", - "cqlTranslatorVersion": "Unknown", - "cqlEngine": "Java CQFramework Engine", - "cqlEngineVersion": "4.1.0" - }, + "CqlFileVersion": "1.0.000", + "CqlOutputPath": "./cql", + "CqlVersion": "1.5", + "testsRunDescription": "Local host test run", + "cqlTranslator": "Java CQFramework Translator", + "cqlTranslatorVersion": "Unknown", + "cqlEngine": "Java CQFramework Engine", + "cqlEngineVersion": "4.1.0", + "SERVER_OFFSET_ISO": "-06:00", + "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" + }, "Tests": { "ResultsPath": "./results", "SkipList": [] From f386e4cc0071a5063b9f3d1b3829f5fb23dd7177 Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Fri, 20 Mar 2026 12:02:38 -0600 Subject: [PATCH 3/8] documentation for timezone offset testing --- README.md | 174 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/README.md b/README.md index 422b11e..9761d4a 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,180 @@ Configuration settings are set in a JSON configuration file. The file `conf/loca Create your own configuration file and reference it when running the commands. You can use `conf/localhost.json` as a template for a new configuration with your own settings. +### Time Zone Configuration + +The CQL Tests Runner uses two settings to control how **DateTime** values are evaluated: + +- `SERVER_OFFSET_ISO` +- `TimeZoneOffsetPolicy` + +These settings are required because CQL allows **DateTime values without a timezone offset**, and different engines interpret those values differently. Without explicitly setting these, tests involving DateTime comparison and extraction may produce inconsistent results (pass, fail, or null). + +Reference: http://cql.hl7.org/CodeSystem/cql-language-capabilities + +--- + +#### TimeZoneOffsetPolicy (what it means) + +A DateTime like this: + +```cql +@2012-04-01T00:00 +``` + +has **no timezone offset**. + +When a CQL engine evaluates this, it must decide: + +πŸ‘‰ Should this value be treated as having a timezone, or not? + +That decision is the **timezone offset policy**. + +--- + +#### Supported policies + +##### `timezone-offset-policy.default-server-offset` + +Offset-less DateTime values are interpreted using the server’s timezone. + +Example (with `SERVER_OFFSET_ISO = -06:00`): + +```cql +@2012-04-01T00:00 +``` + +is treated as: + +```cql +@2012-04-01T00:00-06:00 +``` + +**Result behavior:** +- DateTime comparisons behave as if all values have offsets +- Equality comparisons with explicit offsets often return **true** +- `timezoneoffset from` returns a numeric offset (e.g. `-6`) +- Tests expecting normalization to a server offset β†’ **pass** +- Tests expecting strict offset behavior β†’ **skip** + +--- + +##### `timezone-offset-policy.no-default-offset` + +Offset-less DateTime values remain **without a timezone**. + +```cql +@2012-04-01T00:00 +``` + +remains unchanged. + +**Result behavior:** +- No timezone is assumed +- Comparisons between offset-less and offset values may return **null** or **false** +- `timezoneoffset from` returns **null** +- Tests expecting strict offset behavior β†’ **pass** +- Tests expecting server normalization β†’ **skip** + +--- + +#### SERVER_OFFSET_ISO (what it does) + +Provides the timezone offset value used in test expressions. + +- Format: ISO 8601 offset (e.g. `-06:00`, `+00:00`, `+05:30`) +- Used when a test includes the placeholder `{{SERVER_OFFSET_ISO}}` + +Example: + +```cql +@2012-04-01T00:00 = @2012-04-01T00:00{{SERVER_OFFSET_ISO}} +``` + +With: + +```json +"SERVER_OFFSET_ISO": "-06:00" +``` + +Becomes: + +```cql +@2012-04-01T00:00 = @2012-04-01T00:00-06:00 +``` + +--- + +#### How they work together + +- `TimeZoneOffsetPolicy` determines **whether offset-less DateTime values get a timezone** +- `SERVER_OFFSET_ISO` provides **the timezone value used when needed** + +Example: + +```json +{ + "Build": { + "SERVER_OFFSET_ISO": "-06:00", + "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" + } +} +``` + +This means: +- Use `-06:00` as the server timezone +- Apply that offset to DateTime values that do not include one + +--- + +#### How to set these values + +In your configuration file: + +```json +{ + "Build": { + "SERVER_OFFSET_ISO": "-06:00", + "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" + } +} +``` + +Or using an environment variable (policy only): + +```bash +export TIME_ZONE_OFFSET_POLICY=timezone-offset-policy.no-default-offset +``` + +--- + +#### How the runner determines the active policy + +The runner resolves `TimeZoneOffsetPolicy` in this order: + +1. FHIR server metadata (CapabilityStatement) +2. Environment variable (`TIME_ZONE_OFFSET_POLICY`) +3. Configuration file (`TimeZoneOffsetPolicy`) +4. Runtime probe (detect behavior automatically) +5. Default: `timezone-offset-policy.default-server-offset` + +--- + +#### Expected results summary + +| Policy | Offset-less DateTime | timezoneoffset | Equality vs offset | +|--------|---------------------|----------------|-------------------| +| default-server-offset | gets server offset | number (e.g. `-6`) | often `true` | +| no-default-offset | remains offset-less | `null` | `null` or `false` | + +--- + +#### Notes + +- These settings do not change server behavior; they only control how the runner evaluates tests +- If the server declares a policy in metadata, it overrides configuration +- `SERVER_OFFSET_ISO` is only used where explicitly referenced in test expressions + ### Running the tests The CLI now requires a configuration file path as an argument. Run the tests with the following commands: From c84fe0c958d31e73706e38077faf0efe13d83c92 Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Mon, 23 Mar 2026 13:19:10 -0600 Subject: [PATCH 4/8] fix typo --- src/test-results/cql-test-results.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test-results/cql-test-results.ts b/src/test-results/cql-test-results.ts index 5b3de47..4a6df87 100644 --- a/src/test-results/cql-test-results.ts +++ b/src/test-results/cql-test-results.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { CQLEngine } from '../cql-engine/cql-engine.js'; -import { TestResult, InternalTestResult } from '../models/test-types.orig'; +import { TestResult, InternalTestResult } from '../models/test-types.js'; import { TestResultsSummary, CQLTestResultsData } from '../models/results-types.js'; import { ResultsValidator } from '../conf/results-validator.js'; From 77aa24e77c93f0cd0ef1cc0a3c3b23a53668dc58 Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Mon, 23 Mar 2026 14:28:25 -0600 Subject: [PATCH 5/8] adjustments to clean up --- src/services/test-runner.ts | 493 ++++++++++++++++++++++++++++++++++++ 1 file changed, 493 insertions(+) create mode 100644 src/services/test-runner.ts diff --git a/src/services/test-runner.ts b/src/services/test-runner.ts new file mode 100644 index 0000000..732b89d --- /dev/null +++ b/src/services/test-runner.ts @@ -0,0 +1,493 @@ +import { ConfigLoader } from '../conf/config-loader.js'; +import { CQLEngine } from '../cql-engine/cql-engine.js'; +import { TestLoader } from '../loaders/test-loader.js'; +import { CQLTestResults } from '../test-results/cql-test-results.js'; +import { generateEmptyResults, generateParametersResource } from '../shared/results-shared.js'; +import { InternalTestResult } from '../models/test-types.js'; +import { ResultExtractor } from '../extractors/result-extractor.js'; +import { ServerConnectivity } from '../shared/server-connectivity.js'; +import { buildExtractor } from '../server/extractor-builder.js'; +import { createConfigFromData } from '../server/config-utils.js'; +import { resultsEqual } from '../shared/results-utils.js'; + +export interface TestRunnerOptions { + onProgress?: (current: number, total: number, message?: string) => Promise; + useAxios?: boolean; // For backward compatibility with run-tests-command +} + +export class TestRunner { + public async runTests( + configData: any, + options: TestRunnerOptions = {} + ): Promise { + // Create a temporary config loader from the provided data + const config = createConfigFromData(configData); + const serverBaseUrl = config.FhirServer.BaseUrl; + const cqlEndpoint = config.CqlEndpoint; + + // Verify server connectivity before proceeding + await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); + + const cqlEngine = new CQLEngine( + serverBaseUrl, + cqlEndpoint, + configData.Build.cqlTranslator, + configData.Build.cqlTranslatorVersion, + configData.Build.cqlEngine, + configData.Build.cqlEngineVersion, + configData.Build.SERVER_OFFSET_ISO + ); + cqlEngine.cqlVersion = '1.5'; //default value + const cqlVersion = config.Build?.CqlVersion; + if (typeof cqlVersion === 'string' && cqlVersion.trim() !== '') { + cqlEngine.cqlVersion = cqlVersion; + } + + await cqlEngine.fetch(); + + const activeTimeZonePolicy = await this.resolveTimeZoneOffsetPolicy( + config, + cqlEngine.apiUrl!, + cqlEngine['metadata'], + options.useAxios + ); + console.log('Resolved timezone policy:', activeTimeZonePolicy); + // Load CVL using dynamic import + // @ts-ignore + const cvlModule = await import('../../cvl/cvl.mjs'); + const cvl = cvlModule.default; + + const tests = TestLoader.load(); + const quickTest = config.Debug?.QuickTest || false; + const resultExtractor = buildExtractor(); + const emptyResults = await generateEmptyResults(tests, quickTest); + const skipMap = config.skipListMap(); + + const results = new CQLTestResults(cqlEngine); + + const totalTests = emptyResults.reduce((sum, testFile) => sum + testFile.length, 0); + let completedTests = 0; + + for (const testFile of emptyResults) { + for (const result of testFile) { + if (this.shouldSkipVersionTest(cqlEngine, result)) { + //add to skipMap + const skipReason = + 'test version ' + + result.testVersion + + ' not applicable to engine version ' + + cqlEngine.cqlVersion; + this.addToSkipList( + skipMap, + result.testsName, + result.groupName, + result.testName, + skipReason + ); + } + await this.runTest( + result, + cqlEngine.apiUrl!, + cvl, + resultExtractor, + skipMap, + config, + cqlEngine, + activeTimeZonePolicy, + options.useAxios + ); + results.add(result); + + completedTests++; + if (options.onProgress) { + await options.onProgress( + completedTests, + totalTests, + `Running test ${result.testsName}:${result.groupName}:${result.testName}` + ); + } + } + } + // Return the CQLTestResults instance + return results; + } + + private async runTest( + result: InternalTestResult, + apiUrl: string, + cvl: any, + resultExtractor: ResultExtractor, + skipMap: Map, + config: ConfigLoader, + cqlEngine: CQLEngine, + activeTimeZonePolicy: string, + useAxios: boolean = false + ): Promise { + const key = `${result.testsName}-${result.groupName}-${result.testName}`; + + if (result.testStatus === 'skip') { + result.SkipMessage = 'Skipped by cql-tests-runner'; + return result; + } else if (skipMap.has(key)) { + const reason = skipMap.get(key) || ''; + result.SkipMessage = `Skipped by config: ${reason}`; + result.testStatus = 'skip'; + return result; + } + + const timezonePolicySkipReason = this.shouldSkipTimezonePolicyTest( + result, + activeTimeZonePolicy + ); + if (timezonePolicySkipReason) { + result.SkipMessage = timezonePolicySkipReason; + result.testStatus = 'skip'; + return result; + } + + const data = generateParametersResource(result, config.FhirServer.CqlOperation); + + try { + console.log( + 'Running test %s:%s:%s', + result.testsName, + result.groupName, + result.testName + ); + + this.applyServerOffsetToParameters(data, cqlEngine); + + let response: any; + if (useAxios) { + // Use axios for backward compatibility + const axios = await import('axios'); + const axiosResponse = await axios.default.post(apiUrl, data, { + headers: { + 'Content-Type': 'application/json', + }, + }); + response = { + status: axiosResponse.status, + data: axiosResponse.data, + }; + } else { + // Use fetch (default for new code) + const fetchResponse = await fetch(apiUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + response = { + status: fetchResponse.status, + data: await fetchResponse.json(), + }; + } + + result.responseStatus = response.status; + const responseBody = response.data; + result.actual = resultExtractor.extract(responseBody); + const invalid = result.invalid; + + if (invalid === 'true' || invalid === 'semantic') { + result.testStatus = response.status === 200 ? 'fail' : 'pass'; + } else { + if (response.status === 200) { + result.testStatus = resultsEqual(cvl.parse(result.expected), result.actual) + ? 'pass' + : 'fail'; + } else { + result.testStatus = 'fail'; + } + } + } catch (error: any) { + result.testStatus = 'error'; + result.error = { + message: error.message, + name: error.name || 'Error', + stack: error.stack, + }; + } + + console.log( + 'Test %s:%s:%s status: %s expected: %s actual: %s', + result.testsName, + result.groupName, + result.testName, + result.testStatus, + result.expected, + result.actual + ); + + return result; + } + + private async resolveTimeZoneOffsetPolicy( + config: ConfigLoader, + apiUrl: string, + serverMetadata?: any, + useAxios: boolean = false + ): Promise { + // + const metadataPolicy = this.extractTimeZonePolicyFromMetadata(serverMetadata); + if (metadataPolicy) { + console.log('Resolved timezone policy from metadata:', metadataPolicy); + return metadataPolicy; + } + + const configuredPolicy = + process.env.TIME_ZONE_OFFSET_POLICY?.trim() || + config.Build?.TimeZoneOffsetPolicy?.trim(); + + if (configuredPolicy) { + console.log('Resolved timezone policy from env/config:', configuredPolicy); + return configuredPolicy; + } + + const probedPolicy = await this.detectTimeZoneOffsetPolicy(apiUrl, useAxios); + if (probedPolicy) { + console.log('Resolved timezone policy from probe:', probedPolicy); + return probedPolicy; + } + + const fallbackPolicy = 'timezone-offset-policy.default-server-offset'; + console.log('Resolved timezone policy from fallback:', fallbackPolicy); + return fallbackPolicy; + } + + private async detectTimeZoneOffsetPolicy( + apiUrl: string, + useAxios: boolean = false + ): Promise { + // order of setting timezone offset policy: metadata -> env/config -> probe -> fallback + const data = { + resourceType: 'Parameters', + parameter: [ + { + name: 'expression', + valueString: 'timezoneoffset from @2012-04-01T00:00', + }, + ], + }; + + let response: any; + + if (useAxios) { + const axios = await import('axios'); + const axiosResponse = await axios.default.post(apiUrl, data, { + headers: { 'Content-Type': 'application/json' }, + }); + response = { + status: axiosResponse.status, + data: axiosResponse.data, + }; + } else { + const fetchResponse = await fetch(apiUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data), + }); + response = { + status: fetchResponse.status, + data: await fetchResponse.json(), + }; + } + + if (response.status !== 200) { + return null; + } + + const extracted = this.extractProbeResult(response.data); + + if (extracted === 'null') { + return 'timezone-offset-policy.no-default-offset'; + } + + if (typeof extracted === 'number') { + return 'timezone-offset-policy.default-server-offset'; + } + + if (typeof extracted === 'string') { + const trimmed = extracted.trim().toLowerCase(); + if (trimmed === 'null') { + return 'timezone-offset-policy.no-default-offset'; + } + if (/^-?\d+$/.test(trimmed)) { + return 'timezone-offset-policy.default-server-offset'; + } + } + + return null; + } + + private extractProbeResult(responseBody: any): any { + const parameter = responseBody?.parameter; + if (!Array.isArray(parameter) || parameter.length === 0) { + return null; + } + + const returnParam = parameter.find((p: any) => p.name === 'return') || parameter[0]; + + if (returnParam.valueInteger !== undefined) { + return returnParam.valueInteger; + } + if (returnParam.valueDecimal !== undefined) { + return returnParam.valueDecimal; + } + if (returnParam.valueString !== undefined) { + return returnParam.valueString; + } + if (returnParam.valueBoolean !== undefined) { + return returnParam.valueBoolean; + } + + return null; + } + + private extractTimeZonePolicyFromMetadata(metadata: any): string | null { + if (!metadata || typeof metadata !== 'object') { + return null; + } + + const policyCodes = [ + 'timezone-offset-policy.no-default-offset', + 'timezone-offset-policy.default-server-offset', + ]; + + function findInObject(obj: any): string | null { + if (!obj || typeof obj !== 'object') { + return null; + } + + if (Array.isArray(obj)) { + for (const item of obj) { + const found = findInObject(item); + if (found) return found; + } + return null; + } + + for (const [key, value] of Object.entries(obj)) { + if (typeof value === 'string' && policyCodes.includes(value)) { + return value; + } + + if (key === 'code' && typeof value === 'string' && policyCodes.includes(value)) { + return value; + } + + if ( + key === 'valueCode' && + typeof value === 'string' && + policyCodes.includes(value) + ) { + return value; + } + + if ( + key === 'valueString' && + typeof value === 'string' && + policyCodes.includes(value) + ) { + return value; + } + + if (typeof value === 'object') { + const found = findInObject(value); + if (found) return found; + } + } + + return null; + } + + return findInObject(metadata); + } + + private applyServerOffsetToParameters(data: any, engine: CQLEngine): void { + const expressionParam = data?.parameter?.find((p: any) => p.name === 'expression'); + if (!expressionParam || typeof expressionParam.valueString !== 'string') { + return; + } + + const offset = engine.SERVER_OFFSET_ISO; + if (typeof offset !== 'string' || offset.trim() === '') { + return; + } + + expressionParam.valueString = this.replaceServerOffsetPlaceholder( + expressionParam.valueString, + offset + ); + } + + private replaceServerOffsetPlaceholder(expression: string, serverOffsetISO: string): string { + return expression.replace(/\{\{SERVER_OFFSET_ISO\}\}/g, serverOffsetISO); + } + + private shouldSkipTimezonePolicyTest(test: any, activeTimeZonePolicy: string): string | null { + const requiredCapabilities = test.capability || []; + + const requiredPolicy = requiredCapabilities.find((c: any) => + c.code?.startsWith('timezone-offset-policy.') + )?.code; + + if (!requiredPolicy) { + return null; + } + + if (requiredPolicy !== activeTimeZonePolicy) { + return `requires ${requiredPolicy} but server is ${activeTimeZonePolicy}`; + } + + return null; + } + + private compareVersions(versionA: string | undefined, versionB: string | undefined): number { + // Split into numeric parts (e.g., "1.5.2" β†’ [1,5,2]) + const partsA = String(versionA ?? '') + .trim() + .split('.') + .map(n => parseInt(n, 10) || 0); + const partsB = String(versionB ?? '') + .trim() + .split('.') + .map(n => parseInt(n, 10) || 0); + + const maxLength = Math.max(partsA.length, partsB.length); + + for (let i = 0; i < maxLength; i++) { + const numA = partsA[i] ?? 0; + const numB = partsB[i] ?? 0; + if (numA !== numB) { + return numA < numB ? -1 : 1; // -1 if A < B, 1 if A > B + } + } + return 0; // versions are equal + } + + private shouldSkipVersionTest(cqlEngine: CQLEngine, result: InternalTestResult): boolean { + const engineVersion = cqlEngine?.cqlVersion; + if (!engineVersion) return false; // no version to compare against + // Rule 1: if test.version is set, engine must be >= test.version + if (result.testVersion && this.compareVersions(engineVersion, result.testVersion) < 0) { + return true; + } + // Rule 2: if test.versionTo is set, engine must be <= test.versionTo + if (result.testVersionTo && this.compareVersions(engineVersion, result.testVersionTo) > 0) { + return true; + } + return false; // passes all checks + } + + private addToSkipList( + skipMap: Map, + testsName: string, + groupName: string, + testName: string, + reason: string + ): void { + skipMap.set(`${testsName}-${groupName}-${testName}`, reason); + } +} From cdff70caadc06683e6fb7c2312716008a9121e85 Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Fri, 27 Mar 2026 10:30:02 -0600 Subject: [PATCH 6/8] adjusted README to better state timezone offset --- README.md | 181 ++++++++++++++++++++++++++---------------------------- 1 file changed, 86 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 9761d4a..c7bd126 100644 --- a/README.md +++ b/README.md @@ -73,158 +73,149 @@ Reference: http://cql.hl7.org/CodeSystem/cql-language-capabilities #### TimeZoneOffsetPolicy (what it means) -A DateTime like this: +# πŸ”„ Timezone Offset Policy -```cql +## Background + +CQL allows `DateTime` literals to be specified **with or without a timezone offset**. + +For example: + +``` @2012-04-01T00:00 ``` -has **no timezone offset**. +This value does **not include an explicit timezone offset**. -When a CQL engine evaluates this, it must decide: +--- -πŸ‘‰ Should this value be treated as having a timezone, or not? +## What the CQL Specification Says -That decision is the **timezone offset policy**. +> If no timezone offset is specified, the timezone offset of the evaluation request timestamp is used. ---- +This means: -#### Supported policies +- A `DateTime` literal may omit an offset in its **source representation** +- But at **evaluation time**, the engine must apply an offset +- That offset comes from the **evaluation request timestamp** -##### `timezone-offset-policy.default-server-offset` +πŸ‘‰ In other words, under CQL semantics: -Offset-less DateTime values are interpreted using the server’s timezone. +- DateTimes are **always evaluated with an effective timezone offset** +- The only question is **which offset is applied** -Example (with `SERVER_OFFSET_ISO = -06:00`): +--- -```cql -@2012-04-01T00:00 -``` +## Why This Capability Exists -is treated as: +Although the specification is clear, implementations have historically differed in how they handle `DateTime` values without explicit offsets. -```cql -@2012-04-01T00:00-06:00 -``` +Observed variations include: -**Result behavior:** -- DateTime comparisons behave as if all values have offsets -- Equality comparisons with explicit offsets often return **true** -- `timezoneoffset from` returns a numeric offset (e.g. `-6`) -- Tests expecting normalization to a server offset β†’ **pass** -- Tests expecting strict offset behavior β†’ **skip** +- Applying the evaluation request timestamp offset (spec-compliant behavior) +- Treating the value as offset-less in some operations +- Returning `null` for operations like `timezoneoffset(...)` +- Applying server-local or implicit defaults inconsistently + +This capability exists to explicitly test and document how an engine behaves in these scenarios. --- -##### `timezone-offset-policy.no-default-offset` +## What Is Being Tested -Offset-less DateTime values remain **without a timezone**. +This capability evaluates how an engine handles `DateTime` values that omit a timezone offset, including: -```cql -@2012-04-01T00:00 -``` +- Whether the engine applies the evaluation request timestamp offset +- How functions like `timezoneoffset(...)` behave +- Whether comparisons and arithmetic treat the value consistently -remains unchanged. +--- -**Result behavior:** -- No timezone is assumed -- Comparisons between offset-less and offset values may return **null** or **false** -- `timezoneoffset from` returns **null** -- Tests expecting strict offset behavior β†’ **pass** -- Tests expecting server normalization β†’ **skip** +## Clarification ---- +This capability does **not** test whether a `DateTime` β€œhas a timezone or not.” -#### SERVER_OFFSET_ISO (what it does) +Per the CQL specification: -Provides the timezone offset value used in test expressions. +- A `DateTime` without an explicit offset is still evaluated **as if it has one** +- The offset is derived from the evaluation context (evaluation request timestamp) -- Format: ISO 8601 offset (e.g. `-06:00`, `+00:00`, `+05:30`) -- Used when a test includes the placeholder `{{SERVER_OFFSET_ISO}}` +πŸ‘‰ The purpose of this capability is to verify that engines implement this behavior **correctly and consistently**. -Example: +--- -```cql -@2012-04-01T00:00 = @2012-04-01T00:00{{SERVER_OFFSET_ISO}} -``` +## Suggested Terminology -With: +- ❌ β€œDoes the DateTime have a timezone?” +- βœ… β€œHow does the engine determine the effective timezone offset for DateTimes without an explicit offset?” -```json -"SERVER_OFFSET_ISO": "-06:00" -``` +--- -Becomes: +## Key Clarification + +> This capability is testing **implementation consistency**, not specification ambiguity. -```cql -@2012-04-01T00:00 = @2012-04-01T00:00-06:00 -``` --- -#### How they work together +# πŸ”— CapabilityTests Alignment & Mapping -- `TimeZoneOffsetPolicy` determines **whether offset-less DateTime values get a timezone** -- `SERVER_OFFSET_ISO` provides **the timezone value used when needed** +## Capability Definition -Example: +- Capability: `timezone-offset-policy` +- Focus: Handling of DateTime values without explicit timezone offsets -```json -{ - "Build": { - "SERVER_OFFSET_ISO": "-06:00", - "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" - } -} -``` +--- -This means: -- Use `-06:00` as the server timezone -- Apply that offset to DateTime values that do not include one +## Test Mapping + +### 1. Default Offset Application +- Tests: `timezone-offset-default` +- Verifies: + - Missing offset uses evaluation request timestamp + - No null offset produced --- -#### How to set these values +### 2. timezoneoffset(...) Behavior +- Tests: `timezoneoffset-from-datetime` +- Verifies: + - Returns evaluation offset when not explicitly provided -In your configuration file: +--- -```json -{ - "Build": { - "SERVER_OFFSET_ISO": "-06:00", - "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" - } -} -``` +### 3. Equality / Comparison Consistency +- Tests: `datetime-equality-offset` +- Verifies: + - `@2012-04-01T00:00` equals equivalent explicit-offset value -Or using an environment variable (policy only): +--- -```bash -export TIME_ZONE_OFFSET_POLICY=timezone-offset-policy.no-default-offset -``` +### 4. Arithmetic Behavior +- Tests: `datetime-arithmetic-offset` +- Verifies: + - Duration calculations respect derived offset --- -#### How the runner determines the active policy +## Reviewer Traceability -The runner resolves `TimeZoneOffsetPolicy` in this order: +Spec β†’ Behavior β†’ Test -1. FHIR server metadata (CapabilityStatement) -2. Environment variable (`TIME_ZONE_OFFSET_POLICY`) -3. Configuration file (`TimeZoneOffsetPolicy`) -4. Runtime probe (detect behavior automatically) -5. Default: `timezone-offset-policy.default-server-offset` +- Spec: Offset defaults to evaluation timestamp +- Behavior: Engine applies offset consistently +- Tests: Confirm consistency across functions and operators --- -#### Expected results summary +## Summary -| Policy | Offset-less DateTime | timezoneoffset | Equality vs offset | -|--------|---------------------|----------------|-------------------| -| default-server-offset | gets server offset | number (e.g. `-6`) | often `true` | -| no-default-offset | remains offset-less | `null` | `null` or `false` | +This capability ensures engines: + +- Correctly apply implicit timezone offsets +- Do not treat DateTimes as offset-less at runtime +- Maintain consistency across operations ---- #### Notes From 918dfc34a2095ae705295de6e1c5b5580323212a Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Mon, 13 Apr 2026 14:56:04 -0600 Subject: [PATCH 7/8] merged latest main into TimeZoneOffset branch --- Dockerfile | 6 +- README.md | 86 ++++- .../schema/cql-test-configuration.schema.json | 107 +++--- conf/localhost.json | 20 +- package.json | 20 +- src/bin/cql-tests.ts | 11 +- src/commands/run-tests-command.ts | 33 +- src/conf/config-loader.ts | 41 +- src/cql-engine/cql-engine.ts | 106 +++--- src/extractors/evaluation-error-extractor.ts | 12 +- src/extractors/result-extractor.ts | 19 +- src/extractors/value-map.ts | 38 +- src/jobs/job-processor.ts | 2 +- src/loaders/test-loader.ts | 9 +- src/models/config-types.ts | 13 +- src/models/test-types.ts | 2 + src/server/config-utils.ts | 49 ++- src/server/test-execution-service.ts | 187 ++++------ src/services/test-runner.ts | 62 ++- src/shared/results-shared.ts | 9 +- src/test-results/cql-test-results.ts | 1 + test/extractResults-cql_operations.test.ts | 74 ++++ test/results-utils.test.js | 19 + test/run-DataTransferItemList.test.ts | 352 ++++++++++++++++++ test/server-command.test.ts | 34 +- test/value-Map.test.ts | 44 +++ tsconfig.json | 4 +- vitest.config.ts | 1 + 28 files changed, 1035 insertions(+), 326 deletions(-) create mode 100644 test/results-utils.test.js create mode 100644 test/run-DataTransferItemList.test.ts create mode 100644 test/value-Map.test.ts diff --git a/Dockerfile b/Dockerfile index f813854..e05c322 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Use Node.js v24 as the base image +# Use Node.js v25 as the base image FROM node:25-alpine # Set working directory @@ -24,7 +24,7 @@ RUN cp -r assets dist/ RUN cp -r cvl dist/ # Run the build-cql subcommand to generate CQL libraries -RUN node dist/src/bin/cql-tests.js build-cql conf/localhost.json ./cql +RUN node dist/bin/cql-tests.js build-cql conf/localhost.json ./cql # Remove dev dependencies to reduce image size RUN npm prune --production @@ -42,4 +42,4 @@ EXPOSE 3000 # Set the entrypoint to run the compiled cql-tests.js # The Node.js process will handle SIGINT/SIGTERM signals for graceful shutdown -ENTRYPOINT ["node", "dist/src/bin/cql-tests.js"] +ENTRYPOINT ["node", "dist/bin/cql-tests.js"] diff --git a/README.md b/README.md index c7bd126..2847e92 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,14 @@ # cql-tests-runner +[![Website](https://shields.foundry.hl7.org/website?url=https%3A%2F%2Fcql-tests-runner.quality.hl7.org&logo=fireship&label=try%20it%20now)](https://cql-tests-runner.quality.hl7.org) +[![GitHub contributors](https://shields.foundry.hl7.org/github/contributors/cqframework/cql-tests-runner?logo=github)](https://github.com/cqframework/cql-tests-runner/graphs/contributors) +[![GitHub last commit](https://shields.foundry.hl7.org/github/last-commit/cqframework/cql-tests-runner?logo=github)](https://github.com/cqframework/cql-tests-runner/graphs/commit-activity) +[![GitHub top language](https://shields.foundry.hl7.org/github/languages/top/cqframework/cql-tests-runner?logo=github)](https://github.com/cqframework/cql-tests-runner) +[![Docker automated build](https://shields.foundry.hl7.org/docker/automated/hlseven/quality-cql-tests-runner?logo=docker)](https://hub.docker.com/r/hlseven/quality-cql-tests-runner) +[![Docker pulls](https://shields.foundry.hl7.org/docker/pulls/hlseven/quality-cql-tests-runner?logo=docker)](https://hub.docker.com/r/hlseven/quality-cql-tests-runner) +[![Docker image size](https://shields.foundry.hl7.org/docker/image-size/hlseven/quality-cql-tests-runner?logo=docker)](https://hub.docker.com/r/hlseven/quality-cql-tests-runner) + + Test Runner for the [CQL Tests](https://github.com/cqframework/cql-tests) repository. This node application allows you to run the tests in the CQL Tests repository against a server of your choice using the [$cql](https://hl7.org/fhir/uv/cql/OperationDefinition-cql-cql.html) operation. The runner in its current state uses only this operation, and there is no expectation of any other FHIR server capability made by this runner. Additional capabilities may be required in the future as we expand the runner to support full Library/$evaluate as well. None of the tests in the repository have any expectation of being able to access data (i.e. the tests have no retrieve expressions). The application runs all the tests in the repository and outputs the results as a JSON file in the `results` directory. If the output directory does not exist, it will be created. @@ -8,11 +17,11 @@ Results output from running these tests can be posted to the [CQL Tests Results] ## Setting up the Environment -This application requires Node v24 and makes use of the [Axios](https://axios-http.com/docs/intro) framework for HTTP request/response processing. [Node Download](https://nodejs.org/en/download) +This application requires Node v25 and makes use of the [Axios](https://axios-http.com/docs/intro) framework for HTTP request/response processing. [Node Download](https://nodejs.org/en/download) Install application dependencies using -``` +```sh npm install ``` @@ -20,7 +29,7 @@ npm install The cql-tests folder has been added as a submodule. After pulling, you'll find a cql-tests folder inside cql-tests-runner. However, when you peek inside that folder, depending on your Git version, you might see nothing. Newer versions of Git will handle this automatically, but older versions may require you to explicitly instruct Git to download the contents of cql-tests. -``` +```bash git submodule update --init --recursive ``` @@ -28,7 +37,7 @@ git submodule update --init --recursive Configuration settings are set in a JSON configuration file. The file `conf/localhost.json` provides a sample configuration. -``` +```json { "FhirServer": { "BaseUrl": "https://fhirServerBaseUrl", @@ -37,14 +46,11 @@ Configuration settings are set in a JSON configuration file. The file `conf/loca "Build": { "CqlFileVersion": "1.0.000", "CqlOutputPath": "./cql", - "CqlVersion": "1.5", "testsRunDescription": "Local host test run", "cqlTranslator": "Java CQFramework Translator", "cqlTranslatorVersion": "Unknown", "cqlEngine": "Java CQFramework Engine", - "cqlEngineVersion": "4.1.0", - "SERVER_OFFSET_ISO": "-06:00", - "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" + "cqlEngineVersion": "4.1.0" }, "Tests": { "ResultsPath": "./results", @@ -56,6 +62,50 @@ Configuration settings are set in a JSON configuration file. The file `conf/loca } ``` +To skip tests, add entries to the `SkipList` with the corresponding `testsName`, `groupName`, `testName`, and `reason`. + +```jsonc +{ + "FhirServer": {/* omitted */}, + "Build": {/* omitted */}, + "Tests": { + "ResultsPath": "./results", + "SkipList": [ + { + "testsName": "CqlAggregateTest", + "groupName": "AggregateTests", + "testName": "RolledOutIntervals", + "reason": "CQLtoELM - Could not resolve identifier MedicationRequestIntervals in the current library" + }, + // add more tests to skip as necessary... + ] + }, + "Debug": {/* ommitted */} +} +``` + +To run only a specified set of tests (and skip all others), add entries to the `OnlyList` with the corresponding `testsName`, `groupName`, and `testName`. + +```jsonc +{ + "FhirServer": {/* omitted */}, + "Build": {/* omitted */}, + "Tests": { + "ResultsPath": "./results", + "SkipList": [], + "OnlyList": [ + { + "testsName": "CqlAggregateTest", + "groupName": "AggregateTests", + "testName": "RolledOutIntervals" + }, + // add more tests to only run as necessary... + ] + }, + "Debug": {/* ommitted */} +} +``` + Create your own configuration file and reference it when running the commands. You can use `conf/localhost.json` as a template for a new configuration with your own settings. ### Time Zone Configuration @@ -311,14 +361,16 @@ If using vscode for development, below are some examples for running the tests w ```json { - "type": "node", - "request": "launch", - "name": "Launch Build Command", - "skipFiles": ["/**"], - "program": "${workspaceFolder}/src/bin/cql-tests.ts", - "args": ["build-cql", "conf/localhost.json"], - "runtimeArgs": ["--import", "tsx"] -}, + "type": "node", + "request": "launch", + "name": "Launch Build Command", + "skipFiles": ["/**"], + "program": "${workspaceFolder}/src/bin/cql-tests.ts", + "args": ["build-cql", "conf/localhost.json"], + "runtimeArgs": ["--import", "tsx"] +} +``` +```json { "type": "node", "request": "launch", @@ -329,7 +381,7 @@ If using vscode for development, below are some examples for running the tests w "runtimeArgs": ["--import", "tsx"], "env": { "SERVER_BASE_URL": "http://localhost:3000" - } + } } ``` diff --git a/assets/schema/cql-test-configuration.schema.json b/assets/schema/cql-test-configuration.schema.json index 121416b..ab5edf5 100644 --- a/assets/schema/cql-test-configuration.schema.json +++ b/assets/schema/cql-test-configuration.schema.json @@ -31,49 +31,49 @@ "type": "object", "description": "Build configuration settings", "properties": { - "CqlFileVersion": { - "type": "string", - "pattern": "^\\d+\\.\\d+\\.\\d+$", - "description": "CQL file version in semantic versioning format" - }, - "CqlOutputPath": { - "type": "string", - "description": "Path where CQL files are output" - }, - "CqlVersion": { - "type": "string", - "pattern": "^\\d+\\.\\d+(\\.\\d+)?$", - "description": "CQL engine version for version-based test filtering (optional)" - }, - "testsRunDescription": { - "type": "string", - "description": "Information about this test run" - }, - "cqlTranslator": { - "type": "string", - "description": "Which CQL Translator is used in this test run." - }, - "cqlTranslatorVersion": { - "type": "string", - "description": "Which CQL translator version is used in this test run" - }, - "cqlEngine": { - "type": "string", - "description": "Which CQL engine is used in this test run" - }, - "cqlEngineVersion": { - "type": "string", - "description": "Which version of the CQL engine is used in this test run" - }, - "SERVER_OFFSET_ISO": { - "type": "string", - "description": "ISO 8601 formatted date string to offset the server start time" - }, - "TimeZoneOffsetPolicy": { - "type": "string", - "description": "Which timezone offset policy is used in this test run." - } + "CqlFileVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "description": "CQL file version in semantic versioning format" + }, + "CqlOutputPath": { + "type": "string", + "description": "Path where CQL files are output" + }, + "CqlVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+(\\.\\d+)?$", + "description": "CQL engine version for version-based test filtering (optional)" + }, + "testsRunDescription":{ + "type": "string", + "description": "Information about this test run" + }, + "cqlTranslator": { + "type": "string", + "description": "Which CQL Translator is used in this test run." + }, + "cqlTranslatorVersion": { + "type": "string", + "description": "Which CQL translator version is used in this test run" + }, + "cqlEngine": { + "type": "string", + "description": "Which CQL engine is used in this test run" + }, + "cqlEngineVersion": { + "type": "string", + "description": "Which version of the CQL engine is used in this test run" + }, + "SERVER_OFFSET_ISO": { + "type": "string", + "description": "ISO 8601 formatted date string to offset the server start time" }, + "TimeZoneOffsetPolicy": { + "type": "string", + "description": "Which timezone offset policy is used in this test run." + } + }, "required": ["CqlFileVersion", "CqlOutputPath"], "additionalProperties": false }, @@ -123,6 +123,29 @@ "required": ["testsName", "groupName", "testName", "reason"], "additionalProperties": false } + }, + "OnlyList": { + "type": "array", + "description": "List of tests to run exclusively", + "items": { + "type": "object", + "properties": { + "testsName": { + "type": "string", + "description": "Name of the test suite" + }, + "groupName": { + "type": "string", + "description": "Name of the test group" + }, + "testName": { + "type": "string", + "description": "Name of the specific test" + } + }, + "required": ["testsName", "groupName", "testName"], + "additionalProperties": false + } } }, "required": ["ResultsPath", "SkipList"], diff --git a/conf/localhost.json b/conf/localhost.json index 2b54a2e..1415778 100644 --- a/conf/localhost.json +++ b/conf/localhost.json @@ -4,16 +4,16 @@ "CqlOperation": "$cql" }, "Build": { - "CqlFileVersion": "1.0.000", - "CqlOutputPath": "./cql", - "CqlVersion": "1.5", - "testsRunDescription": "Local host test run", - "cqlTranslator": "Java CQFramework Translator", - "cqlTranslatorVersion": "Unknown", - "cqlEngine": "Java CQFramework Engine", - "cqlEngineVersion": "4.1.0", - "SERVER_OFFSET_ISO": "-06:00", - "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" + "CqlFileVersion": "1.0.000", + "CqlOutputPath": "./cql", + "CqlVersion": "1.5", + "testsRunDescription": "Local host test run", + "cqlTranslator": "Java CQFramework Translator", + "cqlTranslatorVersion": "Unknown", + "cqlEngine": "Java CQFramework Engine", + "cqlEngineVersion": "4.1.0", + "SERVER_OFFSET_ISO": "-06:00", + "TimeZoneOffsetPolicy": "timezone-offset-policy.default-server-offset" }, "Debug": { "QuickTest": false diff --git a/package.json b/package.json index 7d474e9..34028cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cql-tests-runner", - "version": "1.5.0", + "version": "1.7.1", "description": "Server API and command line tools for running CQL tests", "type": "module", "main": "dist/bin/cql-tests.js", @@ -25,17 +25,17 @@ "author": "", "license": "Apache-2.0", "dependencies": { - "@modelcontextprotocol/sdk": "^1.25.3", + "@modelcontextprotocol/sdk": "^1.29.0", "@types/uuid": "^11.0.0", "antlr4": "4.13.2", - "axios": "^1.13.3", + "axios": "^1.14.0", "colors": "^1.4.0", - "commander": "^14.0.2", - "config": "^4.2.0", + "commander": "^14.0.3", + "config": "^4.4.1", "cors": "^2.8.6", "date-fns": "^4.1.0", "express": "^5.2.1", - "fast-xml-parser": "^5.3.3", + "fast-xml-parser": "^5.5.9", "uuid": "^13.0.0", "zod": "^4.3.6", "zod-from-json-schema": "^0.5.2" @@ -44,13 +44,13 @@ "@types/cors": "^2.8.19", "@types/express": "^5.0.6", "@types/fhir": "^0.0.41", - "@types/node": "^25.0.10", - "@types/supertest": "^6.0.3", + "@types/node": "^25.5.0", + "@types/supertest": "^7.2.0", "prettier": "^3.8.1", "supertest": "^7.2.2", "ts-node": "^10.9.2", "tsx": "^4.21.0", - "typescript": "^5.9.3", - "vitest": "^4.0.18" + "typescript": "^6.0.2", + "vitest": "^4.1.2" } } diff --git a/src/bin/cql-tests.ts b/src/bin/cql-tests.ts index 542d6d0..db035a5 100644 --- a/src/bin/cql-tests.ts +++ b/src/bin/cql-tests.ts @@ -43,8 +43,7 @@ program ); process.exit(1); } - console.error(error); - process.exit(1); + throw error; } }); @@ -74,8 +73,7 @@ program ); process.exit(1); } - console.error(error); - process.exit(1); + throw error; } }); @@ -94,7 +92,4 @@ program await serverCommand.start(); }); -program.parseAsync().catch((error: any) => { - console.error(error); - process.exit(1); -}); +program.parse(); diff --git a/src/commands/run-tests-command.ts b/src/commands/run-tests-command.ts index 3b601f4..753450f 100644 --- a/src/commands/run-tests-command.ts +++ b/src/commands/run-tests-command.ts @@ -54,13 +54,38 @@ export class RunCommand { cqlTranslator: config.Build?.cqlTranslator, cqlTranslatorVersion: config.Build?.cqlTranslatorVersion, cqlEngine: config.Build?.cqlEngine, - cqlEngineVersion: config.Build?.cqlEngineVersion, - SERVER_OFFSET_ISO: config.Build?.SERVER_OFFSET_ISO, - TimeZoneOffsetPolicy: config.Build?.TimeZoneOffsetPolicy, + cqlEngineVersion: config.Build?.cqlEngineVersion }, Tests: { ResultsPath: config.Tests.ResultsPath, - SkipList: config.Tests.SkipList, + SkipList: (() => { + const env = process.env.SKIP_LIST; + if (env !== undefined) { + try { + const parsed = JSON.parse(env); + return Array.isArray(parsed) ? parsed : []; + } catch { + console.warn( + 'Failed to parse SKIP_LIST environment variable. Falling back to SkipList in config file.' + ); + } + } + return config.Tests.SkipList; + })(), + OnlyList: (() => { + const env = process.env.ONLY_LIST; + if (env !== undefined) { + try { + const parsed = JSON.parse(env); + return Array.isArray(parsed) ? parsed : []; + } catch { + console.warn( + 'Failed to parse ONLY_LIST environment variable. Falling back to OnlyList in config file.' + ); + } + } + return config.Tests.OnlyList || []; + })() }, Debug: { QuickTest: quick !== undefined ? quick : config.Debug.QuickTest, diff --git a/src/conf/config-loader.ts b/src/conf/config-loader.ts index ff7bcbe..c7aabee 100644 --- a/src/conf/config-loader.ts +++ b/src/conf/config-loader.ts @@ -1,6 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import { Config, SkipItem } from '../models/config-types.js'; +import { Config, SkipItem, OnlyItem } from '../models/config-types.js'; import { ConfigValidator, ValidationError } from './config-validator.js'; export class ConfigLoader implements Config { @@ -23,6 +23,7 @@ export class ConfigLoader implements Config { Tests: { ResultsPath: string; SkipList: SkipItem[]; + OnlyList?: OnlyItem[]; }; Debug: { QuickTest: boolean; @@ -73,7 +74,34 @@ export class ConfigLoader implements Config { this.Tests = { ResultsPath: process.env.RESULTS_PATH || configData.Tests?.ResultsPath || './results', - SkipList: process.env.SKIP_LIST || configData.Tests?.SkipList || [], + SkipList: (() => { + const env = process.env.SKIP_LIST; + if (env !== undefined) { + try { + const parsed = JSON.parse(env); + return Array.isArray(parsed) ? parsed : []; + } catch { + console.warn( + 'Failed to parse SKIP_LIST environment variable. Falling back to SkipList in config file.' + ); + } + } + return configData.Tests?.SkipList || []; + })(), + OnlyList: (() => { + const env = process.env.ONLY_LIST; + if (env !== undefined) { + try { + const parsed = JSON.parse(env); + return Array.isArray(parsed) ? parsed : []; + } catch { + console.warn( + 'Failed to parse ONLY_LIST environment variable. Falling back to OnlyList in config file.' + ); + } + } + return configData.Tests?.OnlyList || []; + })() }; this.Debug = { @@ -148,6 +176,15 @@ export class ConfigLoader implements Config { return skipMap; } + onlyListSet(): Set { + const onlyList = this.Tests.OnlyList ?? []; + return new Set( + onlyList.map( + onlyItem => `${onlyItem.testsName}-${onlyItem.groupName}-${onlyItem.testName}` + ) + ); + } + #validateConfig(configPath: string, configData: any): void { try { const validator = new ConfigValidator(); diff --git a/src/cql-engine/cql-engine.ts b/src/cql-engine/cql-engine.ts index e2730d5..fbbdb85 100644 --- a/src/cql-engine/cql-engine.ts +++ b/src/cql-engine/cql-engine.ts @@ -42,33 +42,27 @@ export class CQLEngine { private description?: string; private _SERVER_OFFSET_ISO?: string; - /** - * Creates a new CQLEngine instance. - * - * @param baseURL Base URL of the FHIR server (e.g. http://localhost:8080/fhir/$cql) - * @param cqlPath Optional path to local CQL files - * @param cqlTranslator Name of the CQL translator - * @param cqlTranslatorVersion Version of the CQL translator - * @param cqlEngine Name of the CQL engine - * @param cqlEngineVersion Version of the CQL engine - * @param SERVER_OFFSET_ISO Timezone offset used by the server (e.g. "+00:00", "-06:00") - */ constructor( - baseURL: string, - cqlPath: string | null = null, - cqlTranslator: string = '', - cqlTranslatorVersion: string = '', - cqlEngine: string = '', - cqlEngineVersion: string = '', - SERVER_OFFSET_ISO: string = '' + /** + * Creates an instance of CQLEngine. + * @param baseURL - The base URL for the CQL engine. + * @param cqlPath - The path for the CQL engine (optional). + * @param cqlTranslator - CQL translator name (optional, from config Build). + * @param cqlTranslatorVersion - CQL translator version (optional). + * @param cqlEngine - CQL engine name (optional). + * @param cqlEngineVersion - CQL engine version (optional). + */ + constructor( + baseURL: string, + cqlPath: string | null = null, + cqlTranslator: string = '', + cqlTranslatorVersion: string = '', + cqlEngine: string = '', + cqlEngineVersion: string = '', + SERVER_OFFSET_ISO: string = '' ) { - this._prepareBaseURL(baseURL, cqlPath); - this._setInformationFields( - cqlTranslator, - cqlTranslatorVersion, - cqlEngine, - cqlEngineVersion - ); - this.SERVER_OFFSET_ISO = SERVER_OFFSET_ISO; + this._prepareBaseURL(baseURL, cqlPath); + this._setInformationFields(cqlTranslator, cqlTranslatorVersion, cqlEngine, cqlEngineVersion); + this.SERVER_OFFSET_ISO = SERVER_OFFSET_ISO; } /** @@ -104,31 +98,31 @@ export class CQLEngine { private _setInformationFields(cqlTranslator: string, cqlTranslatorVersion: string, cqlEngine: string, cqlEngineVersion: string) { - this.info.cqlTranslator = cqlTranslator; - this.info.cqlTranslatorVersion = cqlTranslatorVersion; - this.info.cqlEngine = cqlEngine; - this.info.cqlEngineVersion = cqlEngineVersion; - } + this.info.cqlTranslator = cqlTranslator; + this.info.cqlTranslatorVersion = cqlTranslatorVersion; + this.info.cqlEngine = cqlEngine; + this.info.cqlEngineVersion = cqlEngineVersion; + } - /** - * Fetches metadata from the CQL engine. - * @param force - Whether to force fetching metadata. - * @returns A Promise that resolves when metadata is fetched. - */ - async fetch(force: boolean = false): Promise { - if (this.baseURL) { - if (!this.metadata || force) { - try { - const response: AxiosResponse = await axios.get(`${this.baseURL}/metadata`); - if (response?.data) { - this.metadata = response.data; - } - } catch (e) { - console.error(e); - } - } - } - } + /** + * Fetches metadata from the CQL engine. + * @param force - Whether to force fetching metadata. + * @returns A Promise that resolves when metadata is fetched. + */ + async fetch(force: boolean = false): Promise { + if (this.baseURL) { + if (!this.metadata || force) { + try { + const response: AxiosResponse = await axios.get(`${this.baseURL}/metadata`); + if (response?.data) { + this.metadata = response.data; + } + } catch (e) { + console.error(e); + } + } + } + } /** * Sets the API URL. @@ -258,15 +252,9 @@ export class CQLEngine { * @returns The CQLEngine instance. */ static fromJSON(cqlInfo: CQLEngineInfo): CQLEngine { - const engine = new CQLEngine( - cqlInfo.apiUrl || '', - null, - cqlInfo.cqlTranslator, - cqlInfo.cqlTranslatorVersion, - cqlInfo.cqlEngine, - cqlInfo.cqlEngineVersion, - cqlInfo.SERVER_OFFSET_ISO - ); + const engine = new CQLEngine(cqlInfo.apiUrl || '', null, + cqlInfo.cqlTranslator, cqlInfo.cqlTranslatorVersion, + cqlInfo.cqlEngine, cqlInfo.cqlEngineVersion, cqlInfo.SERVER_OFFSET_ISO); if (cqlInfo?.cqlVersion) { engine.cqlVersion = cqlInfo.cqlVersion; } diff --git a/src/extractors/evaluation-error-extractor.ts b/src/extractors/evaluation-error-extractor.ts index cdcdec8..90d7e6d 100644 --- a/src/extractors/evaluation-error-extractor.ts +++ b/src/extractors/evaluation-error-extractor.ts @@ -1,9 +1,19 @@ import { BaseExtractor } from './base-extractor.js'; +function operationOutcomeMessage(resource: any): string { + const issue = resource?.issue?.[0]; + const fromIssue = + issue?.details?.text ?? issue?.diagnostics ?? (issue != null ? JSON.stringify(issue) : undefined); + if (fromIssue != null) { + return fromIssue; + } + return resource != null ? JSON.stringify(resource) : 'unknown'; +} + export class EvaluationErrorExtractor extends BaseExtractor { protected _process(parameter: any): any { if (parameter.name === 'evaluation error') { - return `EvaluationError:${parameter.resource.issue[0].details.text}`; + return `EvaluationError:${operationOutcomeMessage(parameter.resource)}`; } return undefined; diff --git a/src/extractors/result-extractor.ts b/src/extractors/result-extractor.ts index a53fb70..ea67e67 100644 --- a/src/extractors/result-extractor.ts +++ b/src/extractors/result-extractor.ts @@ -1,6 +1,11 @@ import { ValueMap } from './value-map.js'; import { BaseExtractor } from './base-extractor.js'; +export type ExtractOptions = { + /** Keys that keep a single value as a one-element array after collapse. */ + singletonListKeys?: ReadonlySet; +}; + export class ResultExtractor { private extractors: BaseExtractor; @@ -8,11 +13,14 @@ export class ResultExtractor { this.extractors = extractors; } - private _extractValues(parameters: any[]): ValueMap { - const values = new ValueMap(); + private _extractValues( + parameters: any[], + singletonListKeys: ReadonlySet + ): ValueMap { + const values = new ValueMap(singletonListKeys); for (const parameter of parameters) { const value = parameter.hasOwnProperty('part') - ? this._extractValues(parameter.part) + ? this._extractValues(parameter.part, singletonListKeys) : this.extractors.extractValue(parameter); // If the value is a ValueMap, convert it to a plain object/array @@ -22,7 +30,7 @@ export class ResultExtractor { return values; } - extract(response: any): any { + extract(response: any, options?: ExtractOptions): any { if (!response.hasOwnProperty('resourceType') || response.resourceType !== 'Parameters') { // Anything that can't be structured directly, return as the actual output... return JSON.stringify(response); @@ -32,7 +40,8 @@ export class ResultExtractor { return 'undefined'; } - const extracted_values = this._extractValues(response.parameter); + const singletonListKeys = options?.singletonListKeys ?? new Set(); + const extracted_values = this._extractValues(response.parameter, singletonListKeys); return extracted_values.toResult(); } } diff --git a/src/extractors/value-map.ts b/src/extractors/value-map.ts index c3ede94..a6da420 100644 --- a/src/extractors/value-map.ts +++ b/src/extractors/value-map.ts @@ -2,6 +2,38 @@ export class ValueMap { static NON_NAMED_KEYWORDS = ['return', 'element', 'evaluation error']; private map = new Map(); + /** + * Builds the key set for {@link ValueMap}'s `singletonListKeys` from a parsed CQL + * expected outcome (from `cvl.parse`), so FHIR list encoding matches expected List-typed expectations. + * For: https://github.com/cqframework/cql-tests-runner/issues/82 + */ + static singletonListKeysFromExpected(parsed: unknown): ReadonlySet { + const keys = new Set(); + if (!Array.isArray(parsed)) { + return keys; + } + // Empty list `{}` parses to `[]`. Do not mark `return` as singleton-list: the + // extractor already yields `[]` for FHIR empty-list encoding; wrapping would + // produce `[[]]` vs expected `[]` (see cqframework/cql-tests-runner#90). + if (parsed.length === 0) { + return keys; + } + keys.add('return'); + for (const item of parsed) { + if (Array.isArray(item)) { + keys.add('element'); + } + } + return keys; + } + + /** + * @param singletonListKeys β€” parameter names for which a single collected value + * stays wrapped as a one-element array (matches CQL list vs scalar distinction + * when the test expectation is a list shape). + */ + constructor(private readonly singletonListKeys: ReadonlySet = new Set()) {} + add(key: string, value: any): void { if (!this.map.has(key)) { this.map.set(key, [value]); @@ -14,7 +46,11 @@ export class ValueMap { const collapsed_map = new Map(); for (const [key, value] of this.map) { if (value.length === 1) { - collapsed_map.set(key, value[0]); + if (this.singletonListKeys.has(key)) { + collapsed_map.set(key, [...value]); + } else { + collapsed_map.set(key, value[0]); + } } else if (value.length > 1) { collapsed_map.set(key, value); } diff --git a/src/jobs/job-processor.ts b/src/jobs/job-processor.ts index 9ce636f..e56198a 100644 --- a/src/jobs/job-processor.ts +++ b/src/jobs/job-processor.ts @@ -30,7 +30,7 @@ export class JobProcessor { // Process the tests using the shared TestRunner const results = await this.testRunner.runTests(jobRequest.config, { - onProgress: async (current: number, total: number, message?: string) => { + onProgress: async (current, total, message) => { await this.jobManager.updateJobProgress(jobId, current, total, message); }, }); diff --git a/src/loaders/test-loader.ts b/src/loaders/test-loader.ts index 5f7da49..ef7317c 100644 --- a/src/loaders/test-loader.ts +++ b/src/loaders/test-loader.ts @@ -1,19 +1,18 @@ import * as fs from 'fs'; import * as path from 'path'; -import { XMLParser } from 'fast-xml-parser'; +import { XMLParser, type X2jOptions } from 'fast-xml-parser'; import { Tests } from '../models/test-types.js'; const testsPath = 'cql-tests/tests/cql'; const alwaysArray = ['tests.group', 'tests.group.test']; -const options = { +const options: X2jOptions = { ignoreAttributes: false, attributeNamePrefix: '', parseTagValue: false, - isArray: (name: string, jpath: string, isLeafNode: boolean, isAttribute: boolean): boolean => { - return alwaysArray.indexOf(jpath) !== -1; - }, + isArray: (name, jPathOrMatcher, isLeafNode, isAttribute) => + typeof jPathOrMatcher === 'string' && alwaysArray.indexOf(jPathOrMatcher) !== -1, textNodeName: 'text', }; diff --git a/src/models/config-types.ts b/src/models/config-types.ts index 396c294..16c7d2e 100644 --- a/src/models/config-types.ts +++ b/src/models/config-types.ts @@ -5,6 +5,12 @@ export interface SkipItem { reason: string; } +export interface OnlyItem { + testsName: string; + groupName: string; + testName: string; +} + // Schema-compliant Config type (strictly matches cql-test-configuration.schema.json) export interface Config { FhirServer: { @@ -16,12 +22,17 @@ export interface Config { CqlFileVersion: string; CqlOutputPath: string; CqlVersion?: string; - testsRunDescription?: string; // Note: schema has this misplaced but it's used in code + testsRunDescription?: string; + cqlTranslator?: string; + cqlTranslatorVersion?: string; + cqlEngine?: string; + cqlEngineVersion?: string; SERVER_OFFSET_ISO: string; }; Tests: { ResultsPath: string; SkipList: SkipItem[]; + OnlyList?: OnlyItem[]; }; Debug: { QuickTest: boolean; diff --git a/src/models/test-types.ts b/src/models/test-types.ts index d0e1964..821b0de 100644 --- a/src/models/test-types.ts +++ b/src/models/test-types.ts @@ -64,6 +64,7 @@ export interface CapabilityKV { // Internal Result type used during test execution (allows 'undefined' for invalid) export interface InternalTestResult { testStatus?: 'pass' | 'fail' | 'skip' | 'error'; + skipMessage?: string; responseStatus?: number; actual?: any; expected?: string; @@ -87,6 +88,7 @@ export interface InternalTestResult { // Schema-compliant TestResult type (strictly matches cql-test-results.schema.json) export interface TestResult { testStatus?: 'pass' | 'fail' | 'skip' | 'error'; + skipMessage?: string; responseStatus?: number; actual?: string; expected?: string; diff --git a/src/server/config-utils.ts b/src/server/config-utils.ts index b5626b3..dc3ac42 100644 --- a/src/server/config-utils.ts +++ b/src/server/config-utils.ts @@ -50,21 +50,48 @@ export function createConfigFromData(configData: any): ConfigLoader { }; config.Build = { - CqlFileVersion: process.env.CQL_FILE_VERSION || configData.Build?.CqlFileVersion || '1.0.000', - CqlOutputPath: process.env.CQL_OUTPUT_PATH || configData.Build?.CqlOutputPath || './cql', - CqlVersion: process.env.CQL_VERSION || configData.Build?.CqlVersion, - testsRunDescription: process.env.TESTS_RUN_DESCRIPTION || configData.Build?.testsRunDescription || 'No configuration provided', - cqlTranslator: process.env.CQL_TRANSLATOR || configData.Build?.cqlTranslator || 'No configuration provided', - cqlTranslatorVersion: process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'No configuration provided', - cqlEngine: process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'No configuration provided', - cqlEngineVersion: process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'No configuration provided', - SERVER_OFFSET_ISO: process.env.SERVER_OFFSET_ISO || configData.Build?.SERVER_OFFSET_ISO || '+00:00', - TimeZoneOffsetPolicy: process.env.TIME_ZONE_OFFSET_POLICY || configData.Build?.TimeZoneOffsetPolicy || '', + CqlFileVersion: process.env.CQL_FILE_VERSION || configData.Build?.CqlFileVersion || '1.0.000', + CqlOutputPath: process.env.CQL_OUTPUT_PATH || configData.Build?.CqlOutputPath || './cql', + CqlVersion: process.env.CQL_VERSION || configData.Build?.CqlVersion, + testsRunDescription: process.env.TESTS_RUN_DESCRIPTION || configData.Build?.testsRunDescription || 'No configuration provided', + cqlTranslator: process.env.CQL_TRANSLATOR || configData.Build?.cqlTranslator || 'No configuration provided', + cqlTranslatorVersion: process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'No configuration provided', + cqlEngine: process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'No configuration provided', + cqlEngineVersion: process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'No configuration provided', + SERVER_OFFSET_ISO: process.env.SERVER_OFFSET_ISO || configData.Build?.SERVER_OFFSET_ISO || '+00:00', + TimeZoneOffsetPolicy: process.env.TIME_ZONE_OFFSET_POLICY || configData.Build?.TimeZoneOffsetPolicy || '', }; config.Tests = { ResultsPath: process.env.RESULTS_PATH || configData.Tests?.ResultsPath || './results', - SkipList: process.env.SKIP_LIST || configData.Tests?.SkipList || [] + SkipList: (() => { + const env = process.env.SKIP_LIST; + if (env !== undefined) { + try { + const parsed = JSON.parse(env); + return Array.isArray(parsed) ? parsed : []; + } catch { + console.warn( + 'Failed to parse SKIP_LIST environment variable. Falling back to SkipList in config file.' + ); + } + } + return configData.Tests?.SkipList || []; + })(), + OnlyList: (() => { + const env = process.env.ONLY_LIST; + if (env !== undefined) { + try { + const parsed = JSON.parse(env); + return Array.isArray(parsed) ? parsed : []; + } catch { + console.warn( + 'Failed to parse ONLY_LIST environment variable. Falling back to OnlyList in config file.' + ); + } + } + return configData.Tests?.OnlyList || []; + })() }; config.Debug = { diff --git a/src/server/test-execution-service.ts b/src/server/test-execution-service.ts index 5733322..c10e672 100644 --- a/src/server/test-execution-service.ts +++ b/src/server/test-execution-service.ts @@ -7,17 +7,58 @@ import { generateParametersResource, Result, } from '../shared/results-shared.js'; -import { InternalTestResult } from '../models/test-types.js'; +import { InternalTestResult, Tests } from '../models/test-types.js'; import { ServerConnectivity } from '../shared/server-connectivity.js'; import { ResultExtractor } from '../extractors/result-extractor.js'; import { buildExtractor } from './extractor-builder.js'; import { createConfigFromData } from './config-utils.js'; +import { ValueMap } from '../extractors/value-map.js'; import { resultsEqual } from '../shared/results-utils.js'; -// Type declaration for CVL loader -declare const cvlLoader: () => Promise<[{ default: any }]>; +interface ExecutionContext { + config: ConfigLoader; + cqlEngine: CQLEngine; + cvl: any; + tests: Tests[]; + resultExtractor: ResultExtractor; + skipMap: Map; + onlySet: Set; +} export class TestExecutionService { + /** + * Builds shared execution context from config data (engine, CVL, tests, extractor, skip map). + */ + private async createExecutionContext(configData: any): Promise { + const config = createConfigFromData(configData); + const serverBaseUrl = config.FhirServer.BaseUrl; + const cqlEndpoint = config.CqlEndpoint; + + await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); + + const build = config.Build; + const cqlEngine = new CQLEngine( + serverBaseUrl, + cqlEndpoint, + build?.cqlTranslator ?? '', + build?.cqlTranslatorVersion ?? '', + build?.cqlEngine ?? '', + build?.cqlEngineVersion ?? '' + ); + cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5'; + + // @ts-expect-error - cvl.mjs has no declaration file + const cvlModule = await import('../../cvl/cvl.mjs'); + const cvl = cvlModule.default; + + const tests = TestLoader.load(); + const resultExtractor = buildExtractor(); + const skipMap = config.skipListMap(); + const onlySet = config.onlyListSet(); + + return { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet }; + } + /** * Runs a single test */ @@ -27,6 +68,7 @@ export class TestExecutionService { cvl: any, resultExtractor: ResultExtractor, skipMap: Map, + onlySet: Set, config: ConfigLoader ): Promise { const key = `${result.testsName}-${result.groupName}-${result.testName}`; @@ -34,6 +76,10 @@ export class TestExecutionService { if (result.testStatus === 'skip') { result.SkipMessage = 'Skipped by cql-tests-runner'; return result; + } else if (onlySet.size > 0 && !onlySet.has(key)) { + result.SkipMessage = 'Skipped by OnlyList filter'; + result.testStatus = 'skip'; + return result; } else if (skipMap.has(key)) { const reason = skipMap.get(key) || ''; result.SkipMessage = `Skipped by config: ${reason}`; @@ -55,14 +101,17 @@ export class TestExecutionService { result.responseStatus = response.status; const responseBody = await response.json(); - result.actual = resultExtractor.extract(responseBody); + const parsedExpected = cvl.parse(result.expected); + result.actual = resultExtractor.extract(responseBody, { + singletonListKeys: ValueMap.singletonListKeysFromExpected(parsedExpected), + }); const invalid = result.invalid; if (invalid === 'true' || invalid === 'semantic') { result.testStatus = response.status === 200 ? 'fail' : 'pass'; } else { if (response.status === 200) { - result.testStatus = resultsEqual(cvl.parse(result.expected), result.actual) ? 'pass' : 'fail'; + result.testStatus = resultsEqual(parsedExpected, result.actual) ? 'pass' : 'fail'; } else { result.testStatus = 'fail'; } @@ -86,38 +135,20 @@ export class TestExecutionService { * Runs all tests based on configuration */ async runTests(configData: any): Promise { - // Create a temporary config loader from the provided data - const config = createConfigFromData(configData); - const serverBaseUrl = config.FhirServer.BaseUrl; - const cqlEndpoint = config.CqlEndpoint; - - // Verify server connectivity before proceeding - await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); - - const cqlEngine = new CQLEngine(serverBaseUrl, cqlEndpoint); - cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5'; - - // Load CVL using dynamic import - // @ts-ignore - const cvlModule = await import('../../cvl/cvl.mjs'); - const cvl = cvlModule.default; + const ctx = await this.createExecutionContext(configData); + const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx; - const tests = TestLoader.load(); const quickTest = config.Debug?.QuickTest || false; - const resultExtractor = buildExtractor(); const emptyResults = await generateEmptyResults(tests, quickTest); - const skipMap = config.skipListMap(); - const results = new CQLTestResults(cqlEngine); for (const testFile of emptyResults) { for (const result of testFile) { - await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, config); + await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config); results.add(result); } } - // Return the results data that would normally be written to file return results.toJSON(); } @@ -130,50 +161,22 @@ export class TestExecutionService { testName: string, configData: any ): Promise { - const config = createConfigFromData(configData); - const serverBaseUrl = config.FhirServer.BaseUrl; - const cqlEndpoint = config.CqlEndpoint; - - await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); - - const cqlEngine = new CQLEngine(serverBaseUrl, cqlEndpoint); - cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5'; - - // @ts-ignore - const cvlModule = await import('../../cvl/cvl.mjs'); - const cvl = cvlModule.default; - - const tests = TestLoader.load(); - const resultExtractor = buildExtractor(); - const skipMap = config.skipListMap(); + const ctx = await this.createExecutionContext(configData); + const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx; - // Find the specific test for (const testSuite of tests) { - if (testSuite.name === testsName) { - for (const group of testSuite.group) { - if (group.name === groupName && group.test) { - for (const test of group.test) { - if (test.name === testName) { - const result = new Result(testsName, groupName, test); - await this.runTest( - result, - cqlEngine.apiUrl!, - cvl, - resultExtractor, - skipMap, - config - ); - - // Convert to schema-compliant format - const testResults = new CQLTestResults(cqlEngine); - testResults.add(result); - const jsonResults = testResults.toJSON(); - - // Return just the single test result - return jsonResults.results[0] || null; - } - } - } + if (testSuite.name !== testsName) continue; + for (const group of testSuite.group) { + if (group.name !== groupName || !group.test) continue; + for (const test of group.test) { + if (test.name !== testName) continue; + + const result = new Result(testsName, groupName, test); + await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config); + + const testResults = new CQLTestResults(cqlEngine); + testResults.add(result); + return testResults.toJSON().results[0] ?? null; } } } @@ -189,50 +192,24 @@ export class TestExecutionService { groupName: string, configData: any ): Promise { - const config = createConfigFromData(configData); - const serverBaseUrl = config.FhirServer.BaseUrl; - const cqlEndpoint = config.CqlEndpoint; - - await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); - - const cqlEngine = new CQLEngine(serverBaseUrl, cqlEndpoint); - cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5'; - - // @ts-ignore - const cvlModule = await import('../../cvl/cvl.mjs'); - const cvl = cvlModule.default; - - const tests = TestLoader.load(); - const resultExtractor = buildExtractor(); - const skipMap = config.skipListMap(); + const ctx = await this.createExecutionContext(configData); + const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx; const results = new CQLTestResults(cqlEngine); - // Find and run tests in the specified group for (const testSuite of tests) { - if (testSuite.name === testsName) { - for (const group of testSuite.group) { - if (group.name === groupName && group.test) { - for (const test of group.test) { - const result = new Result(testsName, groupName, test); - await this.runTest( - result, - cqlEngine.apiUrl!, - cvl, - resultExtractor, - skipMap, - config - ); - results.add(result); - } - break; - } + if (testSuite.name !== testsName) continue; + for (const group of testSuite.group) { + if (group.name !== groupName || !group.test) continue; + for (const test of group.test) { + const result = new Result(testsName, groupName, test); + await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config); + results.add(result); } - break; + return results.toJSON().results; } } - const jsonResults = results.toJSON(); - return jsonResults.results; + return results.toJSON().results; } } diff --git a/src/services/test-runner.ts b/src/services/test-runner.ts index 732b89d..2afe9fe 100644 --- a/src/services/test-runner.ts +++ b/src/services/test-runner.ts @@ -8,6 +8,7 @@ import { ResultExtractor } from '../extractors/result-extractor.js'; import { ServerConnectivity } from '../shared/server-connectivity.js'; import { buildExtractor } from '../server/extractor-builder.js'; import { createConfigFromData } from '../server/config-utils.js'; +import { ValueMap } from '../extractors/value-map.js'; import { resultsEqual } from '../shared/results-utils.js'; export interface TestRunnerOptions { @@ -28,15 +29,16 @@ export class TestRunner { // Verify server connectivity before proceeding await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); + const build = config.Build; const cqlEngine = new CQLEngine( - serverBaseUrl, - cqlEndpoint, - configData.Build.cqlTranslator, - configData.Build.cqlTranslatorVersion, - configData.Build.cqlEngine, - configData.Build.cqlEngineVersion, - configData.Build.SERVER_OFFSET_ISO - ); + serverBaseUrl, + cqlEndpoint, + build.cqlTranslator ?? '', + build.cqlTranslatorVersion ?? '', + build.cqlEngine ?? '', + build.cqlEngineVersion ?? '', + build.SERVER_OFFSET_ISO + ); cqlEngine.cqlVersion = '1.5'; //default value const cqlVersion = config.Build?.CqlVersion; if (typeof cqlVersion === 'string' && cqlVersion.trim() !== '') { @@ -62,6 +64,7 @@ export class TestRunner { const resultExtractor = buildExtractor(); const emptyResults = await generateEmptyResults(tests, quickTest); const skipMap = config.skipListMap(); + const onlySet = config.onlyListSet(); const results = new CQLTestResults(cqlEngine); @@ -71,12 +74,12 @@ export class TestRunner { for (const testFile of emptyResults) { for (const result of testFile) { if (this.shouldSkipVersionTest(cqlEngine, result)) { - //add to skipMap const skipReason = - 'test version ' + - result.testVersion + - ' not applicable to engine version ' + - cqlEngine.cqlVersion; + result.testVersionTo && + this.compareVersions(cqlEngine.cqlVersion, result.testVersionTo) > 0 + ? `test versionTo ${result.testVersionTo} not applicable to engine version ${cqlEngine.cqlVersion}` + : `test version ${result.testVersion} not applicable to engine version ${cqlEngine.cqlVersion}`; + this.addToSkipList( skipMap, result.testsName, @@ -91,6 +94,7 @@ export class TestRunner { cvl, resultExtractor, skipMap, + onlySet, config, cqlEngine, activeTimeZonePolicy, @@ -118,6 +122,7 @@ export class TestRunner { cvl: any, resultExtractor: ResultExtractor, skipMap: Map, + onlySet: Set, config: ConfigLoader, cqlEngine: CQLEngine, activeTimeZonePolicy: string, @@ -126,12 +131,34 @@ export class TestRunner { const key = `${result.testsName}-${result.groupName}-${result.testName}`; if (result.testStatus === 'skip') { - result.SkipMessage = 'Skipped by cql-tests-runner'; + if (!result.skipMessage?.trim()) { + result.skipMessage = 'Skipped by cql-tests-runner'; + } + console.log( + 'Test %s:%s:%s status: %s skipMessage: %s', + result.testsName, + result.groupName, + result.testName, + result.testStatus, + result.skipMessage + ); + return result; + } else if (onlySet.size > 0 && !onlySet.has(key)) { + result.SkipMessage = 'Skipped by OnlyList filter'; + result.testStatus = 'skip'; return result; } else if (skipMap.has(key)) { const reason = skipMap.get(key) || ''; result.SkipMessage = `Skipped by config: ${reason}`; result.testStatus = 'skip'; + console.log( + 'Test %s:%s:%s status: %s skipMessage: %s', + result.testsName, + result.groupName, + result.testName, + result.testStatus, + result.skipMessage + ); return result; } @@ -187,14 +214,17 @@ export class TestRunner { result.responseStatus = response.status; const responseBody = response.data; - result.actual = resultExtractor.extract(responseBody); + const parsedExpected = cvl.parse(result.expected); + result.actual = resultExtractor.extract(responseBody, { + singletonListKeys: ValueMap.singletonListKeysFromExpected(parsedExpected), + }); const invalid = result.invalid; if (invalid === 'true' || invalid === 'semantic') { result.testStatus = response.status === 200 ? 'fail' : 'pass'; } else { if (response.status === 200) { - result.testStatus = resultsEqual(cvl.parse(result.expected), result.actual) + result.testStatus = resultsEqual(parsedExpected, result.actual) ? 'pass' : 'fail'; } else { diff --git a/src/shared/results-shared.ts b/src/shared/results-shared.ts index afb75c2..bdc18a3 100644 --- a/src/shared/results-shared.ts +++ b/src/shared/results-shared.ts @@ -3,6 +3,7 @@ import { Parameters } from 'fhir/r4'; export class Result implements InternalTestResult { testStatus!: 'pass' | 'fail' | 'skip' | 'error'; + skipMessage?: string; responseStatus?: number; actual?: any; expected?: string; @@ -21,12 +22,7 @@ export class Result implements InternalTestResult { capability: CapabilityKV[] = []; groupCapability: CapabilityKV[] = []; - constructor( - testsName: string, - groupName: string, - test: Test, - groupCapability: CapabilityKV[] = [] - ) { + constructor(testsName: string, groupName: string, test: Test, groupCapability: CapabilityKV[] = []) { this.testsName = testsName; this.groupName = groupName; this.testName = test.name; @@ -55,6 +51,7 @@ export class Result implements InternalTestResult { } } else { this.testStatus = 'skip'; + this.skipMessage = 'No output specified'; } this.capability = Array.isArray(test.capability) diff --git a/src/test-results/cql-test-results.ts b/src/test-results/cql-test-results.ts index 4a6df87..eb3b26d 100644 --- a/src/test-results/cql-test-results.ts +++ b/src/test-results/cql-test-results.ts @@ -102,6 +102,7 @@ export class CQLTestResults { expression: result.expression, // Optional fields (only include if present) ...(result.testStatus && { testStatus: result.testStatus }), + ...(result.skipMessage?.trim() && { skipMessage: result.skipMessage.trim() }), ...(result.responseStatus !== undefined && { responseStatus: result.responseStatus, }), diff --git a/test/extractResults-cql_operations.test.ts b/test/extractResults-cql_operations.test.ts index f661bfc..3f751d5 100644 --- a/test/extractResults-cql_operations.test.ts +++ b/test/extractResults-cql_operations.test.ts @@ -1,6 +1,7 @@ import { beforeAll, expect, test } from 'vitest'; import { ResultExtractor } from '../src/extractors/result-extractor.js'; +import { ValueMap } from '../src/extractors/value-map.js'; import { buildExtractor } from '../src/server/extractor-builder.js'; let extractor: ResultExtractor | null = null; @@ -107,6 +108,53 @@ test('string response check', () => { ).toBe('abc'); }); +test('singleton list-typed return stays array when singletonListKeys includes return (issue #82)', () => { + expect( + extractor!.extract( + { + resourceType: 'Parameters', + parameter: [ + { + name: 'return', + valueString: 'a', + }, + ], + }, + { singletonListKeys: new Set(['return']) } + ) + ).toEqual(['a']); +}); + +test('FHIR empty list stays [] when expected is empty list (issue #90)', () => { + const emptyListParameters = { + resourceType: 'Parameters', + parameter: [ + { + name: 'return', + extension: [ + { + url: 'http://hl7.org/fhir/StructureDefinition/cqf-cqlType', + valueString: 'List', + }, + ], + _valueBoolean: { + extension: [ + { + url: 'http://hl7.org/fhir/StructureDefinition/cqf-isEmptyList', + valueBoolean: true, + }, + ], + }, + }, + ], + }; + expect( + extractor!.extract(emptyListParameters, { + singletonListKeys: ValueMap.singletonListKeysFromExpected([]), + }) + ).toEqual([]); +}); + test('date response check', () => { expect( extractor!.extract({ @@ -269,6 +317,32 @@ test('error response check', () => { ); }); +test('evaluation error uses diagnostics when details.text is absent', () => { + expect( + extractor!.extract({ + resourceType: 'Parameters', + parameter: [ + { + name: 'evaluation error', + resource: { + resourceType: 'OperationOutcome', + issue: [{ severity: 'error', diagnostics: 'CQL engine message' }], + }, + }, + ], + }) + ).toBe('EvaluationError:CQL engine message'); +}); + +test('evaluation error does not throw when resource shape is minimal', () => { + expect( + extractor!.extract({ + resourceType: 'Parameters', + parameter: [{ name: 'evaluation error', resource: { resourceType: 'OperationOutcome', issue: [] } }], + }) + ).toBe('EvaluationError:{"resourceType":"OperationOutcome","issue":[]}'); +}); + test('period datetime response check', () => { expect( extractor!.extract({ diff --git a/test/results-utils.test.js b/test/results-utils.test.js new file mode 100644 index 0000000..a1b03fd --- /dev/null +++ b/test/results-utils.test.js @@ -0,0 +1,19 @@ +// Author: Preston Lee + +import { expect, test } from 'vitest'; + +import { resultsEqual } from '../src/shared/results-utils.js'; + +test('singleton list does not equal scalar (comparison stays strict)', () => { + expect(resultsEqual(['a'], 'a')).toBe(false); + expect(resultsEqual('a', ['a'])).toBe(false); +}); + +test('equal lists (order-insensitive)', () => { + expect(resultsEqual(['a', 'b'], ['a', 'b'])).toBe(true); + expect(resultsEqual(['a', 'b'], ['b', 'a'])).toBe(false); +}); + +test('nested structures compared key-wise', () => { + expect(resultsEqual({ x: 1 }, { x: 1 })).toBe(true); +}); diff --git a/test/run-DataTransferItemList.test.ts b/test/run-DataTransferItemList.test.ts new file mode 100644 index 0000000..8224723 --- /dev/null +++ b/test/run-DataTransferItemList.test.ts @@ -0,0 +1,352 @@ +import { describe, it, expect, vi, beforeEach, afterEach, Mock } from 'vitest'; +import { TestRunner } from '../src/services/test-runner.js'; +import * as ResultsUtils from '../src/shared/results-utils.js'; + +vi.mock('../src/loaders/test-loader', () => ({ + TestLoader: { load: vi.fn().mockReturnValue([]) }, +})); + +const makeResult = (testsName: string, groupName: string, testName: string) => ({ + testsName, + groupName, + testName, + expression: 'true', + expected: 'true', + invalid: 'false', + capability: [], +}); + +vi.mock('../src/shared/results-shared', () => ({ + generateEmptyResults: vi + .fn() + .mockImplementation(async () => [ + [makeResult('Suite1', 'Group1', 'Test1'), makeResult('Suite1', 'Group1', 'Test2')], + ]), + generateParametersResource: vi.fn().mockReturnValue({}), +})); + +const baseConfig = { + FhirServer: { BaseUrl: 'http://localhost:8080/fhir', CqlOperation: '$cql' }, + Build: {}, + Debug: { QuickTest: false }, + Tests: { ResultsPath: './results', SkipList: [] as any[], OnlyList: [] as any[] }, +}; + +describe('RunTests filtering (TestRunner)', () => { + let runner: TestRunner; + let fetchSpy: Mock; + let resultsEqualSpy: Mock; + beforeEach(() => { + fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ + status: 200, + json: () => Promise.resolve({}), + } as Response); + resultsEqualSpy = vi.spyOn(ResultsUtils, 'resultsEqual').mockReturnValue(true); + runner = new TestRunner(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + delete (process.env as any).ONLY_LIST; + delete (process.env as any).SKIP_LIST; + }); + + it('runs all tests when SkipList and OnlyList are empty', async () => { + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('pass'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 2, + skipCount: 0, + failCount: 0, + errorCount: 0, + }); + }); + + it('skips tests in SkipList', async () => { + const config = { + FhirServer: { BaseUrl: 'http://localhost:8080/fhir', CqlOperation: '$cql' }, + Build: {}, + Debug: { QuickTest: false }, + Tests: { + ResultsPath: './results', + SkipList: [ + { + testsName: 'Suite1', + groupName: 'Group1', + testName: 'Test1', + reason: 'Disabled', + }, + ], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('skip'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); + + it('runs only tests in OnlyList (others skipped)', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + OnlyList: [{ testsName: 'Suite1', groupName: 'Group1', testName: 'Test2' }], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + // Two tests total in our mock, one should be run (pass), the other skipped + const statuses = results.results.map(r => ({ + key: `${r.testsName}-${r.groupName}-${r.testName}`, + status: r.testStatus, + })); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test2', status: 'pass' }); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test1', status: 'skip' }); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); + + it('skips a test present in both OnlyList and SkipList', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + OnlyList: [{ testsName: 'Suite1', groupName: 'Group1', testName: 'Test2' }], + SkipList: [ + { + testsName: 'Suite1', + groupName: 'Group1', + testName: 'Test2', + reason: 'Disabled', + }, + ], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + expect(test2?.testStatus).toBe('skip'); + expect(test1?.testStatus).toBe('skip'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 0, + skipCount: 2, + failCount: 0, + errorCount: 0, + }); + }); + + it('reports failure for unequal results', async () => { + resultsEqualSpy.mockReturnValueOnce(true).mockReturnValueOnce(false); + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('pass'); + expect(test2?.testStatus).toBe('fail'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 0, + failCount: 1, + errorCount: 0, + }); + }); + + it('reports failure for HTTP 500 result', async () => { + fetchSpy + // server connectivity + .mockResolvedValueOnce({ + status: 200, + json: () => Promise.resolve({}), + } as Response) + // test 1 + .mockResolvedValueOnce({ + status: 500, + json: () => Promise.resolve({}), + } as Response) + // test 2 and beyond + .mockResolvedValue({ + status: 200, + json: () => Promise.resolve({}), + } as Response); + + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('fail'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 0, + failCount: 1, + errorCount: 0, + }); + }); + + it('reports error for thrown error on fetch', async () => { + fetchSpy.mockReset(); + fetchSpy + // server connectivity + .mockResolvedValueOnce({ + status: 200, + json: () => Promise.resolve({}), + } as Response) + // test 1 + .mockRejectedValueOnce(new Error('error')) + // test 2 and beyond + .mockResolvedValue({ + status: 200, + json: () => Promise.resolve({}), + } as Response); + + const config = { + ...baseConfig, + Tests: { + ResultsPath: './results', + SkipList: [], + OnlyList: [], + }, + }; + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('error'); + expect(test2?.testStatus).toBe('pass'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 0, + failCount: 0, + errorCount: 1, + }); + }); + + it('SKIP_LIST env var overrides config Tests.SkipList', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + // Config SkipList skips Test1, but env will skip Test2 instead + SkipList: [ + { testsName: 'Suite1', groupName: 'Group1', testName: 'Test1', reason: 'Cfg' }, + ], + OnlyList: [], + }, + }; + + process.env.SKIP_LIST = JSON.stringify([ + { testsName: 'Suite1', groupName: 'Group1', testName: 'Test2', reason: 'Env' }, + ]); + + const results = await runner.runTests(config, { useAxios: false }); + const test1 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test1' + ); + const test2 = results.results.find( + r => r.testsName === 'Suite1' && r.groupName === 'Group1' && r.testName === 'Test2' + ); + expect(test1?.testStatus).toBe('pass'); + expect(test2?.testStatus).toBe('skip'); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); + + it('ONLY_LIST env var overrides config Tests.OnlyList', async () => { + const config = { + ...baseConfig, + Tests: { + ...baseConfig.Tests, + // Config says run Test1 + OnlyList: [{ testsName: 'Suite1', groupName: 'Group1', testName: 'Test1' }], + }, + }; + + // Env says run Test2 instead + process.env.ONLY_LIST = JSON.stringify([ + { testsName: 'Suite1', groupName: 'Group1', testName: 'Test2' }, + ]); + + const results = await runner.runTests(config, { useAxios: false }); + const statuses = results.results.map(r => ({ + key: `${r.testsName}-${r.groupName}-${r.testName}`, + status: r.testStatus, + })); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test2', status: 'pass' }); + expect(statuses).toContainEqual({ key: 'Suite1-Group1-Test1', status: 'skip' }); + expect(results.toJSON().testResultsSummary).toEqual({ + passCount: 1, + skipCount: 1, + failCount: 0, + errorCount: 0, + }); + }); +}); diff --git a/test/server-command.test.ts b/test/server-command.test.ts index 7a12fda..3c6d121 100644 --- a/test/server-command.test.ts +++ b/test/server-command.test.ts @@ -6,23 +6,22 @@ import { ServerCommand } from '../src/commands/server-command.js'; // Test data and mock helpers const createMockConfig = (overrides = {}) => ({ - FhirServer: { - BaseUrl: 'http://localhost:8080/fhir/', - CqlOperation: '$cql', - }, - Build: { - CqlFileVersion: '1.0.000', - CqlOutputPath: './cql', - SERVER_OFFSET_ISO: '+00:00' - }, - Debug: { - QuickTest: false, - }, - Tests: { - ResultsPath: './results', - SkipList: [], - }, - ...overrides, + FhirServer: { + BaseUrl: 'http://localhost:8080/fhir/', + CqlOperation: '$cql', + }, + Build: { + CqlFileVersion: '1.0.000', + CqlOutputPath: './cql', + }, + Debug: { + QuickTest: false, + }, + Tests: { + ResultsPath: './results', + SkipList: [], + }, + ...overrides, }); const createMockResults = () => ({ @@ -46,6 +45,7 @@ vi.mock('../src/conf/config-loader', () => ({ QuickTest: configData?.Debug?.QuickTest || false, }; this.skipListMap = vi.fn().mockReturnValue(new Map()); + this.onlyListSet = vi.fn().mockReturnValue(new Set()); return this; }), })); diff --git a/test/value-Map.test.ts b/test/value-Map.test.ts new file mode 100644 index 0000000..f7e0bb1 --- /dev/null +++ b/test/value-Map.test.ts @@ -0,0 +1,44 @@ +// Author: Preston Lee + +import { expect, test } from 'vitest'; + +import { ValueMap } from '../src/extractors/value-map.js'; + +test('singleton collapse unwraps without hint', () => { + const m = new ValueMap(); + m.add('return', 'a'); + expect(m.toResult()).toBe('a'); +}); + +test('singleton collapse keeps array with hint (issue #82)', () => { + const m = new ValueMap(new Set(['return'])); + m.add('return', 'a'); + expect(m.toResult()).toEqual(['a']); +}); + +test('multiple values stay array without special case', () => { + const m = new ValueMap(); + m.add('return', 'a'); + m.add('return', 'b'); + expect(m.toResult()).toEqual(['a', 'b']); +}); + +test('singletonListKeysFromExpected: non-array yields empty key set', () => { + expect([...ValueMap.singletonListKeysFromExpected('a')]).toEqual([]); + expect([...ValueMap.singletonListKeysFromExpected({ a: 1 })]).toEqual([]); +}); + +test('singletonListKeysFromExpected: empty array adds no keys (issue #90)', () => { + expect([...ValueMap.singletonListKeysFromExpected([])]).toEqual([]); +}); + +test('singletonListKeysFromExpected: array adds return', () => { + expect([...ValueMap.singletonListKeysFromExpected(['a'])]).toEqual(['return']); + expect([...ValueMap.singletonListKeysFromExpected([1, 2])]).toEqual(['return']); +}); + +test('singletonListKeysFromExpected: array of arrays adds return and element', () => { + const keys = ValueMap.singletonListKeysFromExpected([[1, 2]]); + expect(keys.has('return')).toBe(true); + expect(keys.has('element')).toBe(true); +}); diff --git a/tsconfig.json b/tsconfig.json index 860510e..5d5638f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,8 +16,8 @@ "allowSyntheticDefaultImports": true, "experimentalDecorators": true, "emitDecoratorMetadata": true, - "downlevelIteration": true + "rootDir": "./src" }, - "include": ["src/**/*", "test/**/*"], + "include": ["src/**/*"], "exclude": ["node_modules", "dist", "cvl", "cql-tests", "results", "cql"] } diff --git a/vitest.config.ts b/vitest.config.ts index ca57067..adf3706 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ globals: true, environment: 'node', exclude: ['**/node_modules/**', '**/dist/**', '**/.{idea,git,cache,output,temp}/**'], + silent: true, }, resolve: { alias: { From 7b2c1a7e46b15947d7821f722bc15e4bb26aabf0 Mon Sep 17 00:00:00 2001 From: Bryant Austin Date: Wed, 17 Jun 2026 12:05:58 -0600 Subject: [PATCH 8/8] fixed time zone offset code; run with cql-tests cqltimezoneoffsettest.xml from PR #88 --- src/commands/run-tests-command.ts | 4 +++- src/services/test-runner.ts | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/commands/run-tests-command.ts b/src/commands/run-tests-command.ts index 753450f..4d0ac5f 100644 --- a/src/commands/run-tests-command.ts +++ b/src/commands/run-tests-command.ts @@ -54,7 +54,9 @@ export class RunCommand { cqlTranslator: config.Build?.cqlTranslator, cqlTranslatorVersion: config.Build?.cqlTranslatorVersion, cqlEngine: config.Build?.cqlEngine, - cqlEngineVersion: config.Build?.cqlEngineVersion + cqlEngineVersion: config.Build?.cqlEngineVersion, + SERVER_OFFSET_ISO: config.Build?.SERVER_OFFSET_ISO, + TimeZoneOffsetPolicy: config.Build?.TimeZoneOffsetPolicy, }, Tests: { ResultsPath: config.Tests.ResultsPath, diff --git a/src/services/test-runner.ts b/src/services/test-runner.ts index 7d2950c..d8f2874 100644 --- a/src/services/test-runner.ts +++ b/src/services/test-runner.ts @@ -36,7 +36,8 @@ export class TestRunner { build.cqlTranslator ?? '', build.cqlTranslatorVersion ?? '', build.cqlEngine ?? '', - build.cqlEngineVersion ?? '' + build.cqlEngineVersion ?? '', + build.SERVER_OFFSET_ISO ?? '+00:00' ); cqlEngine.cqlVersion = '1.5'; //default value const cqlVersion = config.Build?.CqlVersion; @@ -143,7 +144,7 @@ export class TestRunner { ); return result; } else if (onlySet.size > 0 && !onlySet.has(key)) { - result.SkipMessage = 'Skipped by OnlyList filter'; + result.skipMessage = 'Skipped by OnlyList filter'; result.testStatus = 'skip'; return result; } else if (skipMap.has(key)) { @@ -166,7 +167,7 @@ export class TestRunner { activeTimeZonePolicy ); if (timezonePolicySkipReason) { - result.SkipMessage = timezonePolicySkipReason; + result.skipMessage = timezonePolicySkipReason; result.testStatus = 'skip'; return result; } @@ -182,6 +183,15 @@ export class TestRunner { this.applyServerOffsetToParameters(data, cqlEngine); + // Also resolve the placeholder in result.expression so the results + // log the actual expression sent to the server, not the raw template. + if (cqlEngine.SERVER_OFFSET_ISO?.trim()) { + result.expression = this.replaceServerOffsetPlaceholder( + result.expression, + cqlEngine.SERVER_OFFSET_ISO + ); + } + let response: any; if (useAxios) { // Use axios for backward compatibility