Skip to main content
Glama
token-column.test.ts27.1 kB
// SPDX-FileCopyrightText: Copyright Orangebot, Inc. and Medplum contributors // SPDX-License-Identifier: Apache-2.0 import type { Filter } from '@medplum/core'; import { Operator, getSearchParameter, isUUID } from '@medplum/core'; import type { ResearchStudy } from '@medplum/fhirtypes'; import type { TokenColumnSearchParameterImplementation } from './searchparameter'; import { getSearchParameterImplementation } from './searchparameter'; import { Column, Condition, Disjunction, Negation, SqlBuilder, TypedCondition } from './sql'; import { loadStructureDefinitions } from './structure'; import { buildTokenColumns, buildTokenColumnsSearchFilter, getPaddingElement, hashTokenColumnValue, } from './token-column'; const DELIM = '\x01'; describe('buildTokenColumns', () => { beforeAll(() => { loadStructureDefinitions(); }); test('shared columns', () => { const focus = getSearchParameter('ResearchStudy', 'focus'); if (!focus) { throw new Error('Missing search parameter'); } const focusImpl = getSearchParameterImplementation( 'ResearchStudy', focus ) as TokenColumnSearchParameterImplementation; expect(focusImpl.searchStrategy).toStrictEqual('token-column'); expect(focusImpl.hasDedicatedColumns).toStrictEqual(false); const location = getSearchParameter('ResearchStudy', 'location'); if (!location) { throw new Error('Missing search parameter'); } const locationImpl = getSearchParameterImplementation( 'ResearchStudy', location ) as TokenColumnSearchParameterImplementation; expect(locationImpl.searchStrategy).toStrictEqual('token-column'); expect(locationImpl.hasDedicatedColumns).toStrictEqual(false); const system = 'http://example.com'; const rs: ResearchStudy = { resourceType: 'ResearchStudy', status: 'active', focus: [ { coding: [ { system, code: '123', display: 'ONE TWO THREE', }, ], }, ], location: [ { coding: [ { system, code: '123', display: 'ONE TWO THREE', }, ], }, { coding: [ { system, code: '456', display: 'FOUR FIVE SIX', }, ], }, ], }; const columns: Record<string, any> = {}; buildTokenColumns(focus, focusImpl, columns, rs); expect(columns).toStrictEqual({ __focusSort: '123', __sharedTokens: [ 'focus', 'focus' + DELIM + DELIM + 'ONE TWO THREE', // since Medplum incorrectly supports exact search for :text entries 'focus' + DELIM + system, 'focus' + DELIM + system + DELIM + '123', 'focus' + DELIM + DELIM + '123', ].map(hashTokenColumnValue), __sharedTokensText: ['focus' + DELIM + 'ONE TWO THREE'], }); buildTokenColumns(location, locationImpl, columns, rs); expect(columns).toStrictEqual({ __focusSort: '123', __locationSort: '123', __sharedTokens: [ 'focus', 'focus' + DELIM + DELIM + 'ONE TWO THREE', 'focus' + DELIM + system, 'focus' + DELIM + system + DELIM + '123', 'focus' + DELIM + DELIM + '123', 'location', 'location' + DELIM + DELIM + 'ONE TWO THREE', 'location' + DELIM + system, // system is used twice in location, but should only appear once 'location' + DELIM + system + DELIM + '123', 'location' + DELIM + DELIM + '123', 'location' + DELIM + DELIM + 'FOUR FIVE SIX', 'location' + DELIM + system + DELIM + '456', 'location' + DELIM + DELIM + '456', ].map(hashTokenColumnValue), __sharedTokensText: [ 'focus' + DELIM + 'ONE TWO THREE', 'location' + DELIM + 'ONE TWO THREE', 'location' + DELIM + 'FOUR FIVE SIX', ], }); }); }); describe('buildTokenColumnsSearchFilter', () => { beforeAll(() => { loadStructureDefinitions(); }); describe('EQUALS operator', () => { test('search with system and code', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EQUALS, value: 'http://example.com|12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Condition); const cond = expr as Condition; expect(cond.column).toBeInstanceOf(Column); expect((cond.column as Column).actualColumnName).toBe('__identifier'); expect(cond.operator).toBe('ARRAY_OVERLAPS'); expect(cond.parameterType).toBe('UUID[]'); // Verify the parameter is an array with the hashed value const expectedHash = hashTokenColumnValue('http://example.com' + DELIM + '12345'); expect(cond.parameter).toEqual([expectedHash]); }); test('search with code only (no system)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EQUALS, value: '12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Condition); const cond = expr as Condition; expect(cond.operator).toBe('ARRAY_OVERLAPS'); // Searching for just the value - parameter is now an array const expectedHash = hashTokenColumnValue(DELIM + '12345'); expect(cond.parameter).toEqual([expectedHash]); }); test('search with system only (trailing pipe)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EQUALS, value: 'http://example.com|', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Condition); const cond = expr as Condition; expect(cond.operator).toBe('ARRAY_OVERLAPS'); // Searching for just the system - parameter is now an array const expectedHash = hashTokenColumnValue('http://example.com'); expect(cond.parameter).toEqual([expectedHash]); }); test('search with empty system (leading pipe)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EQUALS, value: '|12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Condition); const cond = expr as Condition; expect(cond.operator).toBe('ARRAY_OVERLAPS'); // Searching for NULL_SYSTEM with value - parameter is now an array const NULL_SYSTEM = '\x02'; const expectedHash = hashTokenColumnValue(NULL_SYSTEM + DELIM + '12345'); expect(cond.parameter).toEqual([expectedHash]); }); test('search with comma-separated values (OR)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EQUALS, value: '12345,67890', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Condition); const cond = expr as Condition; expect(cond.operator).toBe('ARRAY_OVERLAPS'); // Both hashes are now in a single parameter array (more efficient SQL) const expectedHash1 = hashTokenColumnValue(DELIM + '12345'); const expectedHash2 = hashTokenColumnValue(DELIM + '67890'); expect(cond.parameter).toEqual([expectedHash1, expectedHash2]); }); test('search is case-sensitive for identifiers', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EQUALS, value: 'ABC123', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); const cond = expr as Condition; // Should NOT be lowercased (identifiers are case-sensitive) - parameter is now an array const expectedHash = hashTokenColumnValue(DELIM + 'ABC123'); expect(cond.parameter).toEqual([expectedHash]); }); test('non-dedicated columns include code prefix', () => { const param = getSearchParameter('ResearchStudy', 'focus'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'focus', operator: Operator.EQUALS, value: 'http://example.com|test', }; const expr = buildTokenColumnsSearchFilter('ResearchStudy', 'ResearchStudy', param, filter); const cond = expr as Condition; expect((cond.column as Column).actualColumnName).toBe('__sharedTokens'); // Should include the code prefix for non-dedicated columns - parameter is now an array const expectedHash = hashTokenColumnValue('focus' + DELIM + 'http://example.com' + DELIM + 'test'); expect(cond.parameter).toEqual([expectedHash]); }); }); describe('NOT and NOT_EQUALS operators', () => { test('NOT operator with single value', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.NOT, value: '12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Negation); const negation = expr as Negation; expect(negation.expression).toBeInstanceOf(Condition); }); test('NOT_EQUALS operator with single value', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.NOT_EQUALS, value: '12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Negation); const negation = expr as Negation; expect(negation.expression).toBeInstanceOf(Condition); }); test('NOT operator with comma-separated values', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.NOT, value: '12345,67890', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Negation); const negation = expr as Negation; expect(negation.expression).toBeInstanceOf(Condition); // The inner disjunction now has 1 expression with both values in the parameter array const cond = negation.expression as Condition; expect(cond.operator).toBe('ARRAY_OVERLAPS'); const expectedHash1 = hashTokenColumnValue(DELIM + '12345'); const expectedHash2 = hashTokenColumnValue(DELIM + '67890'); expect(cond.parameter).toEqual([expectedHash1, expectedHash2]); }); }); describe('TEXT and CONTAINS operators', () => { test('TEXT operator searches text column', () => { const param = getSearchParameter('Patient', 'telecom'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'telecom', operator: Operator.TEXT, value: '555', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Disjunction); const disjunction = expr as Disjunction; expect(disjunction.expressions).toHaveLength(1); const cond = disjunction.expressions[0] as TypedCondition<'TOKEN_ARRAY_IREGEX'>; expect(cond.operator).toBe('TOKEN_ARRAY_IREGEX'); expect((cond.column as Column).actualColumnName).toBe('__telecomText'); expect(cond.parameterType).toBe('TEXT[]'); // Should search for the value as a regex substring const ARRAY_DELIM = '\x03'; expect(cond.parameter).toContain('555'); expect(cond.parameter).toContain(ARRAY_DELIM); }); test('CONTAINS operator searches text column', () => { const param = getSearchParameter('Patient', 'telecom'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'telecom', operator: Operator.CONTAINS, value: '555', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Disjunction); const disjunction = expr as Disjunction; const cond = disjunction.expressions[0] as TypedCondition<'TOKEN_ARRAY_IREGEX'>; expect(cond.operator).toBe('TOKEN_ARRAY_IREGEX'); }); test('TEXT operator with regex special characters escaped', () => { const param = getSearchParameter('Patient', 'telecom'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'telecom', operator: Operator.TEXT, value: 'test.value*', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); const disjunction = expr as Disjunction; const cond = disjunction.expressions[0] as TypedCondition<'TOKEN_ARRAY_IREGEX'>; // Special regex characters should be escaped expect(cond.parameter).toContain('test\\.value\\*'); }); test('TEXT operator with non-dedicated columns includes code prefix', () => { const param = getSearchParameter('ResearchStudy', 'focus'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'focus', operator: Operator.TEXT, value: 'test', }; const expr = buildTokenColumnsSearchFilter('ResearchStudy', 'ResearchStudy', param, filter); const disjunction = expr as Disjunction; const cond = disjunction.expressions[0] as TypedCondition<'TOKEN_ARRAY_IREGEX'>; expect((cond.column as Column).actualColumnName).toBe('__sharedTokensText'); // Should include the code prefix in the regex pattern const ARRAY_DELIM = '\x03'; expect(cond.parameter).toContain(ARRAY_DELIM + 'focus' + DELIM); }); test('TEXT operator with comma-separated values (OR)', () => { const param = getSearchParameter('Patient', 'telecom'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'telecom', operator: Operator.TEXT, value: '555,777', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(Disjunction); const disjunction = expr as Disjunction; expect(disjunction.expressions).toHaveLength(2); const cond1 = disjunction.expressions[0] as TypedCondition<'TOKEN_ARRAY_IREGEX'>; const cond2 = disjunction.expressions[1] as TypedCondition<'TOKEN_ARRAY_IREGEX'>; expect(cond1.operator).toBe('TOKEN_ARRAY_IREGEX'); expect(cond2.operator).toBe('TOKEN_ARRAY_IREGEX'); expect(cond1.parameter).toContain('555'); expect(cond2.parameter).toContain('777'); }); }); describe('MISSING and PRESENT operators', () => { test('MISSING operator with true value (dedicated columns)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.MISSING, value: 'true', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(TypedCondition); const cond = expr as TypedCondition<'ARRAY_EMPTY'>; expect(cond.operator).toBe('ARRAY_EMPTY'); expect((cond.column as Column).actualColumnName).toBe('__identifier'); expect(cond.parameterType).toBe('UUID[]'); }); test('MISSING operator with false value (dedicated columns)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.MISSING, value: 'false', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(TypedCondition); const cond = expr as TypedCondition<'ARRAY_NOT_EMPTY'>; expect(cond.operator).toBe('ARRAY_NOT_EMPTY'); expect((cond.column as Column).actualColumnName).toBe('__identifier'); }); test('PRESENT operator with true value (dedicated columns)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.PRESENT, value: 'true', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(TypedCondition); const cond = expr as TypedCondition<'ARRAY_NOT_EMPTY'>; expect(cond.operator).toBe('ARRAY_NOT_EMPTY'); }); test('PRESENT operator with false value (dedicated columns)', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.PRESENT, value: 'false', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); expect(expr).toBeInstanceOf(TypedCondition); const cond = expr as TypedCondition<'ARRAY_EMPTY'>; expect(cond.operator).toBe('ARRAY_EMPTY'); }); test('MISSING operator with non-dedicated columns', () => { const param = getSearchParameter('ResearchStudy', 'focus'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'focus', operator: Operator.MISSING, value: 'true', }; const expr = buildTokenColumnsSearchFilter('ResearchStudy', 'ResearchStudy', param, filter); expect(expr).toBeInstanceOf(Negation); const negation = expr as Negation; expect(negation.expression).toBeInstanceOf(TypedCondition); const cond = negation.expression as TypedCondition<'ARRAY_OVERLAPS'>; expect(cond.operator).toBe('ARRAY_OVERLAPS'); expect((cond.column as Column).actualColumnName).toBe('__sharedTokens'); // Should search for the code as a hashed value const expectedHash = hashTokenColumnValue('focus'); expect(cond.parameter).toBe(expectedHash); }); test('PRESENT operator with non-dedicated columns', () => { const param = getSearchParameter('ResearchStudy', 'focus'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'focus', operator: Operator.PRESENT, value: 'true', }; const expr = buildTokenColumnsSearchFilter('ResearchStudy', 'ResearchStudy', param, filter); expect(expr).toBeInstanceOf(TypedCondition); const cond = expr as TypedCondition<'ARRAY_OVERLAPS'>; expect(cond.operator).toBe('ARRAY_OVERLAPS'); expect((cond.column as Column).actualColumnName).toBe('__sharedTokens'); const expectedHash = hashTokenColumnValue('focus'); expect(cond.parameter).toBe(expectedHash); }); }); describe('EXACT operator', () => { test('EXACT operator behaves like EQUALS', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EXACT, value: '12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter) as Condition; expect(expr).toBeInstanceOf(Condition); expect(expr.operator).toBe('ARRAY_OVERLAPS'); }); }); describe('unsupported operators', () => { test('IN operator throws error', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.IN, value: 'test', }; expect(() => buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter)).toThrow(); }); test('STARTS_WITH operator throws error', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.STARTS_WITH, value: 'test', }; expect(() => buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter)).toThrow(); }); test('GREATER_THAN operator throws error', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.GREATER_THAN, value: 'test', }; expect(() => buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter)).toThrow(); }); }); describe('SQL generation', () => { test('generates valid SQL for simple equals search', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.EQUALS, value: '12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); const builder = new SqlBuilder(); expr.buildSql(builder); const sql = builder.toString(); expect(sql).toContain('"Patient"."__identifier"'); expect(sql).toContain('@>'); expect(sql).toContain('ARRAY['); }); test('generates valid SQL for NOT search', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.NOT, value: '12345', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); const builder = new SqlBuilder(); expr.buildSql(builder); const sql = builder.toString(); expect(sql).toContain('NOT ('); expect(sql).toContain('"Patient"."__identifier"'); }); test('generates valid SQL for TEXT search', () => { const param = getSearchParameter('Patient', 'telecom'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'telecom', operator: Operator.TEXT, value: '555', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); const builder = new SqlBuilder(); expr.buildSql(builder); const sql = builder.toString(); expect(sql).toContain('"Patient"."__telecomText"'); expect(sql).toContain('~*'); }); test('generates valid SQL for MISSING search with dedicated columns', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.MISSING, value: 'true', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); const builder = new SqlBuilder(); expr.buildSql(builder); const sql = builder.toString(); expect(sql).toContain('"Patient"."__identifier"'); expect(sql).toContain('ARRAY[]'); }); test('generates valid SQL for PRESENT search with dedicated columns', () => { const param = getSearchParameter('Patient', 'identifier'); if (!param) { throw new Error('Missing search parameter'); } const filter: Filter = { code: 'identifier', operator: Operator.PRESENT, value: 'true', }; const expr = buildTokenColumnsSearchFilter('Patient', 'Patient', param, filter); const builder = new SqlBuilder(); expr.buildSql(builder); const sql = builder.toString(); expect(sql).toContain('"Patient"."__identifier"'); expect(sql).toContain('array_length'); expect(sql).toContain('> 0'); }); }); }); describe('getPaddingElement', () => { test('Math.random is 0.99999999', () => { const rng = jest.fn().mockReturnValue(0.99999999); expect(getPaddingElement({ m: 1, lambda: 150, statisticsTarget: 1 }, rng)).toBeUndefined(); // once to decide (not to) return a padding element expect(rng).toHaveBeenCalledTimes(1); }); test('Math.random is 0', () => { let callCount = 0; const rng = jest.fn().mockImplementation(() => { // first call returns 0 to guarantee padding is returned if (callCount++ === 0) { return 0; } // second call returns 0.99999999 to guarantee the largest padding element is chosen return 0.99999999; }); const paddingElement = getPaddingElement({ m: 20, lambda: 150, statisticsTarget: 1 }, rng) as string; expect(paddingElement).toStrictEqual('00000000-0000-0000-0000-000000000019'); expect(isUUID(paddingElement)).toStrictEqual(true); // once to decide whether to return a padding element, once to decide which padding element expect(rng).toHaveBeenCalledTimes(2); }); });

Latest Blog Posts

MCP directory API

We provide all the information about MCP servers via our MCP API.

curl -X GET 'https://glama.ai/api/mcp/v1/servers/medplum/medplum'

If you have feedback or need assistance with the MCP directory API, please join our Discord server