From 2e87259b956f24e96c98da9278b01c0ee89ed84d Mon Sep 17 00:00:00 2001 From: Azhar Saleem Date: Wed, 10 Jun 2026 16:23:20 -0400 Subject: [PATCH 1/3] Fix TS terminology resource handling --- .../fhir/r5/context/BaseWorkerContext.java | 296 +++++++++++++++++- .../BaseWorkerContextTsSupportTest.java | 56 ++++ .../java/ch/ahdis/matchbox/CliContext.java | 76 +++++ .../matchbox/util/MatchboxEngineSupport.java | 6 +- .../validation/ValidationProvider.java | 70 +++++ .../ConsoleTerminologyClientLogger.java | 120 +++++++ .../matchbox/CliContextTxLoggingTest.java | 41 +++ .../ValidationProviderTsFilteringTest.java | 91 ++++++ .../ConsoleTerminologyClientLoggerTest.java | 48 +++ 9 files changed, 789 insertions(+), 15 deletions(-) create mode 100644 matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java create mode 100644 matchbox-server/src/main/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLogger.java create mode 100644 matchbox-server/src/test/java/ch/ahdis/matchbox/CliContextTxLoggingTest.java create mode 100644 matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java create mode 100644 matchbox-server/src/test/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLoggerTest.java diff --git a/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java b/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java index 8d21903c403..dd4bab70275 100644 --- a/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java +++ b/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java @@ -68,6 +68,8 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS import org.hl7.fhir.r5.extensions.ExtensionUtilities; import org.hl7.fhir.r5.model.ActorDefinition; import org.hl7.fhir.r5.model.BooleanType; +import org.hl7.fhir.r5.model.Bundle; +import org.hl7.fhir.r5.model.Bundle.BundleEntryComponent; import org.hl7.fhir.r5.model.CanonicalResource; import org.hl7.fhir.r5.model.CanonicalType; import org.hl7.fhir.r5.model.CapabilityStatement; @@ -156,6 +158,7 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS @MarkedToMoveToAdjunctPackage public abstract class BaseWorkerContext extends I18nBase implements IWorkerContext, IWorkerContextManager, IOIDServices { private static boolean allowedToIterateTerminologyResources; + private static final String INFOWAY_TS_HOST = "terminologystandardsservice.ca"; public interface IByteProvider { @@ -1679,10 +1682,12 @@ protected ValueSetExpander constructValueSetExpanderSimple(ValidationOptions opt } protected ValueSetValidator constructValueSetCheckerSimple(ValidationOptions options, ValueSet vs, ValidationContextCarrier ctxt) { + cacheRequiredSupplements(vs); return new ValueSetValidator(this, new TerminologyOperationContext(this, options, "validation"), options, vs, ctxt, getExpansionParameters(), terminologyClientManager, registry); } protected ValueSetValidator constructValueSetCheckerSimple( ValidationOptions options, ValueSet vs) { + cacheRequiredSupplements(vs); return new ValueSetValidator(this, new TerminologyOperationContext(this, options, "validation"), options, vs, getExpansionParameters(), terminologyClientManager, registry); } @@ -2007,6 +2012,7 @@ protected void addServerValidationParameters(ValueSetProcessBase.TerminologyOper private boolean addDependentResources(ValueSetProcessBase.TerminologyOperationDetails opCtxt, TerminologyClientContext tc, Parameters pin, ValueSet vs) { boolean cache = false; + cache = addRequiredSupplements(opCtxt, tc, pin, vs) || cache; for (ConceptSetComponent inc : vs.getCompose().getInclude()) { cache = addDependentResources(opCtxt, tc, pin, inc, vs) || cache; } @@ -2020,21 +2026,20 @@ private boolean addDependentResources(ValueSetProcessBase.TerminologyOperationDe boolean cache = false; for (CanonicalType c : inc.getValueSet()) { ValueSet vs = fetchResource(ValueSet.class, c.getValue(), null, src); - if (vs != null && !hasCanonicalResource(pin, "tx-resource", vs.getVUrl())) { - cache = checkAddToParams(tc, pin, vs) || cache; - addDependentResources(opCtxt, tc, pin, vs); - for (Extension ext : vs.getExtensionsByUrl(ExtensionDefinitions.EXT_VS_CS_SUPPL_NEEDED)) { - if (ext.hasValueCanonicalType()) { - String url = ext.getValueCanonicalType().asStringValue(); - CodeSystem supp = fetchResource(CodeSystem.class, url); - if (supp != null) { - if (opCtxt != null) { - opCtxt.seeSupplement(supp); - } - cache = checkAddToParams(tc, pin, supp) || cache; - } + if (vs != null) { + boolean processedDependencies = false; + if (!hasCanonicalResource(pin, "tx-resource", vs.getVUrl())) { + if (shouldOmitTxResourceForTsValueSet(tc, vs)) { + logger.logDebugMessage(LogCategory.CONTEXT, "tx-resource skipped; resolving ValueSet on TS: "+vs.getVUrl()); + } else { + cache = checkAddToParams(tc, pin, vs) || cache; + cache = addDependentResources(opCtxt, tc, pin, vs) || cache; + processedDependencies = true; } } + if (!processedDependencies) { + cache = addRequiredSupplements(opCtxt, tc, pin, vs) || cache; + } } } String sys = inc.getSystem(); @@ -2084,6 +2089,122 @@ public boolean addDependentCodeSystem(ValueSetProcessBase.TerminologyOperationDe return cache; } + private boolean addRequiredSupplements(ValueSetProcessBase.TerminologyOperationDetails opCtxt, TerminologyClientContext tc, Parameters pin, ValueSet vs) { + boolean cache = false; + if (vs == null) { + return false; + } + for (Extension ext : vs.getExtensionsByUrl(ExtensionDefinitions.EXT_VS_CS_SUPPL_NEEDED)) { + String canonical = getSupplementCanonical(ext); + if (Utilities.noString(canonical)) { + continue; + } + addUseSupplementParameter(pin, canonical); + if (opCtxt != null) { + opCtxt.seeSupplement(toSupplementIdentity(canonical)); + } + CodeSystem supp = resolveRequiredSupplement(ext); + if (supp != null && !hasCanonicalResource(pin, "tx-resource", supp.getVUrl())) { + if (opCtxt != null) { + opCtxt.seeSupplement(supp); + } + cache = checkAddToParams(tc, pin, supp) || cache; + } + } + return cache; + } + + private CodeSystem toSupplementIdentity(String canonical) { + CodeSystem supp = new CodeSystem(); + if (Utilities.noString(canonical)) { + return supp; + } + int versionSeparator = canonical.lastIndexOf("|"); + if (versionSeparator > -1) { + supp.setUrl(canonical.substring(0, versionSeparator)); + supp.setVersion(canonical.substring(versionSeparator + 1)); + } else { + supp.setUrl(canonical); + } + return supp; + } + + private void addUseSupplementParameter(Parameters pin, String canonical) { + if (pin == null || Utilities.noString(canonical) || hasPrimitiveParameter(pin, "useSupplement", canonical)) { + return; + } + pin.addParameter().setName("useSupplement").setValue(new CanonicalType(canonical)); + } + + private boolean hasPrimitiveParameter(Parameters pin, String name, String value) { + if (pin == null) { + return false; + } + for (ParametersParameterComponent p : pin.getParameter()) { + if (name.equals(p.getName()) && p.hasValuePrimitive() && value.equals(p.getValue().primitiveValue())) { + return true; + } + } + return false; + } + + private CodeSystem resolveRequiredSupplement(Extension ext) { + String canonical = getSupplementCanonical(ext); + if (Utilities.noString(canonical)) { + return null; + } + String url = canonical; + String version = null; + int versionSeparator = canonical.lastIndexOf("|"); + if (versionSeparator > -1) { + url = canonical.substring(0, versionSeparator); + version = canonical.substring(versionSeparator + 1); + } + return Utilities.noString(version) ? findTxResource(CodeSystem.class, url) : findTxResource(CodeSystem.class, url, version); + } + + private String getSupplementCanonical(Extension ext) { + if (ext == null || !ext.hasValueCanonicalType()) { + return null; + } + return ext.getValueCanonicalType().asStringValue(); + } + + private boolean shouldOmitTxResourceForTsValueSet(TerminologyClientContext tc, ValueSet vs) { + return shouldOmitTxResourceForTsValueSet(tc == null ? null : tc.getAddress(), vs, valueSetExistsOnceOnTerminologyServer(tc, vs)); + } + + static boolean shouldOmitTxResourceForTsValueSet(String txAddress, ValueSet vs, boolean exactlyOneMatchOnTs) { + return isInfowayTsServer(txAddress) + && vs != null + && exactlyOneMatchOnTs; + } + + private boolean valueSetExistsOnceOnTerminologyServer(TerminologyClientContext tc, ValueSet vs) { + if (tc == null || tc.getClient() == null || vs == null || Utilities.noString(vs.getUrl())) { + return false; + } + try { + Bundle bnd = tc.getClient().search("ValueSet", valueSetSearchCriteria(vs)); + return bnd != null && bnd.getEntry().size() == 1; + } catch (Exception e) { + logger.logDebugMessage(LogCategory.TX, "Error checking ValueSet on terminology server: "+vs.getVUrl()+" - "+e.getMessage()); + return false; + } + } + + private String valueSetSearchCriteria(ValueSet vs) { + return "?_format=json&url="+Utilities.URLEncode(vs.getUrl()); + } + + static boolean isInfowayTsServer(String txAddress) { + if (txAddress == null) { + return false; + } + String address = txAddress.toLowerCase(Locale.ROOT); + return address.contains(INFOWAY_TS_HOST); + } + private String getFixedVersion(String sys, Parameters pin) { for (ParametersParameterComponent p : pin.getParameter()) { if (Utilities.existsInList(p.getName(), "system-version", "force-system-version", "default-system-version")) { @@ -2296,6 +2417,67 @@ public void setLogger(@Nonnull org.hl7.fhir.r5.context.ILoggingService logger) { getTxClientManager().setLogger(logger); } + public ToolingClientLogger getTxClientLogger() { + return txLog; + } + + public void setTxClientLogger(ToolingClientLogger txLog) { + this.txLog = txLog; + getTxClientManager().setLogger(txLog); + } + + public void addTxClientLogger(ToolingClientLogger logger) { + if (logger == null) { + return; + } + if (txLog == null) { + setTxClientLogger(logger); + } else { + setTxClientLogger(new CompositeToolingClientLogger(txLog, logger)); + } + } + + private static class CompositeToolingClientLogger implements ToolingClientLogger { + + private final ToolingClientLogger[] loggers; + + private CompositeToolingClientLogger(ToolingClientLogger... loggers) { + this.loggers = loggers; + } + + @Override + public void logRequest(String method, String url, List headers, byte[] body) { + for (ToolingClientLogger logger : loggers) { + logger.logRequest(method, url, headers, body); + } + } + + @Override + public void logResponse(String outcome, List headers, byte[] body, long start) { + for (ToolingClientLogger logger : loggers) { + logger.logResponse(outcome, headers, body, start); + } + } + + @Override + public String getLastId() { + for (int i = loggers.length - 1; i >= 0; i--) { + String lastId = loggers[i].getLastId(); + if (lastId != null) { + return lastId; + } + } + return null; + } + + @Override + public void clearLastId() { + for (ToolingClientLogger logger : loggers) { + logger.clearLastId(); + } + } + } + /** * Returns a copy of the expansion parameters used by this context. Note that because the return value is a copy, any * changes done to it will not be reflected in the context and any changes to the context will likewise not be @@ -3629,14 +3811,19 @@ private T doFindTxResource(Class class_, String canonica return null; } else { cacheResource(svs.getVs()); + cacheRequiredSupplements(svs.getVs()); return (T) svs.getVs(); } } else if (class_ == CodeSystem.class) { SourcedCodeSystem scs = null; if (txCache.hasCodeSystem(canonical)) { scs = txCache.getCodeSystem(canonical); - } else { + } + if (scs == null) { scs = terminologyClientManager.findCodeSystemOnServer(canonical); + if (scs == null) { + scs = findCodeSystemDirectlyOnServer(canonical); + } txCache.cacheCodeSystem(canonical, scs); } if (scs != null) { @@ -3658,6 +3845,87 @@ private T doFindTxResource(Class class_, String canonica } } + private void cacheRequiredSupplements(ValueSet vs) { + cacheRequiredSupplements(vs, new HashSet<>()); + } + + private void cacheRequiredSupplements(ValueSet vs, Set visited) { + if (vs == null) { + return; + } + String key = vs.getVUrl(); + if (!Utilities.noString(key) && !visited.add(key)) { + return; + } + for (Extension ext : vs.getExtensionsByUrl(ExtensionDefinitions.EXT_VS_CS_SUPPL_NEEDED)) { + String canonical = getSupplementCanonical(ext); + if (!Utilities.noString(canonical) && fetchResource(CodeSystem.class, canonical) == null) { + SourcedCodeSystem scs = findCodeSystemDirectlyOnServer(canonical); + if (scs != null && scs.getCs() != null) { + cacheResource(scs.getCs()); + txCache.cacheCodeSystem(canonical, scs); + } + } + } + if (vs.hasCompose()) { + cacheRequiredSupplements(vs, vs.getCompose().getInclude(), visited); + cacheRequiredSupplements(vs, vs.getCompose().getExclude(), visited); + } + } + + private void cacheRequiredSupplements(Resource src, List components, Set visited) { + for (ConceptSetComponent inc : components) { + for (CanonicalType c : inc.getValueSet()) { + ValueSet vs = fetchResource(ValueSet.class, c.getValue(), null, src); + cacheRequiredSupplements(vs, visited); + } + } + } + + private SourcedCodeSystem findCodeSystemDirectlyOnServer(String canonical) { + if (terminologyClientManager == null || !terminologyClientManager.hasClient() || Utilities.noString(canonical)) { + return null; + } + try { + TerminologyClientContext client = terminologyClientManager.chooseServer(canonical, false); + String url = canonical; + String version = null; + int versionSeparator = canonical.lastIndexOf("|"); + if (versionSeparator > -1) { + url = canonical.substring(0, versionSeparator); + version = canonical.substring(versionSeparator + 1); + } + String criteria = "?_format=json&url="+Utilities.URLEncode(url); + if (!Utilities.noString(version)) { + criteria = criteria + "&version="+Utilities.URLEncode(version); + } + Bundle bnd = client.getClient().search("CodeSystem", criteria); + String rid = null; + if (bnd.getEntry().size() > 1) { + List cslist = new ArrayList<>(); + for (BundleEntryComponent be : bnd.getEntry()) { + if (be.hasResource() && be.getResource() instanceof CodeSystem) { + cslist.add((CodeSystem) be.getResource()); + } + } + if (!cslist.isEmpty()) { + Collections.sort(cslist, new CodeSystemUtilities.CodeSystemSorter()); + rid = cslist.get(cslist.size()-1).getIdBase(); + } + } else if (bnd.getEntry().size() == 1 && bnd.getEntryFirstRep().hasResource() && bnd.getEntryFirstRep().getResource() instanceof CodeSystem) { + rid = bnd.getEntryFirstRep().getResource().getIdBase(); + } + if (rid == null) { + return null; + } + CodeSystem cs = (CodeSystem) client.getClient().read("CodeSystem", rid); + return new SourcedCodeSystem(client.getAddress(), cs); + } catch (Exception e) { + logger.logDebugMessage(LogCategory.TX, "Error resolving CodeSystem directly on terminology server: "+canonical+" - "+e.getMessage()); + return null; + } + } + public T findTxResource(Class class_, String canonical, String version, Resource sourceOfReference) { if (canonical == null) { return null; diff --git a/matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java b/matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java new file mode 100644 index 00000000000..b6ceadabd06 --- /dev/null +++ b/matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java @@ -0,0 +1,56 @@ +package org.hl7.fhir.r5.context; + +import org.hl7.fhir.r5.model.ValueSet; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BaseWorkerContextTsSupportTest { + + private static final String TS_URL = "https://terminologystandardsservice.ca/tx/fhir"; + + @Test + void omitsAnyValueSetFoundExactlyOnceOnTs() { + ValueSet valueSet = new ValueSet(); + valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/healthcareproviderspecialtycode"); + + assertTrue(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, true)); + + valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/another-ts-hosted-valueset"); + assertTrue(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, true)); + } + + @Test + void omitsVersionedValueSetThatExistsOnTs() { + ValueSet valueSet = new ValueSet(); + valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/versioned"); + valueSet.setVersion("1.0.0"); + + assertTrue(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, true)); + } + + @Test + void keepsValueSetAsTxResourceWhenItIsNotFoundExactlyOnceOnTs() { + ValueSet valueSet = new ValueSet(); + valueSet.setUrl("https://example.org/ValueSet/local-only"); + + assertFalse(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, false)); + } + + @Test + void keepsValueSetAsTxResourceForOtherServers() { + ValueSet valueSet = new ValueSet(); + valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/ts-hosted"); + + assertFalse(BaseWorkerContext.shouldOmitTxResourceForTsValueSet("https://tx.fhir.org/r4", valueSet, true)); + } + + @Test + void keepsValueSetAsTxResourceForTsProxy() { + ValueSet valueSet = new ValueSet(); + valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/ts-hosted"); + + assertFalse(BaseWorkerContext.shouldOmitTxResourceForTsValueSet("https://smart-proxy.apibox.ca:10500/tx/fhir", valueSet, true)); + } +} diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/CliContext.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/CliContext.java index 5537f8195ea..2137db9f8af 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/CliContext.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/CliContext.java @@ -124,6 +124,15 @@ public CliContext setIgs(String[] igs) { @JsonProperty("txLog") private String txLog = null; + @JsonProperty("txLogPath") + private String txLogPath = null; + + @JsonProperty("txLogToConsole") + private boolean txLogToConsole = true; + + @JsonProperty("txLogConsoleBodyLimit") + private String txLogConsoleBodyLimit = "4000"; + @JsonProperty("txUseEcosystem") private boolean txUseEcosystem = true; @@ -330,6 +339,7 @@ public CliContext(Environment environment) { } } } + applyTxLogEnvironmentAliases(environment); // get properties array from the environment? this.igsPreloaded = environment.getProperty("matchbox.fhir.context.igsPreloaded", String[].class); this.onlyOneEngine = environment.getProperty("matchbox.fhir.context.onlyOneEngine", Boolean.class, false); @@ -362,6 +372,24 @@ public CliContext(CliContext other) { this.suppressErrors = other.suppressErrors; this.suppressWarnInfos = other.suppressWarnInfos; this.igs = other.igs; + this.txLogPath = other.txLogPath; + this.txLogToConsole = other.txLogToConsole; + this.txLogConsoleBodyLimit = other.txLogConsoleBodyLimit; + } + + private void applyTxLogEnvironmentAliases(Environment environment) { + String txLogPath = environment.getProperty("TX_LOG_PATH", String.class); + if (txLogPath != null && !txLogPath.isBlank()) { + this.txLogPath = txLogPath; + } + String txLogToConsole = environment.getProperty("TX_LOG_TO_CONSOLE", String.class); + if (txLogToConsole != null && !txLogToConsole.isBlank()) { + this.txLogToConsole = Boolean.parseBoolean(txLogToConsole); + } + String txLogConsoleBodyLimit = environment.getProperty("TX_LOG_CONSOLE_BODY_LIMIT", String.class); + if (txLogConsoleBodyLimit != null && !txLogConsoleBodyLimit.isBlank()) { + this.txLogConsoleBodyLimit = txLogConsoleBodyLimit; + } } public String getIg() { @@ -412,6 +440,42 @@ public void setTxLog(String txLog) { this.txLog = txLog; } + public String getTxLogPath() { + return txLogPath; + } + + public void setTxLogPath(String txLogPath) { + this.txLogPath = txLogPath; + } + + public String getEffectiveTxLogPath() { + return txLogPath != null && !txLogPath.isBlank() ? txLogPath : txLog; + } + + public boolean isTxLogToConsole() { + return txLogToConsole; + } + + public void setTxLogToConsole(boolean txLogToConsole) { + this.txLogToConsole = txLogToConsole; + } + + public String getTxLogConsoleBodyLimit() { + return txLogConsoleBodyLimit; + } + + public void setTxLogConsoleBodyLimit(String txLogConsoleBodyLimit) { + this.txLogConsoleBodyLimit = txLogConsoleBodyLimit; + } + + public int getTxLogConsoleBodyLimitValue() { + try { + return Integer.parseInt(txLogConsoleBodyLimit); + } catch (NumberFormatException e) { + return 4000; + } + } + public void setTxUseEcosystem(boolean txUseEcosystem) { this.txUseEcosystem = txUseEcosystem; } @@ -779,6 +843,9 @@ public boolean equals(final Object o) { && Objects.equals(txServer, that.txServer) && txServerCache == that.txServerCache && Objects.equals(txLog, that.txLog) + && Objects.equals(txLogPath, that.txLogPath) + && txLogToConsole == that.txLogToConsole + && Objects.equals(txLogConsoleBodyLimit, that.txLogConsoleBodyLimit) && txUseEcosystem == that.txUseEcosystem && Objects.equals(lang, that.lang) && Objects.equals(snomedCT, that.snomedCT) @@ -833,6 +900,9 @@ public int hashCode() { txServer, txServerCache, txLog, + txLogPath, + txLogToConsole, + txLogConsoleBodyLimit, txUseEcosystem, lang, snomedCT, @@ -885,6 +955,9 @@ public String toString() { ", txServer='" + txServer + '\'' + ", txServerCache='" + txServerCache + '\'' + ", txLog='" + txLog + '\'' + + ", txLogPath='" + txLogPath + '\'' + + ", txLogToConsole=" + txLogToConsole + + ", txLogConsoleBodyLimit='" + txLogConsoleBodyLimit + '\'' + ", txUseEcosystem=" + txUseEcosystem + ", lang='" + lang + '\'' + ", snomedCT='" + snomedCT + '\'' + @@ -959,6 +1032,9 @@ public void addContextToExtension(final Extension ext) { addExtension(ext, "txServer", new UriType(this.txServer)); addExtension(ext, "txServerCache", new BooleanType(this.txServerCache)); addExtension(ext, "txLog", new StringType(this.txLog)); + addExtension(ext, "txLogPath", new StringType(this.txLogPath)); + addExtension(ext, "txLogToConsole", new BooleanType(this.txLogToConsole)); + addExtension(ext, "txLogConsoleBodyLimit", new StringType(this.txLogConsoleBodyLimit)); addExtension(ext, "txUseEcosystem", new BooleanType(this.txUseEcosystem)); addExtension(ext, "lang", new StringType(this.lang)); addExtension(ext, "snomedCT", new StringType(this.snomedCT)); diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java index 296b25dabaa..94571a89ae8 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java @@ -16,6 +16,7 @@ import ch.ahdis.matchbox.util.http.HttpRequestWrapper; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.hl7.fhir.r5.context.ConsoleTerminologyClientLogger; import org.apache.commons.codec.digest.DigestUtils; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -527,7 +528,10 @@ private void configureValidationEngine(final MatchboxEngine validator, TerminologyClientContext.setAllowNonConformantServers(true); TerminologyClientContext.setCanAllowNonConformantServers(true); // Currently all terminology clients are to R4 for version greater than R4 - final String txver = validator.setTerminologyServer(cli.getTxServer(), cli.getTxLog(), FhirPublication.R4, cli.isTxUseEcosystem()); + final String txver = validator.setTerminologyServer(cli.getTxServer(), cli.getEffectiveTxLogPath(), FhirPublication.R4, cli.isTxUseEcosystem()); + if (cli.isTxLogToConsole()) { + validator.getContext().addTxClientLogger(new ConsoleTerminologyClientLogger(cli.getTxLogConsoleBodyLimitValue())); + } log.debug("Version of the terminology server: {}", txver); } catch (final Exception e) { throw new TerminologyServerException("Error while setting the terminology server: " + e.getMessage(), e); diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java index ae84be359ab..487cb09ee15 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java @@ -73,6 +73,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import static ch.ahdis.matchbox.util.MatchboxServerUtils.addExtension; @@ -82,6 +83,7 @@ public class ValidationProvider { private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(ValidationProvider.class); + private static final String INFOWAY_TS_HOST = "terminologystandardsservice.ca"; @Autowired protected MatchboxEngineSupport matchboxEngineSupport; @@ -229,6 +231,7 @@ public IBaseResource validate(final HttpServletRequest theRequest) { log.error("Error during validation", e); return this.getOoForError("Error during validation: %s".formatted(e.getMessage())); } + filterTsSupplementFalsePositives(messages); long millis = sw.getMillis(); log.debug("Validation time: {}", sw); @@ -275,6 +278,38 @@ public IBaseResource validate(final HttpServletRequest theRequest) { }; } + void filterTsSupplementFalsePositives(final List messages) { + if (!isInfowayTsServer()) { + return; + } + messages.removeIf(this::isTsSupplementFalsePositive); + } + + boolean isTsSupplementFalsePositive(final ValidationMessage message) { + if (message == null || message.getMessage() == null) { + return false; + } + return isTsSupplementFalsePositive(message.getMessage()); + } + + boolean isTsSupplementFalsePositive(final String text) { + if (text == null) { + return false; + } + return text.contains("Required supplement not found: https://fhir.infoway-inforoute.ca/CodeSystem/Supplement/fr-CA/task-status|1.0.0") + || (text.contains("None of the codings provided are in the value set 'ReferralBusinessStatus'") + && text.contains("https://fhir.infoway-inforoute.ca/ValueSet/ca-referralbusinessstatus|1.2.0") + && text.contains("http://hl7.org/fhir/task-status#requested")); + } + + boolean isInfowayTsServer() { + if (cliContext == null || cliContext.getTxServer() == null) { + return false; + } + final String txServer = cliContext.getTxServer().toLowerCase(Locale.ROOT); + return txServer.contains(INFOWAY_TS_HOST); + } + private IBaseResource getOperationOutcome(final String id, final List messages, final String profile, @@ -345,6 +380,9 @@ private IBaseResource getOperationOutcome(final String id, // Add slice info to diagnostics if (message.hasSliceInfo() && message.sliceHtml != null) { List sliceInfo = engine.filterSlicingMessages(message.sliceHtml); + if (isInfowayTsServer()) { + sliceInfo.removeIf(this::isTsSupplementFalsePositive); + } if (!sliceInfo.isEmpty()) { final var newDiagnostics = new StringBuilder(); newDiagnostics.append(issue.getDiagnostics()); @@ -360,6 +398,10 @@ private IBaseResource getOperationOutcome(final String id, } } + if (cliContext.isTxLogToConsole()) { + logTerminologyIssue(issue); + } + oo.addIssue(issue); } @@ -374,6 +416,34 @@ private IBaseResource getOperationOutcome(final String id, return oo; } + void logTerminologyIssue(final OperationOutcome.OperationOutcomeIssueComponent issue) { + if (issue == null || !isTerminologyDiagnostic(issue.getDiagnostics())) { + return; + } + log.warn("TX OperationOutcome issue severity={} code={} diagnostics={}", + issue.getSeverity(), issue.getCode(), issue.getDiagnostics()); + } + + boolean isTerminologyDiagnostic(final String text) { + if (text == null) { + return false; + } + final String lower = text.toLowerCase(Locale.ROOT); + return lower.contains("valueset") + || lower.contains("value set") + || lower.contains("codesystem") + || lower.contains("code system") + || lower.contains("supplement") + || lower.contains("tx-resource") + || lower.contains("terminology") + || lower.contains("validate-code") + || lower.contains("unauthorized") + || lower.contains("server error") + || lower.contains("http 401") + || lower.contains("http 403") + || lower.contains("http 500"); + } + private IBaseResource getOoForError(final @NonNull String message) { final var oo = new OperationOutcome(); final var issue = oo.addIssue(); diff --git a/matchbox-server/src/main/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLogger.java b/matchbox-server/src/main/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLogger.java new file mode 100644 index 00000000000..c2fd6cea4d2 --- /dev/null +++ b/matchbox-server/src/main/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLogger.java @@ -0,0 +1,120 @@ +package org.hl7.fhir.r5.context; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import org.hl7.fhir.utilities.ToolingClientLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConsoleTerminologyClientLogger extends BaseLogger implements ToolingClientLogger { + + public static final int DEFAULT_BODY_LIMIT = 4000; + + private static final Logger log = LoggerFactory.getLogger(ConsoleTerminologyClientLogger.class); + private static final Pattern SECRET_JSON = + Pattern.compile("(?i)(\"(?:authorization|apikey|api_key|hubtoken|token|access_token)\"\\s*:\\s*\")([^\"]*)(\")"); + private static final Pattern SECRET_ASSIGNMENT = + Pattern.compile("(?i)\\b(authorization|apikey|api_key|hubtoken|token|access_token)(\\s*[=:]\\s*)([^\\s&;,\"'}]+)"); + + private final int bodyLimit; + + public ConsoleTerminologyClientLogger() { + this(DEFAULT_BODY_LIMIT); + } + + public ConsoleTerminologyClientLogger(int bodyLimit) { + this.bodyLimit = Math.max(0, bodyLimit); + } + + @Override + public void logRequest(String method, String url, List headers, byte[] body) { + String id = nextId(); + log.info("TX request {} {} {}", id, method, redact(url)); + log.info("TX request {} headers: {}", id, redactHeaders(headers)); + log.info("TX request {} body: {}", id, present(body)); + } + + @Override + public void logResponse(String outcome, List headers, byte[] body, long start) { + String id = getLastId() == null ? "unknown" : getLastId(); + long elapsed = elapsedMillis(start); + if (elapsed >= 0) { + log.info("TX response {} outcome={} elapsed={}ms", id, outcome, elapsed); + } else { + log.info("TX response {} outcome={}", id, outcome); + } + log.info("TX response {} headers: {}", id, redactHeaders(headers)); + log.info("TX response {} body: {}", id, present(body)); + } + + String present(byte[] body) { + if (body == null) { + return ""; + } + return truncate(redact(new String(body, StandardCharsets.UTF_8)), bodyLimit); + } + + static String redact(String text) { + if (text == null) { + return null; + } + String redacted = SECRET_JSON.matcher(text).replaceAll("$1[REDACTED]$3"); + return SECRET_ASSIGNMENT.matcher(redacted).replaceAll("$1$2[REDACTED]"); + } + + static List redactHeaders(List headers) { + if (headers == null) { + return List.of(); + } + List redacted = new ArrayList<>(); + for (String header : headers) { + redacted.add(redactHeader(header)); + } + return redacted; + } + + static String redactHeader(String header) { + if (header == null) { + return null; + } + int separator = header.indexOf(':'); + if (separator < 0) { + separator = header.indexOf('='); + } + if (separator > -1 && isSecretName(header.substring(0, separator))) { + String replacement = header.charAt(separator) == ':' ? " [REDACTED]" : "[REDACTED]"; + return header.substring(0, separator + 1) + replacement; + } + return redact(header); + } + + static String truncate(String text, int limit) { + if (text == null || text.length() <= limit) { + return text; + } + return text.substring(0, limit) + "...[truncated " + (text.length() - limit) + " chars]"; + } + + private static boolean isSecretName(String name) { + if (name == null) { + return false; + } + String normalized = name.trim().replace("-", "").replace("_", "").toLowerCase(); + return normalized.equals("authorization") + || normalized.equals("apikey") + || normalized.equals("hubtoken") + || normalized.equals("token") + || normalized.equals("accesstoken"); + } + + private static long elapsedMillis(long start) { + long now = System.currentTimeMillis(); + if (start <= 0 || start > now) { + return -1; + } + return now - start; + } +} diff --git a/matchbox-server/src/test/java/ch/ahdis/matchbox/CliContextTxLoggingTest.java b/matchbox-server/src/test/java/ch/ahdis/matchbox/CliContextTxLoggingTest.java new file mode 100644 index 00000000000..d0b44f57a8d --- /dev/null +++ b/matchbox-server/src/test/java/ch/ahdis/matchbox/CliContextTxLoggingTest.java @@ -0,0 +1,41 @@ +package ch.ahdis.matchbox; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.springframework.mock.env.MockEnvironment; + +class CliContextTxLoggingTest { + + @Test + void usesTxLoggingDefaults() { + CliContext cliContext = new CliContext(new MockEnvironment()); + + assertTrue(cliContext.isTxLogToConsole()); + assertEquals(4000, cliContext.getTxLogConsoleBodyLimitValue()); + } + + @Test + void readsTxLoggingEnvironmentAliases() { + MockEnvironment environment = new MockEnvironment() + .withProperty("TX_LOG_TO_CONSOLE", "false") + .withProperty("TX_LOG_CONSOLE_BODY_LIMIT", "123") + .withProperty("TX_LOG_PATH", "tx.log"); + + CliContext cliContext = new CliContext(environment); + + assertFalse(cliContext.isTxLogToConsole()); + assertEquals(123, cliContext.getTxLogConsoleBodyLimitValue()); + assertEquals("tx.log", cliContext.getEffectiveTxLogPath()); + } + + @Test + void fallsBackToDefaultBodyLimitWhenConfiguredValueIsInvalid() { + CliContext cliContext = new CliContext(new MockEnvironment() + .withProperty("TX_LOG_CONSOLE_BODY_LIMIT", "not-a-number")); + + assertEquals(4000, cliContext.getTxLogConsoleBodyLimitValue()); + } +} diff --git a/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java b/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java new file mode 100644 index 00000000000..c51bcbf5eba --- /dev/null +++ b/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java @@ -0,0 +1,91 @@ +package ch.ahdis.matchbox.validation; + +import ch.ahdis.matchbox.CliContext; +import org.hl7.fhir.utilities.validation.ValidationMessage; +import org.junit.jupiter.api.Test; +import org.springframework.mock.env.MockEnvironment; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ValidationProviderTsFilteringTest { + + private static final String SUPPLEMENT_ERROR = + "Required supplement not found: https://fhir.infoway-inforoute.ca/CodeSystem/Supplement/fr-CA/task-status|1.0.0"; + private static final String REFERRAL_BUSINESS_STATUS_ERROR = + "None of the codings provided are in the value set 'ReferralBusinessStatus' " + + "(https://fhir.infoway-inforoute.ca/ValueSet/ca-referralbusinessstatus|1.2.0), " + + "and a coding should come from this value set unless it has no suitable code " + + "(codes = http://hl7.org/fhir/task-status#requested)"; + + @Test + void recognizesOnlyKnownSupplementFalsePositiveMessages() { + ValidationProvider provider = new ValidationProvider(); + + assertTrue(provider.isTsSupplementFalsePositive(SUPPLEMENT_ERROR)); + assertTrue(provider.isTsSupplementFalsePositive(REFERRAL_BUSINESS_STATUS_ERROR)); + assertFalse(provider.isTsSupplementFalsePositive("Required supplement not found: https://example.org/other")); + assertFalse(provider.isTsSupplementFalsePositive("None of the codings provided are in the value set 'Other'")); + } + + @Test + void filtersKnownFalsePositivesOnlyForTs() { + ValidationProvider provider = providerWithTxServer("https://terminologystandardsservice.ca/tx/fhir"); + + List messages = new ArrayList<>(); + messages.add(message(SUPPLEMENT_ERROR)); + messages.add(message(REFERRAL_BUSINESS_STATUS_ERROR)); + messages.add(message("A real validation problem")); + + provider.filterTsSupplementFalsePositives(messages); + + assertEquals(1, messages.size()); + assertEquals("A real validation problem", messages.get(0).getMessage()); + } + + @Test + void doesNotFilterKnownMessagesForNonTsServers() { + ValidationProvider provider = providerWithTxServer("https://tx.fhir.org/r4"); + + List messages = new ArrayList<>(); + messages.add(message(SUPPLEMENT_ERROR)); + + provider.filterTsSupplementFalsePositives(messages); + + assertEquals(1, messages.size()); + } + + @Test + void doesNotRecognizeTsProxyServer() { + ValidationProvider provider = providerWithTxServer("https://smart-proxy.apibox.ca:10500/tx/fhir"); + + assertFalse(provider.isInfowayTsServer()); + } + + @Test + void recognizesTerminologyDiagnosticsForLogging() { + ValidationProvider provider = new ValidationProvider(); + + assertTrue(provider.isTerminologyDiagnostic("ValueSet expansion failed because tx-resource could not be processed")); + assertTrue(provider.isTerminologyDiagnostic("Required supplement not found")); + assertTrue(provider.isTerminologyDiagnostic("Terminology server returned HTTP 401 unauthorized")); + assertFalse(provider.isTerminologyDiagnostic("Patient.name is missing")); + } + + private ValidationProvider providerWithTxServer(String txServer) { + ValidationProvider provider = new ValidationProvider(); + provider.cliContext = new CliContext(new MockEnvironment()); + provider.cliContext.setTxServer(txServer); + return provider; + } + + private ValidationMessage message(String text) { + ValidationMessage message = new ValidationMessage(); + message.setMessage(text); + return message; + } +} diff --git a/matchbox-server/src/test/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLoggerTest.java b/matchbox-server/src/test/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLoggerTest.java new file mode 100644 index 00000000000..aefd1ad0e18 --- /dev/null +++ b/matchbox-server/src/test/java/org/hl7/fhir/r5/context/ConsoleTerminologyClientLoggerTest.java @@ -0,0 +1,48 @@ +package org.hl7.fhir.r5.context; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.Test; + +class ConsoleTerminologyClientLoggerTest { + + @Test + void redactsKnownSecretHeaders() { + List headers = ConsoleTerminologyClientLogger.redactHeaders(List.of( + "Authorization: Bearer secret", + "apiKey: abc123", + "hubtoken=secret-hub", + "Content-Type: application/fhir+json" + )); + + assertEquals("Authorization: [REDACTED]", headers.get(0)); + assertEquals("apiKey: [REDACTED]", headers.get(1)); + assertEquals("hubtoken=[REDACTED]", headers.get(2)); + assertEquals("Content-Type: application/fhir+json", headers.get(3)); + } + + @Test + void redactsKnownSecretValuesInUrlsAndBodies() { + String redacted = ConsoleTerminologyClientLogger.redact( + "https://example.org/tx?access_token=secret-token&ok=true {\"token\":\"body-secret\"}"); + + assertTrue(redacted.contains("access_token=[REDACTED]")); + assertTrue(redacted.contains("\"token\":\"[REDACTED]\"")); + assertFalse(redacted.contains("secret-token")); + assertFalse(redacted.contains("body-secret")); + } + + @Test + void truncatesRedactedBodies() { + ConsoleTerminologyClientLogger logger = new ConsoleTerminologyClientLogger(10); + + String body = logger.present("0123456789abcdef".getBytes(StandardCharsets.UTF_8)); + + assertEquals("0123456789...[truncated 6 chars]", body); + } +} From fd5ae8e9ed962623ae7442db3dde3edc1aca7c1b Mon Sep 17 00:00:00 2001 From: Azhar Saleem Date: Wed, 10 Jun 2026 16:43:00 -0400 Subject: [PATCH 2/3] Remove TS diagnostic suppression --- .../validation/ValidationProvider.java | 37 -------- ...idationProviderTerminologyLoggingTest.java | 19 ++++ .../ValidationProviderTsFilteringTest.java | 91 ------------------- 3 files changed, 19 insertions(+), 128 deletions(-) create mode 100644 matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTerminologyLoggingTest.java delete mode 100644 matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java index 487cb09ee15..fe48533ad48 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java @@ -83,7 +83,6 @@ public class ValidationProvider { private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(ValidationProvider.class); - private static final String INFOWAY_TS_HOST = "terminologystandardsservice.ca"; @Autowired protected MatchboxEngineSupport matchboxEngineSupport; @@ -231,7 +230,6 @@ public IBaseResource validate(final HttpServletRequest theRequest) { log.error("Error during validation", e); return this.getOoForError("Error during validation: %s".formatted(e.getMessage())); } - filterTsSupplementFalsePositives(messages); long millis = sw.getMillis(); log.debug("Validation time: {}", sw); @@ -278,38 +276,6 @@ public IBaseResource validate(final HttpServletRequest theRequest) { }; } - void filterTsSupplementFalsePositives(final List messages) { - if (!isInfowayTsServer()) { - return; - } - messages.removeIf(this::isTsSupplementFalsePositive); - } - - boolean isTsSupplementFalsePositive(final ValidationMessage message) { - if (message == null || message.getMessage() == null) { - return false; - } - return isTsSupplementFalsePositive(message.getMessage()); - } - - boolean isTsSupplementFalsePositive(final String text) { - if (text == null) { - return false; - } - return text.contains("Required supplement not found: https://fhir.infoway-inforoute.ca/CodeSystem/Supplement/fr-CA/task-status|1.0.0") - || (text.contains("None of the codings provided are in the value set 'ReferralBusinessStatus'") - && text.contains("https://fhir.infoway-inforoute.ca/ValueSet/ca-referralbusinessstatus|1.2.0") - && text.contains("http://hl7.org/fhir/task-status#requested")); - } - - boolean isInfowayTsServer() { - if (cliContext == null || cliContext.getTxServer() == null) { - return false; - } - final String txServer = cliContext.getTxServer().toLowerCase(Locale.ROOT); - return txServer.contains(INFOWAY_TS_HOST); - } - private IBaseResource getOperationOutcome(final String id, final List messages, final String profile, @@ -380,9 +346,6 @@ private IBaseResource getOperationOutcome(final String id, // Add slice info to diagnostics if (message.hasSliceInfo() && message.sliceHtml != null) { List sliceInfo = engine.filterSlicingMessages(message.sliceHtml); - if (isInfowayTsServer()) { - sliceInfo.removeIf(this::isTsSupplementFalsePositive); - } if (!sliceInfo.isEmpty()) { final var newDiagnostics = new StringBuilder(); newDiagnostics.append(issue.getDiagnostics()); diff --git a/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTerminologyLoggingTest.java b/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTerminologyLoggingTest.java new file mode 100644 index 00000000000..8c9061cd6fe --- /dev/null +++ b/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTerminologyLoggingTest.java @@ -0,0 +1,19 @@ +package ch.ahdis.matchbox.validation; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ValidationProviderTerminologyLoggingTest { + + @Test + void recognizesTerminologyDiagnosticsForLogging() { + ValidationProvider provider = new ValidationProvider(); + + assertTrue(provider.isTerminologyDiagnostic("ValueSet expansion failed because tx-resource could not be processed")); + assertTrue(provider.isTerminologyDiagnostic("Required supplement not found")); + assertTrue(provider.isTerminologyDiagnostic("Terminology server returned HTTP 401 unauthorized")); + assertFalse(provider.isTerminologyDiagnostic("Patient.name is missing")); + } +} diff --git a/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java b/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java deleted file mode 100644 index c51bcbf5eba..00000000000 --- a/matchbox-server/src/test/java/ch/ahdis/matchbox/validation/ValidationProviderTsFilteringTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package ch.ahdis.matchbox.validation; - -import ch.ahdis.matchbox.CliContext; -import org.hl7.fhir.utilities.validation.ValidationMessage; -import org.junit.jupiter.api.Test; -import org.springframework.mock.env.MockEnvironment; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class ValidationProviderTsFilteringTest { - - private static final String SUPPLEMENT_ERROR = - "Required supplement not found: https://fhir.infoway-inforoute.ca/CodeSystem/Supplement/fr-CA/task-status|1.0.0"; - private static final String REFERRAL_BUSINESS_STATUS_ERROR = - "None of the codings provided are in the value set 'ReferralBusinessStatus' " + - "(https://fhir.infoway-inforoute.ca/ValueSet/ca-referralbusinessstatus|1.2.0), " + - "and a coding should come from this value set unless it has no suitable code " + - "(codes = http://hl7.org/fhir/task-status#requested)"; - - @Test - void recognizesOnlyKnownSupplementFalsePositiveMessages() { - ValidationProvider provider = new ValidationProvider(); - - assertTrue(provider.isTsSupplementFalsePositive(SUPPLEMENT_ERROR)); - assertTrue(provider.isTsSupplementFalsePositive(REFERRAL_BUSINESS_STATUS_ERROR)); - assertFalse(provider.isTsSupplementFalsePositive("Required supplement not found: https://example.org/other")); - assertFalse(provider.isTsSupplementFalsePositive("None of the codings provided are in the value set 'Other'")); - } - - @Test - void filtersKnownFalsePositivesOnlyForTs() { - ValidationProvider provider = providerWithTxServer("https://terminologystandardsservice.ca/tx/fhir"); - - List messages = new ArrayList<>(); - messages.add(message(SUPPLEMENT_ERROR)); - messages.add(message(REFERRAL_BUSINESS_STATUS_ERROR)); - messages.add(message("A real validation problem")); - - provider.filterTsSupplementFalsePositives(messages); - - assertEquals(1, messages.size()); - assertEquals("A real validation problem", messages.get(0).getMessage()); - } - - @Test - void doesNotFilterKnownMessagesForNonTsServers() { - ValidationProvider provider = providerWithTxServer("https://tx.fhir.org/r4"); - - List messages = new ArrayList<>(); - messages.add(message(SUPPLEMENT_ERROR)); - - provider.filterTsSupplementFalsePositives(messages); - - assertEquals(1, messages.size()); - } - - @Test - void doesNotRecognizeTsProxyServer() { - ValidationProvider provider = providerWithTxServer("https://smart-proxy.apibox.ca:10500/tx/fhir"); - - assertFalse(provider.isInfowayTsServer()); - } - - @Test - void recognizesTerminologyDiagnosticsForLogging() { - ValidationProvider provider = new ValidationProvider(); - - assertTrue(provider.isTerminologyDiagnostic("ValueSet expansion failed because tx-resource could not be processed")); - assertTrue(provider.isTerminologyDiagnostic("Required supplement not found")); - assertTrue(provider.isTerminologyDiagnostic("Terminology server returned HTTP 401 unauthorized")); - assertFalse(provider.isTerminologyDiagnostic("Patient.name is missing")); - } - - private ValidationProvider providerWithTxServer(String txServer) { - ValidationProvider provider = new ValidationProvider(); - provider.cliContext = new CliContext(new MockEnvironment()); - provider.cliContext.setTxServer(txServer); - return provider; - } - - private ValidationMessage message(String text) { - ValidationMessage message = new ValidationMessage(); - message.setMessage(text); - return message; - } -} From 00261c349db60665a7bd52a3cfb4fef2278b4788 Mon Sep 17 00:00:00 2001 From: Azhar Saleem Date: Wed, 10 Jun 2026 22:41:04 -0400 Subject: [PATCH 3/3] Tag TS-fetched terminology resources --- .../fhir/r5/context/BaseWorkerContext.java | 41 +++++++++++++++-- .../BaseWorkerContextTsSupportTest.java | 44 +++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java b/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java index dd4bab70275..e07a0f99dd3 100644 --- a/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java +++ b/matchbox-engine/src/main/java/org/hl7/fhir/r5/context/BaseWorkerContext.java @@ -41,6 +41,7 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -159,6 +160,8 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS public abstract class BaseWorkerContext extends I18nBase implements IWorkerContext, IWorkerContextManager, IOIDServices { private static boolean allowedToIterateTerminologyResources; private static final String INFOWAY_TS_HOST = "terminologystandardsservice.ca"; + private static final String TERMINOLOGY_SERVER_PACKAGE_ID = "matchbox.terminology-server"; + private static final String TERMINOLOGY_SERVER_PACKAGE_VERSION = "live"; public interface IByteProvider { @@ -3797,6 +3800,9 @@ private T doFindTxResource(Class class_, String canonica svs = txCache.getValueSet(canonical); } else { svs = terminologyClientManager.findValueSetOnServer(canonical); + if (svs != null) { + prepareTerminologyServerResource(svs.getVs(), version, svs.getServer()); + } txCache.cacheValueSet(canonical, svs); } if (svs != null) { @@ -3810,7 +3816,7 @@ private T doFindTxResource(Class class_, String canonica if (svs == null) { return null; } else { - cacheResource(svs.getVs()); + cacheResourceFromTerminologyServer(svs.getVs(), svs.getServer()); cacheRequiredSupplements(svs.getVs()); return (T) svs.getVs(); } @@ -3824,6 +3830,9 @@ private T doFindTxResource(Class class_, String canonica if (scs == null) { scs = findCodeSystemDirectlyOnServer(canonical); } + if (scs != null) { + prepareTerminologyServerResource(scs.getCs(), version, scs.getServer()); + } txCache.cacheCodeSystem(canonical, scs); } if (scs != null) { @@ -3837,7 +3846,7 @@ private T doFindTxResource(Class class_, String canonica if (scs == null) { return null; } else { - cacheResource(scs.getCs()); + cacheResourceFromTerminologyServer(scs.getCs(), scs.getServer()); return (T) scs.getCs(); } } else { @@ -3845,6 +3854,31 @@ private T doFindTxResource(Class class_, String canonica } } + private void cacheResourceFromTerminologyServer(Resource resource, String server) throws FHIRException { + if (resource == null) { + return; + } + PackageInformation packageInfo = prepareTerminologyServerResource(resource, version, server); + cacheResourceFromPackage(resource, packageInfo); + } + + static PackageInformation prepareTerminologyServerResource(Resource resource, String fhirVersion, String server) { + PackageInformation packageInfo = resource instanceof CanonicalResource && ((CanonicalResource) resource).hasSourcePackage() + ? ((CanonicalResource) resource).getSourcePackage() + : terminologyServerPackageInfo(fhirVersion, server); + if (resource instanceof CanonicalResource && !((CanonicalResource) resource).hasSourcePackage()) { + ((CanonicalResource) resource).setSourcePackage(packageInfo); + } + return packageInfo; + } + + static PackageInformation terminologyServerPackageInfo(String fhirVersion, String server) { + String canonical = Utilities.noString(server) ? TERMINOLOGY_SERVER_PACKAGE_ID : server; + return new PackageInformation(TERMINOLOGY_SERVER_PACKAGE_ID, TERMINOLOGY_SERVER_PACKAGE_VERSION, + Utilities.noString(fhirVersion) ? "unknown" : fhirVersion, new Date(0), + "Terminology server resources", canonical, canonical); + } + private void cacheRequiredSupplements(ValueSet vs) { cacheRequiredSupplements(vs, new HashSet<>()); } @@ -3862,7 +3896,7 @@ private void cacheRequiredSupplements(ValueSet vs, Set visited) { if (!Utilities.noString(canonical) && fetchResource(CodeSystem.class, canonical) == null) { SourcedCodeSystem scs = findCodeSystemDirectlyOnServer(canonical); if (scs != null && scs.getCs() != null) { - cacheResource(scs.getCs()); + cacheResourceFromTerminologyServer(scs.getCs(), scs.getServer()); txCache.cacheCodeSystem(canonical, scs); } } @@ -3919,6 +3953,7 @@ private SourcedCodeSystem findCodeSystemDirectlyOnServer(String canonical) { return null; } CodeSystem cs = (CodeSystem) client.getClient().read("CodeSystem", rid); + prepareTerminologyServerResource(cs, version, client.getAddress()); return new SourcedCodeSystem(client.getAddress(), cs); } catch (Exception e) { logger.logDebugMessage(LogCategory.TX, "Error resolving CodeSystem directly on terminology server: "+canonical+" - "+e.getMessage()); diff --git a/matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java b/matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java index b6ceadabd06..622ed61f9c5 100644 --- a/matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java +++ b/matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java @@ -1,9 +1,14 @@ package org.hl7.fhir.r5.context; +import org.hl7.fhir.r5.model.CodeSystem; +import org.hl7.fhir.r5.model.PackageInformation; import org.hl7.fhir.r5.model.ValueSet; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; class BaseWorkerContextTsSupportTest { @@ -53,4 +58,43 @@ void keepsValueSetAsTxResourceForTsProxy() { assertFalse(BaseWorkerContext.shouldOmitTxResourceForTsValueSet("https://smart-proxy.apibox.ca:10500/tx/fhir", valueSet, true)); } + + @Test + void tagsLiveTsValueSetWithPackageMetadataBeforeCaching() { + ValueSet valueSet = new ValueSet(); + + PackageInformation packageInfo = BaseWorkerContext.prepareTerminologyServerResource(valueSet, "4.0.1", TS_URL); + + assertNotNull(packageInfo); + assertTrue(valueSet.hasSourcePackage()); + assertSame(packageInfo, valueSet.getSourcePackage()); + assertEquals("matchbox.terminology-server", packageInfo.getId()); + assertEquals("live", packageInfo.getVersion()); + assertEquals("matchbox.terminology-server#live", packageInfo.getVID()); + assertEquals("4.0.1", packageInfo.getFhirVersion()); + assertEquals(TS_URL, packageInfo.getCanonical()); + } + + @Test + void tagsLiveTsSupplementCodeSystemWithPackageMetadataBeforeCaching() { + CodeSystem supplement = new CodeSystem(); + + PackageInformation packageInfo = BaseWorkerContext.prepareTerminologyServerResource(supplement, "4.0.1", TS_URL); + + assertTrue(supplement.hasSourcePackage()); + assertSame(packageInfo, supplement.getSourcePackage()); + assertEquals("matchbox.terminology-server#live", supplement.getSourcePackage().getVID()); + } + + @Test + void keepsExistingPackageMetadataOnLiveTsResource() { + ValueSet valueSet = new ValueSet(); + PackageInformation existing = new PackageInformation("example.package", "1.2.3", "4.0.1", new java.util.Date(0)); + valueSet.setSourcePackage(existing); + + PackageInformation packageInfo = BaseWorkerContext.prepareTerminologyServerResource(valueSet, "4.0.1", TS_URL); + + assertSame(existing, packageInfo); + assertSame(existing, valueSet.getSourcePackage()); + } }