Fix TS terminology resource handling

matchbox-engine/src/test/java/org/hl7/fhir/r5/context/BaseWorkerContextTsSupportTest.java

@@ -0,0 +1,100 @@
+package org.hl7.fhir.r5.context;
+
+import org.hl7.fhir.r5.model.CodeSystem;
+import org.hl7.fhir.r5.model.PackageInformation;
+import org.hl7.fhir.r5.model.ValueSet;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class BaseWorkerContextTsSupportTest {
+
+  private static final String TS_URL = "https://terminologystandardsservice.ca/tx/fhir";
+
+  @Test
+  void omitsAnyValueSetFoundExactlyOnceOnTs() {
+    ValueSet valueSet = new ValueSet();
+    valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/healthcareproviderspecialtycode");
+
+    assertTrue(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, true));
+
+    valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/another-ts-hosted-valueset");
+    assertTrue(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, true));
+  }
+
+  @Test
+  void omitsVersionedValueSetThatExistsOnTs() {
+    ValueSet valueSet = new ValueSet();
+    valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/versioned");
+    valueSet.setVersion("1.0.0");
+
+    assertTrue(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, true));
+  }
+
+  @Test
+  void keepsValueSetAsTxResourceWhenItIsNotFoundExactlyOnceOnTs() {
+    ValueSet valueSet = new ValueSet();
+    valueSet.setUrl("https://example.org/ValueSet/local-only");
+
+    assertFalse(BaseWorkerContext.shouldOmitTxResourceForTsValueSet(TS_URL, valueSet, false));
+  }
+
+  @Test
+  void keepsValueSetAsTxResourceForOtherServers() {
+    ValueSet valueSet = new ValueSet();
+    valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/ts-hosted");
+
+    assertFalse(BaseWorkerContext.shouldOmitTxResourceForTsValueSet("https://tx.fhir.org/r4", valueSet, true));
+  }
+
+  @Test
+  void keepsValueSetAsTxResourceForTsProxy() {
+    ValueSet valueSet = new ValueSet();
+    valueSet.setUrl("https://fhir.infoway-inforoute.ca/ValueSet/ts-hosted");
+
+    assertFalse(BaseWorkerContext.shouldOmitTxResourceForTsValueSet("https://smart-proxy.apibox.ca:10500/tx/fhir", valueSet, true));
+  }
+
+  @Test
+  void tagsLiveTsValueSetWithPackageMetadataBeforeCaching() {
+    ValueSet valueSet = new ValueSet();
+
+    PackageInformation packageInfo = BaseWorkerContext.prepareTerminologyServerResource(valueSet, "4.0.1", TS_URL);
+
+    assertNotNull(packageInfo);
+    assertTrue(valueSet.hasSourcePackage());
+    assertSame(packageInfo, valueSet.getSourcePackage());
+    assertEquals("matchbox.terminology-server", packageInfo.getId());
+    assertEquals("live", packageInfo.getVersion());
+    assertEquals("matchbox.terminology-server#live", packageInfo.getVID());
+    assertEquals("4.0.1", packageInfo.getFhirVersion());
+    assertEquals(TS_URL, packageInfo.getCanonical());
+  }
+
+  @Test
+  void tagsLiveTsSupplementCodeSystemWithPackageMetadataBeforeCaching() {
+    CodeSystem supplement = new CodeSystem();
+
+    PackageInformation packageInfo = BaseWorkerContext.prepareTerminologyServerResource(supplement, "4.0.1", TS_URL);
+
+    assertTrue(supplement.hasSourcePackage());
+    assertSame(packageInfo, supplement.getSourcePackage());
+    assertEquals("matchbox.terminology-server#live", supplement.getSourcePackage().getVID());
+  }
+
+  @Test
+  void keepsExistingPackageMetadataOnLiveTsResource() {
+    ValueSet valueSet = new ValueSet();
+    PackageInformation existing = new PackageInformation("example.package", "1.2.3", "4.0.1", new java.util.Date(0));
+    valueSet.setSourcePackage(existing);
+
+    PackageInformation packageInfo = BaseWorkerContext.prepareTerminologyServerResource(valueSet, "4.0.1", TS_URL);
+
+    assertSame(existing, packageInfo);
+    assertSame(existing, valueSet.getSourcePackage());
+  }
+}

matchbox-server/src/main/java/ch/ahdis/matchbox/CliContext.java

@@ -124,6 +124,15 @@ public CliContext setIgs(String[] igs) {
   @JsonProperty("txLog")
   private String txLog = null;
 
+  @JsonProperty("txLogPath")
+  private String txLogPath = null;
+
+  @JsonProperty("txLogToConsole")
+  private boolean txLogToConsole = true;
+
+  @JsonProperty("txLogConsoleBodyLimit")
+  private String txLogConsoleBodyLimit = "4000";
+
   @JsonProperty("txUseEcosystem")
   private boolean txUseEcosystem = true;
 
@@ -330,6 +339,7 @@ public CliContext(Environment environment) {
         }
       }
     }
+    applyTxLogEnvironmentAliases(environment);
     // get properties array from the environment?
     this.igsPreloaded = environment.getProperty("matchbox.fhir.context.igsPreloaded", String[].class);
     this.onlyOneEngine = environment.getProperty("matchbox.fhir.context.onlyOneEngine", Boolean.class, false);
@@ -362,6 +372,24 @@ public CliContext(CliContext other) {
     this.suppressErrors = other.suppressErrors;
     this.suppressWarnInfos = other.suppressWarnInfos;
     this.igs = other.igs;
+    this.txLogPath = other.txLogPath;
+    this.txLogToConsole = other.txLogToConsole;
+    this.txLogConsoleBodyLimit = other.txLogConsoleBodyLimit;
+  }
+
+  private void applyTxLogEnvironmentAliases(Environment environment) {
+    String txLogPath = environment.getProperty("TX_LOG_PATH", String.class);
+    if (txLogPath != null && !txLogPath.isBlank()) {
+      this.txLogPath = txLogPath;
+    }
+    String txLogToConsole = environment.getProperty("TX_LOG_TO_CONSOLE", String.class);
+    if (txLogToConsole != null && !txLogToConsole.isBlank()) {
+      this.txLogToConsole = Boolean.parseBoolean(txLogToConsole);
+    }
+    String txLogConsoleBodyLimit = environment.getProperty("TX_LOG_CONSOLE_BODY_LIMIT", String.class);
+    if (txLogConsoleBodyLimit != null && !txLogConsoleBodyLimit.isBlank()) {
+      this.txLogConsoleBodyLimit = txLogConsoleBodyLimit;
+    }
   }
 
   public String getIg() {
@@ -412,6 +440,42 @@ public void setTxLog(String txLog) {
     this.txLog = txLog;
   }
 
+  public String getTxLogPath() {
+    return txLogPath;
+  }
+
+  public void setTxLogPath(String txLogPath) {
+    this.txLogPath = txLogPath;
+  }
+
+  public String getEffectiveTxLogPath() {
+    return txLogPath != null && !txLogPath.isBlank() ? txLogPath : txLog;
+  }
+
+  public boolean isTxLogToConsole() {
+    return txLogToConsole;
+  }
+
+  public void setTxLogToConsole(boolean txLogToConsole) {
+    this.txLogToConsole = txLogToConsole;
+  }
+
+  public String getTxLogConsoleBodyLimit() {
+    return txLogConsoleBodyLimit;
+  }
+
+  public void setTxLogConsoleBodyLimit(String txLogConsoleBodyLimit) {
+    this.txLogConsoleBodyLimit = txLogConsoleBodyLimit;
+  }
+
+  public int getTxLogConsoleBodyLimitValue() {
+    try {
+      return Integer.parseInt(txLogConsoleBodyLimit);
+    } catch (NumberFormatException e) {
+      return 4000;
+    }
+  }
+
   public void setTxUseEcosystem(boolean txUseEcosystem) {
     this.txUseEcosystem = txUseEcosystem;
   }
@@ -779,6 +843,9 @@ public boolean equals(final Object o) {
         && Objects.equals(txServer, that.txServer)
         && txServerCache == that.txServerCache
         && Objects.equals(txLog, that.txLog)
+        && Objects.equals(txLogPath, that.txLogPath)
+        && txLogToConsole == that.txLogToConsole
+        && Objects.equals(txLogConsoleBodyLimit, that.txLogConsoleBodyLimit)
         && txUseEcosystem == that.txUseEcosystem
         && Objects.equals(lang, that.lang)
         && Objects.equals(snomedCT, that.snomedCT)
@@ -833,6 +900,9 @@ public int hashCode() {
         txServer,
         txServerCache,
         txLog,
+        txLogPath,
+        txLogToConsole,
+        txLogConsoleBodyLimit,
         txUseEcosystem,
         lang,
         snomedCT,
@@ -885,6 +955,9 @@ public String toString() {
         ", txServer='" + txServer + '\'' +
         ", txServerCache='" + txServerCache + '\'' +
         ", txLog='" + txLog + '\'' +
+        ", txLogPath='" + txLogPath + '\'' +
+        ", txLogToConsole=" + txLogToConsole +
+        ", txLogConsoleBodyLimit='" + txLogConsoleBodyLimit + '\'' +
         ", txUseEcosystem=" + txUseEcosystem +
         ", lang='" + lang + '\'' +
         ", snomedCT='" + snomedCT + '\'' +
@@ -959,6 +1032,9 @@ public void addContextToExtension(final Extension ext) {
     addExtension(ext, "txServer", new UriType(this.txServer));
     addExtension(ext, "txServerCache", new BooleanType(this.txServerCache));
     addExtension(ext, "txLog", new StringType(this.txLog));
+    addExtension(ext, "txLogPath", new StringType(this.txLogPath));
+    addExtension(ext, "txLogToConsole", new BooleanType(this.txLogToConsole));
+    addExtension(ext, "txLogConsoleBodyLimit", new StringType(this.txLogConsoleBodyLimit));
     addExtension(ext, "txUseEcosystem", new BooleanType(this.txUseEcosystem));
     addExtension(ext, "lang", new StringType(this.lang));
     addExtension(ext, "snomedCT", new StringType(this.snomedCT));

matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java

@@ -16,6 +16,7 @@
 import ch.ahdis.matchbox.util.http.HttpRequestWrapper;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
+import org.hl7.fhir.r5.context.ConsoleTerminologyClientLogger;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -527,7 +528,10 @@ private void configureValidationEngine(final MatchboxEngine validator,
 				TerminologyClientContext.setAllowNonConformantServers(true);
 				TerminologyClientContext.setCanAllowNonConformantServers(true);
 				// Currently all terminology clients are to R4 for version greater than R4
-				final String txver = validator.setTerminologyServer(cli.getTxServer(), cli.getTxLog(), FhirPublication.R4, cli.isTxUseEcosystem());
+				final String txver = validator.setTerminologyServer(cli.getTxServer(), cli.getEffectiveTxLogPath(), FhirPublication.R4, cli.isTxUseEcosystem());
+				if (cli.isTxLogToConsole()) {
+					validator.getContext().addTxClientLogger(new ConsoleTerminologyClientLogger(cli.getTxLogConsoleBodyLimitValue()));
+				}
 				log.debug("Version of the terminology server: {}", txver);
 			} catch (final Exception e) {
 				throw new TerminologyServerException("Error while setting the terminology server: " + e.getMessage(), e);

matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java

@@ -73,6 +73,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 
 import static ch.ahdis.matchbox.util.MatchboxServerUtils.addExtension;
 
@@ -360,6 +361,10 @@ private IBaseResource getOperationOutcome(final String id,
 				}
 			}
 
+			if (cliContext.isTxLogToConsole()) {
+				logTerminologyIssue(issue);
+			}
+
 			oo.addIssue(issue);
 		}
 
@@ -374,6 +379,34 @@ private IBaseResource getOperationOutcome(final String id,
 		return oo;
 	}
 
+	void logTerminologyIssue(final OperationOutcome.OperationOutcomeIssueComponent issue) {
+		if (issue == null || !isTerminologyDiagnostic(issue.getDiagnostics())) {
+			return;
+		}
+		log.warn("TX OperationOutcome issue severity={} code={} diagnostics={}",
+			issue.getSeverity(), issue.getCode(), issue.getDiagnostics());
+	}
+
+	boolean isTerminologyDiagnostic(final String text) {
+		if (text == null) {
+			return false;
+		}
+		final String lower = text.toLowerCase(Locale.ROOT);
+		return lower.contains("valueset")
+			|| lower.contains("value set")
+			|| lower.contains("codesystem")
+			|| lower.contains("code system")
+			|| lower.contains("supplement")
+			|| lower.contains("tx-resource")
+			|| lower.contains("terminology")
+			|| lower.contains("validate-code")
+			|| lower.contains("unauthorized")
+			|| lower.contains("server error")
+			|| lower.contains("http 401")
+			|| lower.contains("http 403")
+			|| lower.contains("http 500");
+	}
+
 	private IBaseResource getOoForError(final @NonNull String message) {
 		final var oo = new OperationOutcome();
 		final var issue = oo.addIssue();