Related verses (#385)

pom.xml

@@ -3,7 +3,7 @@
 
     <groupId>com.tyndalehouse.step</groupId>
     <artifactId>step-mvn</artifactId>
-    <version>26.5.2</version>
+    <version>26.5.3</version>
     <packaging>pom</packaging>
     <name>STEP :: Scripture Tools for Every pastor</name>
 

step-assembly/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>step-mvn</artifactId>
         <groupId>com.tyndalehouse.step</groupId>
-        <version>26.5.2</version>
+        <version>26.5.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <properties>

step-build/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>com.tyndalehouse.step</groupId>
 		<artifactId>step-mvn</artifactId>
-		<version>26.5.2</version>
+		<version>26.5.3</version>
 	</parent>
 
 	<artifactId>step-build</artifactId>

step-core-data/pom.xml

@@ -7,7 +7,7 @@
     <parent>
         <groupId>com.tyndalehouse.step</groupId>
         <artifactId>step-mvn</artifactId>
-        <version>26.5.2</version>
+        <version>26.5.3</version>
     </parent>
 
     <artifactId>step-core-data</artifactId>

step-core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.tyndalehouse.step</groupId>
         <artifactId>step-mvn</artifactId>
-        <version>26.5.2</version>
+        <version>26.5.3</version>
     </parent>
 
     <groupId>com.tyndalehouse.step</groupId>

step-core/src/main/java/com/tyndalehouse/step/core/guice/StepCoreModule.java

@@ -88,6 +88,7 @@ protected void doConfigure() {
         bind(OriginalWordSuggestionService.class).to(OriginalWordSuggestionServiceImpl.class);
         bind(SupportRequestService.class).to(SupportRequestServiceImpl.class);
         bind(JSwordRelatedVersesService.class).to(JSwordRelatedVersesServiceImpl.class);
+        bind(SemanticRelatedVersesService.class).to(SemanticRelatedVersesServiceImpl.class).asEagerSingleton();
 
         bind(new TypeLiteral<List<String>>() {
         }).annotatedWith(Names.named("defaultVersions")).toProvider(DefaultVersionsProvider.class);

step-core/src/main/java/com/tyndalehouse/step/core/models/SearchToken.java

@@ -17,6 +17,7 @@ public class SearchToken implements Serializable {
     public static final String MEANINGS = "meanings";
     public static final String TOPIC_BY_REF = "topicref";
     public static final String RELATED_VERSES = "relatedrefs";
+    public static final String RELATED_VERSES_SEMANTIC = "relatedrefsSemantic";
     public static final String EXACT_FORM = "exactForm";
     public static final String SYNTAX = "syntax";
     public static final String LIMIT = "limit";

step-core/src/main/java/com/tyndalehouse/step/core/service/SemanticRelatedVersesService.java

@@ -0,0 +1,14 @@
+package com.tyndalehouse.step.core.service;
+
+import java.util.List;
+
+public interface SemanticRelatedVersesService {
+    /** Returns ranked semantically-related NRSV verse OSIS refs for an NRSV input ref.
+     *  Returns empty list if the ref is not in the dataset.
+     *
+     *  CONTRACT: input MUST already be in NRSV versification. The dataset is
+     *  NRSV-keyed by construction. Callers are responsible for inbound
+     *  (user-v11n -> NRSV) and outbound (NRSV -> user-v11n) verse mapping.
+     *  This service has no JSword dependency. */
+    List<String> getRelatedNrsvRefs(String nrsvOsisRef);
+}

step-core/src/main/java/com/tyndalehouse/step/core/service/impl/SearchType.java

@@ -31,6 +31,11 @@ public enum SearchType {
      * Finds all verses that are related to another verse
      */
     RELATED_VERSES("verse_related"),
+    /**
+     * Semantic related-verses lookup. Internal dispatch only — normalized to
+     * RELATED_VERSES on the wire by SearchServiceImpl#getBestSearchType.
+     */
+    RELATED_VERSES_SEMANTIC("verse_related"),
 
     /** A timeline description search */
     TIMELINE_DESCRIPTION("search_timeline"),

step-core/src/main/java/com/tyndalehouse/step/core/service/impl/SemanticRelatedVersesServiceImpl.java

@@ -0,0 +1,111 @@
+package com.tyndalehouse.step.core.service.impl;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.tyndalehouse.step.core.data.create.ModuleLoader;
+import com.tyndalehouse.step.core.exceptions.StepInternalException;
+import com.tyndalehouse.step.core.service.SemanticRelatedVersesService;
+
+import javax.inject.Singleton;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+@Singleton
+public class SemanticRelatedVersesServiceImpl implements SemanticRelatedVersesService {
+
+    private static final String RESOURCE_PATH =
+            "related-verses/bible_semantic_export_minified.json";
+
+    private final String[] pool;
+    private final int[][] related;
+
+    public SemanticRelatedVersesServiceImpl() {
+        final JsonFactory factory = new JsonFactory();
+        factory.disable(JsonFactory.Feature.INTERN_FIELD_NAMES);
+
+        final InputStream stream = ModuleLoader.class.getResourceAsStream(RESOURCE_PATH);
+        if (stream == null) {
+            throw new StepInternalException("Unable to read resource: " + RESOURCE_PATH);
+        }
+
+        final HashMap<String, String[]> raw = new HashMap<String, String[]>(40_000);
+        final HashMap<String, String> internCache = new HashMap<String, String>(40_000);
+
+        try {
+            final JsonParser p = factory.createParser(stream);
+            try {
+                if (p.nextToken() != JsonToken.START_OBJECT) {
+                    throw new StepInternalException("Expected JSON object at root: " + RESOURCE_PATH);
+                }
+
+                while (p.nextToken() == JsonToken.FIELD_NAME) {
+                    final String key = p.getText();
+                    if (p.nextToken() != JsonToken.START_ARRAY) {
+                        throw new StepInternalException("Expected array at " + key);
+                    }
+                    final List<String> vals = new ArrayList<String>(100);
+                    while (p.nextToken() != JsonToken.END_ARRAY) {
+                        final String val = p.getText();
+                        final String prev = internCache.putIfAbsent(val, val);
+                        vals.add(prev != null ? prev : val);
+                    }
+                    raw.put(key, vals.toArray(new String[0]));
+                }
+            } finally {
+                p.close();
+            }
+        } catch (IOException e) {
+            throw new StepInternalException(
+                    "Failed to load semantic related verses from " + RESOURCE_PATH + ": " + e.getMessage(), e);
+        } finally {
+            try { stream.close(); } catch (IOException ignored) { }
+        }
+
+        final TreeSet<String> union = new TreeSet<String>();
+        for (Map.Entry<String, String[]> e : raw.entrySet()) {
+            union.add(e.getKey());
+            for (String v : e.getValue()) {
+                union.add(v);
+            }
+        }
+
+        this.pool = union.toArray(new String[0]);
+        this.related = new int[this.pool.length][];
+
+        for (Map.Entry<String, String[]> e : raw.entrySet()) {
+            final int idx = Arrays.binarySearch(this.pool, e.getKey());
+            if (idx < 0) {
+                throw new StepInternalException("Pool missing key during finalization: " + e.getKey());
+            }
+            final String[] vals = e.getValue();
+            final int[] indices = new int[vals.length];
+            for (int i = 0; i < vals.length; i++) {
+                final int vIdx = Arrays.binarySearch(this.pool, vals[i]);
+                if (vIdx < 0) {
+                    throw new StepInternalException("Pool missing value during finalization: " + vals[i]);
+                }
+                indices[i] = vIdx;
+            }
+            this.related[idx] = indices;
+        }
+    }
+
+    @Override
+    public List<String> getRelatedNrsvRefs(final String nrsvOsisRef) {
+        if (nrsvOsisRef == null || nrsvOsisRef.isEmpty()) return Collections.emptyList();
+        final int idx = Arrays.binarySearch(this.pool, nrsvOsisRef);
+        if (idx < 0 || this.related[idx] == null) return Collections.emptyList();
+        final int[] indices = this.related[idx];
+        final List<String> out = new ArrayList<String>(indices.length);
+        for (int i : indices) out.add(this.pool[i]);
+        return out;
+    }
+}

step-core/src/main/java/com/tyndalehouse/step/core/service/search/impl/SearchServiceImpl.java

@@ -78,6 +78,7 @@ public class SearchServiceImpl implements SearchService {
     private VersionResolver versionResolver;
     private LexiconDefinitionService lexiconDefinitionService;
     private JSwordRelatedVersesService relatedVerseService;
+    private SemanticRelatedVersesService semanticRelatedVersesService;
 
     /**
      * @param jswordSearch              the search service
@@ -96,7 +97,8 @@ public SearchServiceImpl(final JSwordSearchService jswordSearch,
                              final EntityManager entityManager,
                              final VersionResolver versionResolver,
                              final LexiconDefinitionService lexiconDefinitionService,
-                             final JSwordRelatedVersesService relatedVerseService) {
+                             final JSwordRelatedVersesService relatedVerseService,
+                             final SemanticRelatedVersesService semanticRelatedVersesService) {
         this.jswordSearch = jswordSearch;
         this.jswordMetadata = jswordMetadata;
         this.versificationService = versificationService;
@@ -106,6 +108,7 @@ public SearchServiceImpl(final JSwordSearchService jswordSearch,
         this.versionResolver = versionResolver;
         this.lexiconDefinitionService = lexiconDefinitionService;
         this.relatedVerseService = relatedVerseService;
+        this.semanticRelatedVersesService = semanticRelatedVersesService;
         this.definitions = entityManager.getReader("definition");
         this.specificForms = entityManager.getReader("specificForm");
         this.timelineEvents = entityManager.getReader("timelineEvent");
@@ -330,6 +333,10 @@ private void enhanceSearchTokens(final String masterVersion, final List<SearchTo
                 final TextSuggestion enhancedTokenInfo = new TextSuggestion();
                 enhancedTokenInfo.setText(st.getToken());
                 st.setEnhancedTokenInfo(enhancedTokenInfo);
+            } else if (SearchToken.RELATED_VERSES_SEMANTIC.equals(st.getTokenType())) {
+                final TextSuggestion enhancedTokenInfo = new TextSuggestion();
+                enhancedTokenInfo.setText(st.getToken());
+                st.setEnhancedTokenInfo(enhancedTokenInfo);
             }
             //nothing to do 
             // for subject searches or 
@@ -391,6 +398,8 @@ private AbstractComplexSearch runCorrectSearch(final List<String> versions, fina
                 addSearch(SearchType.SUBJECT_RELATED, versions, references, st.getToken(), null, individualSearches);
             } else if (SearchToken.RELATED_VERSES.equals(tokenType)) {
                 addSearch(SearchType.RELATED_VERSES, versions, references, st.getToken(), null, individualSearches);
+            } else if (SearchToken.RELATED_VERSES_SEMANTIC.equals(tokenType)) {
+                addSearch(SearchType.RELATED_VERSES_SEMANTIC, versions, references, st.getToken(), null, individualSearches);
             } else if (SearchToken.SYNTAX.equals(tokenType)) {
                 //add a number of searches from the query syntax given...
                 final IndividualSearch[] searches = new SearchQuery(st.getToken(), versions.toArray(new String[versions.size()]), null,
@@ -549,6 +558,10 @@ private SearchResult doSearch(final SearchQuery sq, final String options, final
     }
 
     private SearchType getBestSearchType(final SearchQuery sq) {
+        return normalizeForFrontend(rawBestSearchType(sq));
+    }
+
+    private SearchType rawBestSearchType(final SearchQuery sq) {
         IndividualSearch[] searches = sq.getSearches();
         for (IndividualSearch s : searches) {
             //we never return subject searches if we can avoid it
@@ -565,6 +578,10 @@ private SearchType getBestSearchType(final SearchQuery sq) {
         return searches.length == 1 ? searches[0].getType() : SearchType.TEXT;
     }
 
+    private static SearchType normalizeForFrontend(SearchType t) {
+        return t == SearchType.RELATED_VERSES_SEMANTIC ? SearchType.RELATED_VERSES : t;
+    }
+
     /**
      * Prefers the secondary restriction, if available, over the main range from the text/query syntax
      *
@@ -1038,6 +1055,8 @@ private SearchResult executeOneSearch(final SearchQuery sq, final String options
                 return runMeaningSearch(sq);
             case RELATED_VERSES:
                 return runRelatedVerses(sq);
+            case RELATED_VERSES_SEMANTIC:
+                return runSemanticRelatedVerses(sq);
             default:
                 throw new TranslatedException("search_unknown");
         }
@@ -1083,6 +1102,58 @@ private SearchResult runRelatedVerses(final SearchQuery sq) {
                 this.relatedVerseService.getRelatedVerses(sq.getCurrentSearch().getVersions()[0], sq.getCurrentSearch().getQuery()), ""); // Options from user was not passed to this method
     }
 
+    private SearchResult runSemanticRelatedVerses(final SearchQuery sq) {
+        final IndividualSearch curr = sq.getCurrentSearch();
+        final String version = curr.getVersions()[0];
+        final String userInputRef = curr.getQuery();
+
+        final Versification userV11n;
+        try {
+            userV11n = this.versificationService.getVersificationForVersion(version);
+        } catch (com.tyndalehouse.step.core.exceptions.StepInternalException e) {
+            return emptyRelatedVersesResult(sq);
+        }
+        final Versification nrsv = org.crosswire.jsword.versification.system.Versifications
+                .instance().getVersification(
+                        org.crosswire.jsword.versification.system.SystemNRSV.V11N_NAME);
+
+        final String nrsvOsisRef;
+        try {
+            final Verse userVerse = VerseFactory.fromString(userV11n, userInputRef);
+            if (userVerse == null) return emptyRelatedVersesResult(sq);
+            final VerseKey nrsvKey = VersificationsMapper.instance().mapVerse(userVerse, nrsv);
+            final Iterator<Key> it = nrsvKey.iterator();
+            if (!it.hasNext()) return emptyRelatedVersesResult(sq);
+            nrsvOsisRef = ((Verse) it.next()).getOsisID();
+        } catch (NoSuchVerseException e) {
+            return emptyRelatedVersesResult(sq);
+        }
+
+        final List<String> orderedNrsvRefs =
+                this.semanticRelatedVersesService.getRelatedNrsvRefs(nrsvOsisRef);
+        if (orderedNrsvRefs.isEmpty()) return emptyRelatedVersesResult(sq);
+
+        final Passage userPassage;
+        try {
+            final String joinedNrsvRefs = String.join(" ", orderedNrsvRefs);
+            final Passage nrsvPassage = PassageKeyFactory.instance().getKey(nrsv, joinedNrsvRefs);
+            userPassage = VersificationsMapper.instance().map(nrsvPassage, userV11n);
+        } catch (NoSuchKeyException e) {
+            return emptyRelatedVersesResult(sq);
+        }
+        if (userPassage.getCardinality() == 0) return emptyRelatedVersesResult(sq);
+
+        return this.buildCombinedVerseBasedResults(sq, userPassage, "");
+    }
+
+    private SearchResult emptyRelatedVersesResult(final SearchQuery sq) {
+        final SearchResult r = new SearchResult();
+        r.setResults(Collections.<SearchEntry>emptyList());
+        r.setTotal(0);
+        r.setQuery(sq.getOriginalQuery());
+        return r;
+    }
+
     /**
      * Runs a text search, collapsing the restrictions if need be
      *

step-core/src/main/java/com/tyndalehouse/step/core/utils/ValidateUtils.java

@@ -125,7 +125,7 @@ public static boolean validateInputParm(final String key, final String value) {
             else
                 return false;
         }
-        else if (key.equals("topicref") || key.equals("relatedrefs")) {
+        else if (key.equals("topicref") || key.equals("relatedrefs") || key.equals("relatedrefsSemantic")) {
             if (value.length() > 2000) {
                 System.out.println("XSS kill unexpected reference length: " + value);
                 return false;

step-core/src/test/java/com/tyndalehouse/step/core/service/search/impl/SearchServiceImplTest.java

@@ -110,7 +110,7 @@ private SearchServiceImpl getSearchServiceUnderTest() {
         subjects = new SubjectSearchServiceImpl(entityManager,
                 jswordSearch, meta, module, versificationService);
         return new SearchServiceImpl(jswordSearch, meta, versificationService, subjects, new TimelineServiceImpl(entityManager, jsword), null, entityManager, TestUtils.mockVersionResolver(),
-                mock(LexiconDefinitionServiceImpl.class), null
+                mock(LexiconDefinitionServiceImpl.class), null, null
         );
     }
 }

step-packages/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.tyndalehouse.step</groupId>
         <artifactId>step-mvn</artifactId>
-        <version>26.5.2</version>
+        <version>26.5.3</version>
     </parent>
 
     <groupId>com.tyndalehouse.step</groupId>

step-server/pom.xml

@@ -2,7 +2,7 @@
 	<parent>
 		<groupId>com.tyndalehouse.step</groupId>
 		<artifactId>step-mvn</artifactId>
-		<version>26.5.2</version>
+		<version>26.5.3</version>
 	</parent>
 
 	<modelVersion>4.0.0</modelVersion>

step-test/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.tyndalehouse.step</groupId>
         <artifactId>step-mvn</artifactId>
-        <version>26.5.2</version>
+        <version>26.5.3</version>
     </parent>
 
     <groupId>com.tyndalehouse.step</groupId>

step-tools/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>com.tyndalehouse.step</groupId>
         <artifactId>step-mvn</artifactId>
-        <version>26.5.2</version>
+        <version>26.5.3</version>
     </parent>
 
     <groupId>com.tyndalehouse.step</groupId>

step-war-precompiled/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>step-mvn</artifactId>
         <groupId>com.tyndalehouse.step</groupId>
-        <version>26.5.2</version>
+        <version>26.5.3</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>