Add support for gherkin with markdown

CHANGELOG.md

@@ -9,9 +9,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 - Add Scala support ([#237](https://github.com/cucumber/language-service/issues/237))
+- Support for Gherkin-in-Markdown (`.feature.md`) documents: `parseGherkinDocument` and all LSP service entry points (`getGherkinCompletionItems`, `getGherkinDiagnostics`, `getGherkinDocumentFeatureSymbol`, `getGherkinFormattingEdits`, `getGherkinSemanticTokens`, `getStepDefinitionLocationLinks`) now accept an optional `uri` argument; when it ends with `.feature.md`, the Markdown token matcher is used. Exposed a new `isMarkdownFeatureUri` helper.
 
 ### Fixed
 - Missing `parameter:cucumber` token for Scenario Outline ([#246](https://github.com/cucumber/language-service/issues/246))
+- `getGherkinSemanticTokens` no longer throws on incomplete AST nodes produced by the Markdown matcher (missing keyword/name or column); the affected token is skipped instead.
+
+### Changed
+- `getGherkinFormattingEdits` returns no edits for `.feature.md` documents, since `pretty()` would rewrite them as classic Gherkin and destroy the Markdown layout.
 
 ## [1.7.0] - 2025-05-18
 ### Added

src/gherkin/index.ts

@@ -1,2 +1,3 @@
 export * from './extractStepTexts.js'
+export * from './isMarkdownFeatureUri.js'
 export * from './parseGherkinDocument.js'

src/gherkin/isMarkdownFeatureUri.ts

@@ -0,0 +1,10 @@
+/**
+ * Returns true when the given URI designates a Gherkin-in-Markdown (`.feature.md`) file.
+ *
+ * Tolerates `file://` URIs as well as trailing query strings (`?…`) or fragments (`#…`),
+ * so both plain paths (`features/foo.feature.md`) and LSP-style URIs
+ * (`file:///abs/features/foo.feature.md?v=1`) are recognized.
+ */
+export function isMarkdownFeatureUri(uri?: string): boolean {
+  return !!uri && /\.feature\.md(?:[?#]|$)/.test(uri)
+}

src/gherkin/parseGherkinDocument.ts

@@ -1,6 +1,14 @@
-import { AstBuilder, Errors, GherkinClassicTokenMatcher, Parser } from '@cucumber/gherkin'
+import {
+  AstBuilder,
+  Errors,
+  GherkinClassicTokenMatcher,
+  GherkinInMarkdownTokenMatcher,
+  Parser,
+} from '@cucumber/gherkin'
 import { GherkinDocument, IdGenerator } from '@cucumber/messages'
 
+import { isMarkdownFeatureUri } from './isMarkdownFeatureUri.js'
+
 const uuidFn = IdGenerator.uuid()
 
 export type ParseResult = {
@@ -10,10 +18,15 @@ export type ParseResult = {
 
 /**
  * Incrementally parses a Gherkin Document, allowing some syntax errors to occur.
+ *
+ * When `uri` ends with `.feature.md`, the document is parsed as Gherkin-in-Markdown;
+ * otherwise (including when `uri` is omitted) classic Gherkin is used.
  */
-export function parseGherkinDocument(gherkinSource: string): ParseResult {
+export function parseGherkinDocument(gherkinSource: string, uri?: string): ParseResult {
   const builder = new AstBuilder(uuidFn)
-  const matcher = new GherkinClassicTokenMatcher()
+  const matcher = isMarkdownFeatureUri(uri)
+    ? new GherkinInMarkdownTokenMatcher()
+    : new GherkinClassicTokenMatcher()
   const parser = new Parser(builder, matcher)
   try {
     return {

src/service/getGherkinCompletionItems.ts

@@ -13,9 +13,10 @@ import { lspCompletionSnippet } from './snippet/lspCompletionSnippet.js'
 export function getGherkinCompletionItems(
   gherkinSource: string,
   position: Position,
-  index: Index
+  index: Index,
+  uri?: string
 ): readonly CompletionItem[] {
-  const stepRange = getStepRange(gherkinSource, position)
+  const stepRange = getStepRange(gherkinSource, position, uri)
   if (!stepRange) return []
   const suggestions = index(stepRange.stepText)
   // https://github.com/microsoft/language-server-protocol/issues/898#issuecomment-593968008

src/service/getGherkinDiagnostics.ts

@@ -10,10 +10,11 @@ import { CONTAINS_PARAMETERS, diagnosticCodeUndefinedStep } from './constants.js
 // https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#diagnostic
 export function getGherkinDiagnostics(
   gherkinSource: string,
-  expressions: readonly Expression[]
+  expressions: readonly Expression[],
+  uri?: string
 ): Diagnostic[] {
   const lines = gherkinSource.split(/\r?\n/)
-  const { gherkinDocument, error } = parseGherkinDocument(gherkinSource)
+  const { gherkinDocument, error } = parseGherkinDocument(gherkinSource, uri)
   const diagnostics: Diagnostic[] = []
   const errors: Error[] =
     error instanceof Errors.CompositeParserException ? error.errors : error ? [error] : []

src/service/getGherkinDocumentFeatureSymbol.ts

@@ -7,8 +7,11 @@ import { parseGherkinDocument } from '../gherkin/parseGherkinDocument.js'
 type SymbolsKey = 'feature' | 'parent'
 type Symbols = Partial<Record<SymbolsKey, DocumentSymbol>>
 
-export function getGherkinDocumentFeatureSymbol(gherkinSource: string): DocumentSymbol | null {
-  const { gherkinDocument } = parseGherkinDocument(gherkinSource)
+export function getGherkinDocumentFeatureSymbol(
+  gherkinSource: string,
+  uri?: string
+): DocumentSymbol | null {
+  const { gherkinDocument } = parseGherkinDocument(gherkinSource, uri)
   if (!gherkinDocument) {
     return null
   }

src/service/getGherkinFormattingEdits.ts

@@ -1,12 +1,21 @@
 import { pretty } from '@cucumber/gherkin-utils'
 import { TextEdit } from 'vscode-languageserver-types'
 
+import { isMarkdownFeatureUri } from '../gherkin/isMarkdownFeatureUri.js'
 import { parseGherkinDocument } from '../gherkin/parseGherkinDocument.js'
 
 // https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocument_formatting
-export function getGherkinFormattingEdits(gherkinSource: string): TextEdit[] {
-  const { gherkinDocument } = parseGherkinDocument(gherkinSource)
+export function getGherkinFormattingEdits(gherkinSource: string, uri?: string): TextEdit[] {
+  const { gherkinDocument } = parseGherkinDocument(gherkinSource, uri)
   if (gherkinDocument === undefined) return []
+
+  // pretty() emits classic Gherkin and would destroy the Markdown layout of a .feature.md
+  // document (headers, fenced steps, etc.). Until gherkin-utils gains a Markdown-aware
+  // formatter, skip formatting for MDG files.
+  if (isMarkdownFeatureUri(uri)) {
+    return []
+  }
+
   const newText = pretty(gherkinDocument)
   const lines = gherkinSource.split(/\r?\n/)
   const line = lines.length - 1

src/service/getGherkinSemanticTokens.ts

@@ -30,9 +30,10 @@ const indexByType = Object.fromEntries(semanticTokenTypes.map((type, index) => [
 // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens
 export function getGherkinSemanticTokens(
   gherkinSource: string,
-  expressions: readonly Expression[]
+  expressions: readonly Expression[],
+  uri?: string
 ): SemanticTokens {
-  const { gherkinDocument } = parseGherkinDocument(gherkinSource)
+  const { gherkinDocument } = parseGherkinDocument(gherkinSource, uri)
   if (!gherkinDocument) {
     return {
       data: [],
@@ -42,24 +43,29 @@ export function getGherkinSemanticTokens(
 
   function makeLocationToken(
     location: messages.Location,
-    token: string,
+    token: string | undefined,
     type: SemanticTokenTypes,
     data: TokenLines
   ) {
+    // The MDG parser in @cucumber/gherkin can produce nodes with missing
+    // keyword/name fields when the markdown structure does not match its
+    // expectations (e.g. tags placed before the `# Feature:` heading).
+    // Skip emitting a token rather than crashing — the rest of the document
+    // still gets highlighted.
+    if (token === undefined || location.column === undefined) return data
     const lineNumber = location.line - 1
-    if (location.column === undefined)
-      throw new Error(`Incomplete location: ${JSON.stringify(location)}`)
     const character = location.column - 1
     return makeToken(lineNumber, character, token, type, data)
   }
 
   function makeToken(
     lineNumber: number,
     character: number,
-    token: string,
+    token: string | undefined,
     type: SemanticTokenTypes,
     data: TokenLines
   ) {
+    if (token === undefined) return data
     const copy = [...data]
     copy[lineNumber] = (copy[lineNumber] ?? []).concat({
       typeIndex: indexByType[type],

src/service/getStepDefinitionLocationLinks.ts

@@ -6,9 +6,10 @@ import { getStepRange } from './helpers.js'
 export function getStepDefinitionLocationLinks(
   gherkinSource: string,
   position: Position,
-  expressionLinks: readonly ExpressionLink[]
+  expressionLinks: readonly ExpressionLink[],
+  uri?: string
 ): LocationLink[] {
-  const stepRange = getStepRange(gherkinSource, position)
+  const stepRange = getStepRange(gherkinSource, position, uri)
   if (!stepRange) return []
 
   const locationLinks: LocationLink[] = []

src/service/helpers.ts

@@ -9,8 +9,12 @@ export type StepRange = {
   range: Range
 }
 
-export function getStepRange(gherkinSource: string, position: Position): StepRange | undefined {
-  const { gherkinDocument } = parseGherkinDocument(gherkinSource)
+export function getStepRange(
+  gherkinSource: string,
+  position: Position,
+  uri?: string
+): StepRange | undefined {
+  const { gherkinDocument } = parseGherkinDocument(gherkinSource, uri)
   if (!gherkinDocument) {
     return undefined
   }

test/gherkin/isMarkdownFeatureUri.test.ts

@@ -0,0 +1,34 @@
+import assert from 'assert'
+
+import { isMarkdownFeatureUri } from '../../src/gherkin/isMarkdownFeatureUri.js'
+
+describe('isMarkdownFeatureUri', () => {
+  it('returns true for a .feature.md path', () => {
+    assert.strictEqual(isMarkdownFeatureUri('features/login.feature.md'), true)
+  })
+
+  it('returns true for a file:// URI pointing at .feature.md', () => {
+    assert.strictEqual(isMarkdownFeatureUri('file:///abs/features/login.feature.md'), true)
+  })
+
+  it('tolerates a trailing query string', () => {
+    assert.strictEqual(isMarkdownFeatureUri('file:///x.feature.md?v=1'), true)
+  })
+
+  it('tolerates a trailing fragment', () => {
+    assert.strictEqual(isMarkdownFeatureUri('file:///x.feature.md#L10'), true)
+  })
+
+  it('returns false for a classic .feature file', () => {
+    assert.strictEqual(isMarkdownFeatureUri('features/login.feature'), false)
+  })
+
+  it('returns false for an unrelated .md file', () => {
+    assert.strictEqual(isMarkdownFeatureUri('docs/README.md'), false)
+  })
+
+  it('returns false when uri is undefined or empty', () => {
+    assert.strictEqual(isMarkdownFeatureUri(undefined), false)
+    assert.strictEqual(isMarkdownFeatureUri(''), false)
+  })
+})

test/gherkin/parseGherkinDocument.test.ts

@@ -14,4 +14,41 @@ describe('parseGherkinDocument', () => {
       'Parser errors:\n(3:0): unexpected end of file, expected: #TagLine, #RuleLine, #Comment, #Empty'
     )
   })
+
+  it('parses a Markdown-with-Gherkin document when uri ends with .feature.md', () => {
+    const source = `# Feature: Login
+
+Description paragraph that is just prose.
+
+## Scenario: Successful login
+* Given the user is on the login page
+* When the user enters valid credentials
+* Then the user should be logged in
+
+## Scenario: Failed login
+* Given the user is on the login page
+* When the user enters invalid credentials
+* Then the user should see an error message
+`
+    const { gherkinDocument, error } = parseGherkinDocument(source, 'file:///x.feature.md')
+    assert.strictEqual(error, undefined)
+    assert.strictEqual(gherkinDocument!.feature!.name, 'Login')
+    assert.strictEqual(gherkinDocument!.feature!.children!.length, 2)
+    assert.deepStrictEqual(
+      gherkinDocument!.feature!.children!.map((c) => c.scenario?.name),
+      ['Successful login', 'Failed login']
+    )
+  })
+
+  it('falls back to the classic matcher when uri is omitted', () => {
+    // This source is valid classic Gherkin but would be garbage in MDG mode
+    // (no `#` headers) — it must parse as classic because no uri is given.
+    const source = `Feature: Hello
+  Scenario: Hi
+    Given something
+`
+    const { gherkinDocument, error } = parseGherkinDocument(source)
+    assert.strictEqual(error, undefined)
+    assert.strictEqual(gherkinDocument!.feature!.name, 'Hello')
+  })
 })

test/service/getGherkinDiagnostics.test.ts

@@ -204,6 +204,41 @@ describe('getGherkinDiagnostics', () => {
     assert.deepStrictEqual(diagnostics, expectedDiagnostics)
   })
 
+  it('returns no diagnostics for a valid .feature.md document when uri is provided', () => {
+    const mdgSource = `# Feature: Login
+
+## Scenario: Successful login
+* Given the user is on the login page
+* When the user enters valid credentials
+* Then the user should be logged in
+`
+    const registry = new ParameterTypeRegistry()
+    const expressions = [
+      new CucumberExpression('the user is on the login page', registry),
+      new CucumberExpression('the user enters valid credentials', registry),
+      new CucumberExpression('the user should be logged in', registry),
+    ]
+    const diagnostics = getGherkinDiagnostics(mdgSource, expressions, 'file:///x.feature.md')
+    assert.deepStrictEqual(diagnostics, [])
+  })
+
+  it('returns undefined-step diagnostic in a .feature.md document', () => {
+    const mdgSource = `# Feature: Login
+
+## Scenario: Hi
+* Given a defined step
+* And an undefined step
+`
+    const diagnostics = getGherkinDiagnostics(
+      mdgSource,
+      [new CucumberExpression('a defined step', new ParameterTypeRegistry())],
+      'file:///x.feature.md'
+    )
+    assert.strictEqual(diagnostics.length, 1)
+    assert.strictEqual(diagnostics[0].severity, DiagnosticSeverity.Warning)
+    assert.strictEqual(diagnostics[0].message, 'Undefined step: an undefined step')
+  })
+
   it('returns diagnostic for incomplete docstring', () => {
     const diagnostics = getGherkinDiagnostics(
       `Feature: Hello

test/service/getGherkinDocumentFeatureSymbol.test.ts

@@ -222,4 +222,19 @@ describe('getGherkinDocumentFeatureSymbol', () => {
 
     assert.deepStrictEqual(symbol, expected)
   })
+
+  it('creates a document symbol from a .feature.md document', () => {
+    const mdgSource = `# Feature: Login
+
+## Scenario: Successful login
+* Given the user is on the login page
+`
+    const symbol = getGherkinDocumentFeatureSymbol(mdgSource, 'file:///x.feature.md')
+    assert.ok(symbol, 'expected a symbol')
+    assert.strictEqual(symbol!.name, 'Feature: Login')
+    assert.strictEqual(symbol!.kind, SymbolKind.File)
+    assert.strictEqual(symbol!.children!.length, 1)
+    assert.strictEqual(symbol!.children![0].name, 'Scenario: Successful login')
+    assert.strictEqual(symbol!.children![0].kind, SymbolKind.Event)
+  })
 })

test/service/getGherkinFormattingEdits.test.ts

@@ -4,6 +4,15 @@ import { TextEdit } from 'vscode-languageserver-types'
 import { getGherkinFormattingEdits } from '../../src/service/getGherkinFormattingEdits.js'
 
 describe('getGherkinFormattingEdits', () => {
+  it('returns no edits for a .feature.md document (Markdown layout is preserved)', () => {
+    const mdgSource = `# Feature: Login
+
+## Scenario: Successful login
+* Given the user is on the login page
+`
+    assert.deepStrictEqual(getGherkinFormattingEdits(mdgSource, 'file:///x.feature.md'), [])
+  })
+
   it('returns text edits that prettifies a Gherkin document', () => {
     const gherkinSource = `Feature: Hello
 Scenario: World

test/service/getGherkinSemanticTokens.test.ts

@@ -121,6 +121,23 @@ Feature: making drinks
     assert.deepStrictEqual(actual, expected)
   })
 
+  it('emits keyword tokens for a .feature.md document', () => {
+    const mdgSource = `# Feature: Login
+
+## Scenario: Successful login
+* Given the user is on the login page
+* When the user enters valid credentials
+* Then the user should be logged in
+`
+    const semanticTokens = getGherkinSemanticTokens(mdgSource, [], 'file:///x.feature.md')
+    const actual = tokenize(mdgSource, semanticTokens.data)
+    const keywords = actual
+      .filter(([, type]) => type === SemanticTokenTypes.keyword)
+      .map(([text]) => text.trim())
+    // The MDG matcher normalizes `*` bullets to the proper Given/When/Then keyword.
+    assert.deepStrictEqual(keywords, ['Feature', 'Scenario', 'Given', 'When', 'Then'])
+  })
+
   it('applies parameter token for scenario outline', () => {
     const gherkinSource = `
 Feature: a