feat: auto-detect Kubernetes crd schema

README.md

@@ -41,6 +41,8 @@ The following settings are supported:
 - `yaml.schemas`: Helps you associate schemas with files in a glob pattern
 - `yaml.schemaStore.enable`: When set to true the YAML language server will pull in all available schemas from [JSON Schema Store](https://www.schemastore.org)
 - `yaml.schemaStore.url`: URL of a schema store catalog to use when downloading schemas.
+- `yaml.kubernetesCRDStore.enable`: When set to true the YAML language server will parse Kubernetes CRDs automatically and download them from the [CRD store](https://github.com/datreeio/CRDs-catalog).
+- `yaml.kubernetesCRDStore.url`: URL of a crd store catalog to use when downloading schemas. Defaults to `https://raw.githubusercontent.com/datreeio/CRDs-catalog/main`.
 - `yaml.customTags`: Array of custom tags that the parser will validate against. It has two ways to be used. Either an item in the array is a custom tag such as "!Ref" and it will automatically map !Ref to scalar or you can specify the type of the object !Ref should be e.g. "!Ref sequence". The type of object can be either scalar (for strings and booleans), sequence (for arrays), map (for objects).
 - `yaml.maxItemsComputed`: The maximum number of outline symbols and folding regions computed (limited for performance reasons).
 - `[yaml].editor.tabSize`: the number of spaces to use when autocompleting. Takes priority over editor.tabSize.

src/languageserver/handlers/settingsHandlers.ts

@@ -84,6 +84,14 @@ export class SettingsHandler {
           this.yamlSettings.schemaStoreUrl = settings.yaml.schemaStore.url;
         }
       }
+
+      if (settings.yaml.kubernetesCRDStore) {
+        this.yamlSettings.kubernetesCRDStoreEnabled = settings.yaml.kubernetesCRDStore.enable;
+        if (settings.yaml.kubernetesCRDStore.url?.length !== 0) {
+          this.yamlSettings.kubernetesCRDStoreUrl = settings.yaml.kubernetesCRDStore.url;
+        }
+      }
+
       if (settings.files?.associations) {
         for (const [ext, languageId] of Object.entries(settings.files.associations)) {
           if (languageId === 'yaml') {

src/languageservice/services/crdUtil.ts

@@ -0,0 +1,92 @@
+import { SingleYAMLDocument } from '../parser/yamlParser07';
+import { JSONDocument } from '../parser/jsonParser07';
+
+import { ResolvedSchema } from 'vscode-json-languageservice/lib/umd/services/jsonSchemaService';
+import { JSONSchema } from '../jsonSchema';
+
+/**
+ * Retrieve schema by auto-detecting the Kubernetes GroupVersionKind (GVK) from the document.
+ * If there is no definition for the GVK in the main kubernetes schema,
+ * the schema is then retrieved from the CRD catalog.
+ * Public for testing purpose, not part of the API.
+ * @param doc
+ * @param crdCatalogURI The URL of the CRD catalog to retrieve the schema from
+ * @param kubernetesSchema The main kubernetes schema, if it includes a definition for the GVK it will be used
+ */
+export function autoDetectKubernetesSchemaFromDocument(
+  doc: SingleYAMLDocument | JSONDocument,
+  crdCatalogURI: string,
+  kubernetesSchema: ResolvedSchema
+): string | undefined {
+  const res = getGroupVersionKindFromDocument(doc);
+  if (!res) {
+    return undefined;
+  }
+  const { group, version, kind } = res;
+  if (!group || !version || !kind) {
+    return undefined;
+  }
+
+  const k8sSchema: JSONSchema = kubernetesSchema.schema;
+  const kubernetesBuildIns: string[] = (k8sSchema.oneOf || [])
+    .map((s) => {
+      if (typeof s === 'boolean') {
+        return undefined;
+      }
+      return s._$ref || s.$ref;
+    })
+    .filter((ref) => ref)
+    .map((ref) => ref.replace('_definitions.json#/definitions/', '').toLowerCase());
+  const groupWithoutK8sIO = group.replace('.k8s.io', '');
+  const k8sTypeName = `io.k8s.api.${groupWithoutK8sIO.toLowerCase()}.${version.toLowerCase()}.${kind.toLowerCase()}`;
+
+  if (kubernetesBuildIns.includes(k8sTypeName)) {
+    return undefined;
+  }
+
+  if (k8sTypeName.includes('openshift.io')) {
+    return `${crdCatalogURI}/openshift/v4.15-strict/${kind.toLowerCase()}_${group.toLowerCase()}_${version.toLowerCase()}.json`;
+  }
+
+  const schemaURL = `${crdCatalogURI}/${group.toLowerCase()}/${kind.toLowerCase()}_${version.toLowerCase()}.json`;
+  return schemaURL;
+}
+
+/**
+ * Retrieve the group, version and kind from the document.
+ * Public for testing purpose, not part of the API.
+ * @param doc
+ */
+export function getGroupVersionKindFromDocument(
+  doc: SingleYAMLDocument | JSONDocument
+): { group: string; version: string; kind: string } | undefined {
+  if (doc instanceof SingleYAMLDocument) {
+    try {
+      const rootJSON = doc.root.internalNode.toJSON();
+      if (!rootJSON) {
+        return undefined;
+      }
+
+      const groupVersion = rootJSON['apiVersion'];
+      if (!groupVersion) {
+        return undefined;
+      }
+
+      const [group, version] = groupVersion.split('/');
+      if (!group || !version) {
+        return undefined;
+      }
+
+      const kind = rootJSON['kind'];
+      if (!kind) {
+        return undefined;
+      }
+
+      return { group, version, kind };
+    } catch (error) {
+      console.error('Error parsing YAML document:', error);
+      return undefined;
+    }
+  }
+  return undefined;
+}

src/languageservice/services/yamlSchemaService.ts

@@ -6,6 +6,7 @@
 
 import { JSONSchema, JSONSchemaMap, JSONSchemaRef } from '../jsonSchema';
 import { SchemaPriority, SchemaRequestService, WorkspaceContextService } from '../yamlLanguageService';
+import { SettingsState } from '../../yamlSettings';
 import {
   UnresolvedSchema,
   ResolvedSchema,
@@ -31,6 +32,8 @@
 import Ajv4 from 'ajv-draft-04';
 import Ajv2019 from 'ajv/dist/2019';
 import Ajv2020 from 'ajv/dist/2020';
+import { autoDetectKubernetesSchemaFromDocument } from './crdUtil';
+import { CRD_CATALOG_URL, KUBERNETES_SCHEMA_URL } from '../utils/schemaUrls';
 
 const ajv4 = new Ajv4({ allErrors: true });
 const ajv7 = new Ajv({ allErrors: true });
@@ -120,19 +123,22 @@
   private filePatternAssociations: JSONSchemaService.FilePatternAssociation[];
   private contextService: WorkspaceContextService;
   private requestService: SchemaRequestService;
+  private yamlSettings: SettingsState;
   public schemaPriorityMapping: Map<string, Set<SchemaPriority>>;
 
   private schemaUriToNameAndDescription = new Map<string, SchemaStoreSchema>();
 
   constructor(
     requestService: SchemaRequestService,
     contextService?: WorkspaceContextService,
-    promiseConstructor?: PromiseConstructor
+    promiseConstructor?: PromiseConstructor,
+    yamlSettings?: SettingsState
   ) {
     super(requestService, contextService, promiseConstructor);
     this.customSchemaProvider = undefined;
     this.requestService = requestService;
     this.schemaPriorityMapping = new Map();
+    this.yamlSettings = yamlSettings;
   }
 
   registerCustomSchemaProvider(customSchemaProvider: CustomSchemaProvider): void {
@@ -396,7 +402,7 @@
           schema.schema.url = schemaHandle.url;
         }
 
         if (
           schema.schema &&
           schema.schema.schemaSequence &&
           schema.schema.schemaSequence[(<SingleYAMLDocument>doc).currentDocIndex]
@@ -408,16 +414,35 @@
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const resolveSchema = (): any => {
+    const resolveSchema = async (): Promise<any> => {
       const seen: { [schemaId: string]: boolean } = Object.create(null);
       const schemas: string[] = [];
+      let k8sAllSchema: ResolvedSchema = undefined;
 
       for (const entry of this.filePatternAssociations) {
         if (entry.matchesPattern(resource)) {
           for (const schemaId of entry.getURIs()) {
             if (!seen[schemaId]) {
-              schemas.push(schemaId);
-              seen[schemaId] = true;
+              if (this.yamlSettings?.kubernetesCRDStoreEnabled && schemaId === KUBERNETES_SCHEMA_URL) {
+                if (!k8sAllSchema) {
+                  k8sAllSchema = await this.getResolvedSchema(KUBERNETES_SCHEMA_URL);
+                }
+                const kubeSchema = autoDetectKubernetesSchemaFromDocument(
+                  doc,
+                  this.yamlSettings.kubernetesCRDStoreUrl ?? CRD_CATALOG_URL,
+                  k8sAllSchema
+                );
+                if (kubeSchema) {
+                  schemas.push(kubeSchema);
+                  seen[schemaId] = true;
+                } else {
+                  schemas.push(schemaId);
+                  seen[schemaId] = true;
+                }
+              } else {
+                schemas.push(schemaId);
+                seen[schemaId] = true;
+              }
             }
           }
         }
@@ -435,6 +460,7 @@
     if (modelineSchema) {
       return resolveSchemaForResource([modelineSchema]);
     }
+
     if (this.customSchemaProvider) {
       return this.customSchemaProvider(resource)
         .then((schemaUri) => {
@@ -477,9 +503,8 @@
             return resolveSchema();
           }
         );
-    } else {
-      return resolveSchema();
     }
+    return resolveSchema();
   }
 
   // Set the priority of a schema in the schema service

src/languageservice/utils/schemaUrls.ts

@@ -8,6 +8,7 @@ import { isRelativePath, relativeToAbsolutePath } from './paths';
 export const KUBERNETES_SCHEMA_URL =
   'https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.32.1-standalone-strict/all.json';
 export const JSON_SCHEMASTORE_URL = 'https://www.schemastore.org/api/json/catalog.json';
+export const CRD_CATALOG_URL = 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main';
 
 export function checkSchemaURI(
   workspaceFolders: WorkspaceFolder[],

src/languageservice/yamlLanguageService.ts

@@ -197,7 +197,7 @@ export function getLanguageService(params: {
   yamlSettings?: SettingsState;
   clientCapabilities?: ClientCapabilities;
 }): LanguageService {
-  const schemaService = new YAMLSchemaService(params.schemaRequestService, params.workspaceContext);
+  const schemaService = new YAMLSchemaService(params.schemaRequestService, params.workspaceContext, null, params.yamlSettings);
   const completer = new YamlCompletion(schemaService, params.clientCapabilities, yamlDocumentsCache, params.telemetry);
   const hover = new YAMLHover(schemaService, params.telemetry);
   const yamlDocumentSymbols = new YAMLDocumentSymbols(schemaService, params.telemetry);

src/yamlSettings.ts

@@ -4,7 +4,7 @@ import { ISchemaAssociations } from './requestTypes';
 import { URI } from 'vscode-uri';
 import { JSONSchema } from './languageservice/jsonSchema';
 import { TextDocument } from 'vscode-languageserver-textdocument';
-import { JSON_SCHEMASTORE_URL } from './languageservice/utils/schemaUrls';
+import { CRD_CATALOG_URL, JSON_SCHEMASTORE_URL } from './languageservice/utils/schemaUrls';
 import { YamlVersion } from './languageservice/parser/yamlParser07';
 
 // Client settings interface to grab settings relevant for the language server
@@ -20,6 +20,10 @@ export interface Settings {
       url: string;
       enable: boolean;
     };
+    kubernetesCRDStore: {
+      url: string;
+      enable: boolean;
+    };
     disableDefaultProperties: boolean;
     disableAdditionalProperties: boolean;
     suggest: {
@@ -78,6 +82,8 @@ export class SettingsState {
   customTags = [];
   schemaStoreEnabled = true;
   schemaStoreUrl = JSON_SCHEMASTORE_URL;
+  kubernetesCRDStoreEnabled = true;
+  kubernetesCRDStoreUrl = CRD_CATALOG_URL;
   indentation: string | undefined = undefined;
   disableAdditionalProperties = false;
   disableDefaultProperties = false;

test/schema.test.ts

@@ -14,6 +14,7 @@ import { LanguageService, SchemaPriority } from '../src';
 import { MarkupContent, Position } from 'vscode-languageserver-types';
 import { LineCounter } from 'yaml';
 import { getSchemaFromModeline } from '../src/languageservice/services/modelineUtil';
+import { getGroupVersionKindFromDocument } from '../src/languageservice/services/crdUtil';
 
 const requestServiceMock = function (uri: string): Promise<string> {
   return Promise.reject<string>(`Resource ${uri} not found.`);
@@ -701,6 +702,70 @@ describe('JSON Schema', () => {
     });
   });
 
+  describe('Test getGroupVersionKindFromDocument', function () {
+    it('builtin kubernetes resource group should not get resolved', async () => {
+      checkReturnGroupVersionKind('apiVersion: v1\nkind: Pod', true, undefined, 'v1', 'Pod');
+    });
+
+    it('builtin kubernetes resource with complex apiVersion should get resolved ', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: admissionregistration.k8s.io/v1\nkind: MutatingWebhook',
+        false,
+        'admissionregistration.k8s.io',
+        'v1',
+        'MutatingWebhook'
+      );
+    });
+
+    it('custom argo application CRD should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: argoproj.io/v1alpha1\nkind: Application',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    it('custom argo application CRD with whitespace should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: argoproj.io/v1alpha1\nkind: Application ',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    it('custom argo application CRD with other fields should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'someOtherVal: test\napiVersion: argoproj.io/v1alpha1\nkind: Application\nmetadata:\n  name: my-app',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    function checkReturnGroupVersionKind(
+      content: string,
+      error: boolean,
+      expectedGroup: string,
+      expectedVersion: string,
+      expectedKind: string
+    ): void {
+      const yamlDoc = parser.parse(content);
+      const res = getGroupVersionKindFromDocument(yamlDoc.documents[0]);
+      if (error) {
+        assert.strictEqual(res, undefined);
+      } else {
+        assert.strictEqual(res.group, expectedGroup);
+        assert.strictEqual(res.version, expectedVersion);
+        assert.strictEqual(res.kind, expectedKind);
+      }
+    }
+  });
+
   describe('Test getSchemaFromModeline', function () {
     it('simple case', async () => {
       checkReturnSchemaUrl('# yaml-language-server: $schema=expectedUrl', 'expectedUrl');