Support S3 metadata

Signed-off-by: Denis Jannot <denis.jannot@solo.io>

Author: djannot

README.md

@@ -229,6 +229,8 @@ Configuration is managed through two files:
         *   `encoding`: (Optional) Text file encoding (defaults to `'utf8'`). Does not apply to binary files (PDF, DOC, DOCX).
         *   `url_rewrite_prefix`: (Optional) URL prefix to rewrite `s3://` URLs (e.g., `'https://docs.example.com'`).
 
+        **S3 user metadata resolution:** The `product_name` and `version` fields support a `metadata(...)` syntax to dynamically resolve values from S3 object user metadata. For example, `product_name: 'metadata(x-amz-meta-product-name)'` will set `product_name` to the value of the `x-amz-meta-product-name` user metadata on each S3 object. If the metadata key doesn't exist on an object, an empty string is used. Literal values (without the `metadata(...)` wrapper) work as before.
+
         Authentication uses the AWS SDK default credential chain: environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`), `~/.aws/credentials`, IAM roles, etc.
 
         Incremental sync tracks object `LastModified` timestamps so only new or updated objects are processed on subsequent runs. Deleted objects are automatically cleaned up.
@@ -365,7 +367,7 @@ Configuration is managed through two files:
       
       # S3 bucket source example
       - type: 's3'
-        product_name: 'my-docs'
+        product_name: 'metadata(x-amz-meta-product-name)'
         version: 'latest'
         bucket: 'my-documentation-bucket'
         prefix: 'docs/v2/'

doc2vec.ts

@@ -851,7 +851,15 @@ export class Doc2Vec {
                     fileUrl = `s3://${config.bucket}/${obj.key}`;
                 }
 
-                const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
+                // Resolve metadata(...) references in product_name and version
+                const s3Meta = getResponse.Metadata || {};
+                const resolvedConfig = {
+                    ...config,
+                    product_name: this.resolveS3MetadataValue(config.product_name, s3Meta),
+                    version: this.resolveS3MetadataValue(config.version, s3Meta),
+                };
+
+                const chunks = await this.contentProcessor.chunkMarkdown(content, resolvedConfig, fileUrl);
                 logger.info(`Created ${chunks.length} chunks for ${obj.key}`);
 
                 await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
@@ -901,6 +909,21 @@ export class Doc2Vec {
         logger.info(`Finished processing S3 bucket: ${config.bucket}`);
     }
 
+    /**
+     * Resolves a config value that may use the metadata(...) syntax.
+     * e.g. "metadata(x-amz-meta-product-name)" looks up "product-name" in the S3 object's user metadata.
+     * Returns the original value if no metadata(...) pattern is found.
+     * Returns empty string if the referenced metadata key doesn't exist on the object.
+     */
+    private resolveS3MetadataValue(configValue: string, s3Metadata: Record<string, string>): string {
+        const match = configValue.match(/^metadata\((.+)\)$/);
+        if (!match) return configValue;
+        const metaKey = match[1];
+        // AWS SDK returns user metadata keys without the x-amz-meta- prefix
+        const lookupKey = metaKey.replace(/^x-amz-meta-/, '');
+        return s3Metadata[lookupKey] ?? '';
+    }
+
     private async processCodeSource(config: CodeSourceConfig, parentLogger: Logger): Promise<void> {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for code source (${config.source})`);

package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.8.0",
+    "version": "2.9.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",