Handle nested null arrays correctly

document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java

@@ -1915,6 +1915,26 @@ void testNotContainsAndUnnestFilters(String dataStoreName) throws IOException {
         dataStoreName, iterator, "query/unwind_not_contains_filter_response.json", 2);
   }
 
+  @ParameterizedTest
+  @ArgumentsSource(PostgresProvider.class)
+  public void testUnnestTopLevelArrayWithNullValue(String dataStoreName) throws IOException {
+    Collection collection = getFlatCollection(dataStoreName);
+
+    Query query =
+        Query.builder()
+            .addSelection(IdentifierExpression.of("item"))
+            .addSelection(IdentifierExpression.of("categoryTags"))
+            .addFromClause(UnnestExpression.of(IdentifierExpression.of("categoryTags"), true))
+            .setFilter(
+                RelationalExpression.of(
+                    IdentifierExpression.of("item"), EQ, ConstantExpression.of("Bottle")))
+            .build();
+
+    Iterator<Document> iterator = collection.aggregate(query);
+    assertDocsAndSizeEqualWithoutOrder(
+        dataStoreName, iterator, "query/unnest_null_top_level_array_response.json", 1);
+  }
+
   @ParameterizedTest
   @ArgumentsSource(AllProvider.class)
   public void testQueryV1DistinctCountWithSortingSpecs(String dataStoreName) throws IOException {
@@ -5575,6 +5595,49 @@ void testNotInOnUnnestedArray(String dataStoreName) throws Exception {
       assertEquals(12, count, "Should return unnested locations not matching the filter");
     }
 
+    @ParameterizedTest
+    @ArgumentsSource(PostgresProvider.class)
+    public void testUnnestNestedArrayWithNullValue(String dataStoreName) throws IOException {
+      Collection collection = getFlatCollection(dataStoreName);
+
+      Query query =
+          Query.builder()
+              .addSelection(IdentifierExpression.of("item"))
+              .addFromClause(
+                  UnnestExpression.of(JsonIdentifierExpression.of("props", "source-loc"), true))
+              .setFilter(
+                  RelationalExpression.of(
+                      IdentifierExpression.of("_id"), EQ, ConstantExpression.of(1)))
+              .build();
+
+      Iterator<Document> iterator = collection.aggregate(query);
+      assertDocsAndSizeEqualWithoutOrder(
+          dataStoreName, iterator, "query/unnest_null_nested_array_response.json", 2);
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(PostgresProvider.class)
+    public void testArrayFilterAnyWithJsonNullArray(String dataStoreName) {
+      Collection collection = getFlatCollection(dataStoreName);
+
+      Query query =
+          Query.builder()
+              .addSelection(IdentifierExpression.of("item"))
+              .setFilter(
+                  ArrayRelationalFilterExpression.builder()
+                      .operator(ArrayOperator.ANY)
+                      .filter(
+                          RelationalExpression.of(
+                              JsonIdentifierExpression.of("props", "colors"),
+                              EQ,
+                              ConstantExpression.of("Blue")))
+                      .build())
+              .build();
+
+      long count = collection.count(query);
+      assertEquals(2, count, "Should find 2 items with 'Blue' color");
+    }
+
     @ParameterizedTest
     @ArgumentsSource(PostgresProvider.class)
     void testExistsOnArrays(String dataStoreName) throws JsonProcessingException {

document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json

@@ -6,7 +6,7 @@
     "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z', false,\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\n'{1, 2}',\n'{5.0, 10.0}',\n'{true, true}'\n)",
     "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z', true,\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"product-code\": \"SOAP-LIF-005\", \"source-loc\": [\"warehouse-C\"], \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{3, 6, 9}',\n'{7.5}',\n'{false}'\n)",
     "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z', true,\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\n'{20, 30}',\n'{6.0, 8.0}',\n'{true, false}'\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z', false,\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"product-code\": null, \"source-loc\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{10}',\n'{3.0}',\n'{false, false, false}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z', false,\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"product-code\": null, \"source-loc\": null, \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{10}',\n'{3.0}',\n'{false, false, false}'\n)",
     "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z', true,\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\n'{1, 10, 20}',\n'{2.5, 5.0}',\n'{true}'\n)",
     "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z', false,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
     "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z', true,\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)"

document-store/src/integrationTest/resources/query/unnest_null_nested_array_response.json

@@ -0,0 +1,8 @@
+[
+  {
+    "item": "Soap"
+  },
+  {
+    "item": "Soap"
+  }
+]

document-store/src/integrationTest/resources/query/unnest_null_top_level_array_response.json

@@ -0,0 +1,5 @@
+[
+  {
+    "item": "Bottle"
+  }
+]

document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFilterTypeExpressionVisitor.java

@@ -222,8 +222,8 @@ private String getFilterStringForAnyOperator(final ArrayRelationalFilterExpressi
     } else {
       // For nested collections OR JSONB arrays in flat collections, use jsonb_array_elements()
       return String.format(
-          "EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(%s, '[]'::jsonb)) AS \"%s\" WHERE %s)",
-          parsedLhs, alias, parsedFilter);
+          "EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(%s) = 'array' THEN %s ELSE '[]'::jsonb END) AS \"%s\" WHERE %s)",
+          parsedLhs, parsedLhs, alias, parsedFilter);
     }
   }
 
@@ -331,9 +331,10 @@ private String getFilterStringForAnyOperator(final DocumentArrayFilterExpression
           parsedLhs, arrayTypeCast, alias, parsedFilter);
     } else {
       // For nested collections OR JSONB arrays in flat collections, use jsonb_array_elements()
+      // Use jsonb_typeof() to handle JSON null values - COALESCE only handles SQL NULL
       return String.format(
-          "EXISTS (SELECT 1 FROM jsonb_array_elements(COALESCE(%s, '[]'::jsonb)) AS \"%s\" WHERE %s)",
-          parsedLhs, alias, parsedFilter);
+          "EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(%s) = 'array' THEN %s ELSE '[]'::jsonb END) AS \"%s\" WHERE %s)",
+          parsedLhs, parsedLhs, alias, parsedFilter);
     }
   }
 }

document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java

@@ -24,7 +24,8 @@ public class PostgresFromTypeExpressionVisitor implements FromTypeExpressionVisi
       "%s as (SELECT * from %s %s, %s %s)";
   private static final String PRESERVE_NULL_AND_EMPTY_TABLE_QUERY_FMT =
       "%s as (SELECT * from %s %s LEFT JOIN LATERAL %s %s on TRUE)";
-  private static final String JSONB_UNWIND_EXP_FMT = "jsonb_array_elements(%s)";
+  private static final String JSONB_UNWIND_EXP_FMT =
+      "jsonb_array_elements(CASE WHEN jsonb_typeof(%s) = 'array' THEN %s ELSE '[]'::jsonb END)";
   private static final String NATIVE_UNWIND_EXP_FMT = "unnest(%s)";
   private static final String UNWIND_EXP_ALIAS_FMT = "p%s(%s)";
 
@@ -80,7 +81,10 @@ public String visit(UnnestExpression unnestExpression) {
     String preTable = "table" + preIndex;
     String newTable = "table" + nextIndex;
     String tableAlias = "t" + preIndex;
-    String unwindExpr = String.format(unnestFunction, transformedFieldName);
+    String unwindExpr =
+        unnestFunction.equals(JSONB_UNWIND_EXP_FMT)
+            ? String.format(unnestFunction, transformedFieldName, transformedFieldName)
+            : String.format(unnestFunction, transformedFieldName);
 
     // we'll quote the col name to prevent folding to lower case for top-level array fields
     String unwindExprAlias =