Merge branch 'postgres:master' into main

Author: pmp-p

.cirrus.tasks.yml

@@ -2,6 +2,12 @@
 #
 # For instructions on how to enable the CI integration in a repository and
 # further details, see src/tools/ci/README
+#
+#
+# NB: Different tasks intentionally test with different, non-default,
+# configurations, to increase the chance of catching problems. Each task with
+# non-obvious non-default documents their oddity at the top of the task,
+# prefixed by "SPECIAL:".
 
 
 env:
@@ -17,10 +23,13 @@ env:
   CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
   CHECKFLAGS: -Otarget
   PROVE_FLAGS: --timer
+  # Build test dependencies as part of the build step, to see compiler
+  # errors/warnings in one place.
+  MBUILD_TARGET: all testprep
   MTEST_ARGS: --print-errorlogs --no-rebuild -C build
   PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
   TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
-  PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
+  PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance oauth
 
 
 # What files to preserve in case tests fail
@@ -52,6 +61,10 @@ on_failure_meson: &on_failure_meson
 
 # To avoid unnecessarily spinning up a lot of VMs / containers for entirely
 # broken commits, have a minimal task that all others depend on.
+#
+# SPECIAL:
+# - Builds with --auto-features=disabled and thus almost no enabled
+#   dependencies
 task:
   name: SanityCheck
 
@@ -99,7 +112,7 @@ task:
     EOF
   build_script: |
     su postgres <<-EOF
-      ninja -C build -j${BUILD_JOBS}
+      ninja -C build -j${BUILD_JOBS} ${MBUILD_TARGET}
     EOF
   upload_caches: ccache
 
@@ -122,21 +135,33 @@ task:
       src/tools/ci/cores_backtrace.sh linux /tmp/cores
 
 
+# SPECIAL:
+# - Uses postgres specific CPPFLAGS that increase test coverage
+# - Specifies configuration options that test reading/writing/copying of node trees
+# - Specifies debug_parallel_query=regress, to catch related issues during CI
+# - Also runs tests against a running postgres instance, see test_running_script
 task:
-  name: FreeBSD - 13 - Meson
+  name: FreeBSD - Meson
 
   env:
     CPUS: 4
     BUILD_JOBS: 4
     TEST_JOBS: 8
-    IMAGE_FAMILY: pg-ci-freebsd-13
+    IMAGE_FAMILY: pg-ci-freebsd
     DISK_SIZE: 50
 
     CCACHE_DIR: /tmp/ccache_dir
     CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
     CFLAGS: -Og -ggdb
 
-    PG_TEST_INITDB_EXTRA_OPTS: -c debug_copy_parse_plan_trees=on -c debug_write_read_parse_plan_trees=on -c debug_raw_expression_coverage_test=on
+    # Several buildfarm animals enable these options. Without testing them
+    # during CI, it would be easy to cause breakage on the buildfarm with CI
+    # passing.
+    PG_TEST_INITDB_EXTRA_OPTS: >-
+      -c debug_copy_parse_plan_trees=on
+      -c debug_write_read_parse_plan_trees=on
+      -c debug_raw_expression_coverage_test=on
+      -c debug_parallel_query=regress
     PG_TEST_PG_UPGRADE_MODE: --link
 
   <<: *freebsd_task_template
@@ -152,8 +177,7 @@ task:
 
   ccache_cache:
     folder: $CCACHE_DIR
-  # Work around performance issues due to 32KB block size
-  repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
+  setup_ram_disk_script: src/tools/ci/gcp_ram_disk.sh
   create_user_script: |
     pw useradd postgres
     chown -R postgres:postgres .
@@ -178,7 +202,7 @@ task:
         -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
         build
     EOF
-  build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
+  build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS} ${MBUILD_TARGET}'
   upload_caches: ccache
 
   test_world_script: |
@@ -213,12 +237,120 @@ task:
     cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores
 
 
+task:
+  depends_on: SanityCheck
+  trigger_type: manual
+
+  env:
+    # Below are experimentally derived to be a decent choice.
+    CPUS: 4
+    BUILD_JOBS: 8
+    TEST_JOBS: 8
+
+    # Default working directory is /tmp, but its total size (1.2 GB) is not
+    # enough, so different working and cache directory are set.
+    CIRRUS_WORKING_DIR: /home/postgres/postgres
+    CCACHE_DIR: /home/postgres/cache
+
+    PATH: /usr/sbin:$PATH
+    CORE_DUMP_DIR: /var/crash
+
+  matrix:
+    - name: NetBSD - Meson
+      only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*netbsd.*'
+      env:
+        OS_NAME: netbsd
+        IMAGE_FAMILY: pg-ci-netbsd-postgres
+        PKGCONFIG_PATH: '/usr/lib/pkgconfig:/usr/pkg/lib/pkgconfig'
+        # initdb fails with: 'invalid locale settings' error on NetBSD.
+        # Force 'LANG' and 'LC_*' variables to be 'C'.
+        # See https://postgr.es/m/2490325.1734471752%40sss.pgh.pa.us
+        LANG: "C"
+        LC_ALL: "C"
+        # -Duuid is not set for the NetBSD, see the comment below, above
+        # configure_script, for more information.
+      setup_additional_packages_script: |
+        #pkgin -y install ...
+      <<: *netbsd_task_template
+
+    - name: OpenBSD - Meson
+      only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*openbsd.*'
+      env:
+        OS_NAME: openbsd
+        IMAGE_FAMILY: pg-ci-openbsd-postgres
+        PKGCONFIG_PATH: '/usr/lib/pkgconfig:/usr/local/lib/pkgconfig'
+        UUID: -Duuid=e2fs
+        TCL: -Dtcl_version=tcl86
+      setup_additional_packages_script: |
+        #pkg_add -I ...
+      # Always core dump to ${CORE_DUMP_DIR}
+      set_core_dump_script: sysctl -w kern.nosuidcoredump=2
+      <<: *openbsd_task_template
+
+  sysinfo_script: |
+    locale
+    id
+    uname -a
+    ulimit -a -H && ulimit -a -S
+    env
+
+  ccache_cache:
+    folder: $CCACHE_DIR
+  setup_ram_disk_script: src/tools/ci/gcp_ram_disk.sh
+  create_user_script: |
+    useradd postgres
+    chown -R postgres:users /home/postgres
+    mkdir -p ${CCACHE_DIR}
+    chown -R postgres:users ${CCACHE_DIR}
+  setup_core_files_script: |
+    mkdir -p ${CORE_DUMP_DIR}
+    chmod -R 770 ${CORE_DUMP_DIR}
+    chown -R postgres:users ${CORE_DUMP_DIR}
+
+  # -Duuid=bsd is not set since 'bsd' uuid option
+  # is not working on NetBSD & OpenBSD. See
+  # https://www.postgresql.org/message-id/17358-89806e7420797025@postgresql.org
+  # And other uuid options are not available on NetBSD.
+  configure_script: |
+    su postgres <<-EOF
+      meson setup \
+        --buildtype=debugoptimized \
+        --pkg-config-path ${PKGCONFIG_PATH} \
+        -Dcassert=true -Dinjection_points=true \
+        -Dssl=openssl ${UUID} ${TCL} \
+        -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
+        build
+    EOF
+
+  build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS} ${MBUILD_TARGET}'
+  upload_caches: ccache
+
+  test_world_script: |
+    su postgres <<-EOF
+      ulimit -c unlimited
+      # Otherwise tests will fail on OpenBSD, due to inability to start enough
+      # processes.
+      ulimit -p 256
+      meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
+    EOF
+
+  on_failure:
+    <<: *on_failure_meson
+    cores_script: |
+      # Although we try to configure the OS to core dump inside
+      # ${CORE_DUMP_DIR}, they may not obey this. So, move core files to the
+      # ${CORE_DUMP_DIR} directory.
+      find build/ -type f -name '*.core' -exec mv '{}' ${CORE_DUMP_DIR} \;
+      src/tools/ci/cores_backtrace.sh ${OS_NAME} ${CORE_DUMP_DIR}
+
+
 # configure feature flags, shared between the task running the linux tests and
 # the CompilerWarnings task
 LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
   --with-gssapi
   --with-icu
   --with-ldap
+  --with-libcurl
   --with-libxml
   --with-libxslt
   --with-llvm
@@ -238,6 +370,7 @@ LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >-
   -Duuid=e2fs
 
 
+# Check SPECIAL in the matrix: below
 task:
   env:
     CPUS: 4
@@ -316,6 +449,10 @@ task:
     #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
 
   matrix:
+    # SPECIAL:
+    # - Uses address sanitizer, sanitizer failures are typically printed in
+    #   the server log
+    # - Configures postgres with a small segment size
     - name: Linux - Debian Bookworm - Autoconf
 
       env:
@@ -334,6 +471,8 @@ task:
             --enable-cassert --enable-injection-points --enable-debug \
             --enable-tap-tests --enable-nls \
             --with-segsize-blocks=6 \
+            --with-libnuma \
+            --with-liburing \
             \
             ${LINUX_CONFIGURE_FEATURES} \
             \
@@ -351,11 +490,18 @@ task:
       on_failure:
         <<: *on_failure_ac
 
+    # SPECIAL:
+    # - Uses undefined behaviour and alignment sanitizers, sanitizer failures
+    #   are typically printed in the server log
+    # - Test both 64bit and 32 bit builds
+    # - uses io_method=io_uring
     - name: Linux - Debian Bookworm - Meson
 
       env:
         CCACHE_MAXSIZE: "400M" # tests two different builds
         SANITIZER_FLAGS: -fsanitize=alignment,undefined
+        PG_TEST_INITDB_EXTRA_OPTS: >-
+          -c io_method=io_uring
 
       configure_script: |
         su postgres <<-EOF
@@ -378,11 +524,21 @@ task:
             -Dllvm=disabled \
             --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \
             -DPERL=perl5.36-i386-linux-gnu \
+            -Dlibnuma=disabled \
             build-32
         EOF
 
-      build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
-      build_32_script: su postgres -c 'ninja -C build-32 -j${BUILD_JOBS}'
+      build_script: |
+        su postgres <<-EOF
+          ninja -C build -j${BUILD_JOBS} ${MBUILD_TARGET}
+          ninja -C build -t missingdeps
+        EOF
+
+      build_32_script: |
+        su postgres <<-EOF
+          ninja -C build-32 -j${BUILD_JOBS} ${MBUILD_TARGET}
+          ninja -C build -t missingdeps
+        EOF
 
       upload_caches: ccache
 
@@ -411,6 +567,11 @@ task:
     cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
 
 
+# NB: macOS is by far the most expensive OS to run CI for, therefore no
+# expensive additional checks should be added.
+#
+# SPECIAL:
+# - Enables --clone for pg_upgrade and pg_combinebackup
 task:
   name: macOS - Sonoma - Meson
 
@@ -503,7 +664,7 @@ task:
       -Duuid=e2fs -Ddtrace=auto \
       build
 
-  build_script: ninja -C build -j${BUILD_JOBS}
+  build_script: ninja -C build -j${BUILD_JOBS} ${MBUILD_TARGET}
   upload_caches: ccache
 
   test_world_script: |
@@ -576,7 +737,8 @@ task:
 
   build_script: |
     vcvarsall x64
-    ninja -C build
+    ninja -C build %MBUILD_TARGET%
+    ninja -C build -t missingdeps
 
   check_world_script: |
     vcvarsall x64
@@ -634,7 +796,7 @@ task:
     %BASH% -c "meson setup -Ddebug=true -Doptimization=g -Dcassert=true -Dinjection_points=true -Db_pch=true -Dnls=disabled -DTAR=%TAR% build"
 
   build_script: |
-    %BASH% -c "ninja -C build"
+    %BASH% -c "ninja -C build ${MBUILD_TARGET}"
 
   upload_caches: ccache
 

.cirrus.yml

@@ -52,6 +52,16 @@ default_freebsd_task_template: &freebsd_task_template
     PLATFORM: freebsd
   <<: *cirrus_community_vm_template
 
+default_netbsd_task_template: &netbsd_task_template
+  env:
+    PLATFORM: netbsd
+  <<: *cirrus_community_vm_template
+
+default_openbsd_task_template: &openbsd_task_template
+  env:
+    PLATFORM: openbsd
+  <<: *cirrus_community_vm_template
+
 
 default_windows_task_template: &windows_task_template
   env: