CI: add gpMgmt Behave test

.github/workflows/build-cloudberry.yml

@@ -223,7 +223,6 @@ jobs:
           DEFAULT_ENABLE_CGROUPS=false
           DEFAULT_ENABLE_CORE_CHECK=true
           DEFAULT_PG_SETTINGS_OPTIMIZER=""
-
           # Define base test configurations
           ALL_TESTS='{
             "include": [
@@ -1573,8 +1572,6 @@ jobs:
               continue
             fi
 
-            # Parse this configuration's results
-
             MAKE_NAME="${{ matrix.test }}-config$i" \
             "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log"
             status_code=$?

gpAux/gpdemo/demo_cluster.sh

@@ -314,8 +314,10 @@ cat >> $CLUSTER_CONFIG <<-EOF
 
 	COORDINATOR_PORT=${COORDINATOR_DEMO_PORT}
 
-	# Shell to use to execute commands on all hosts
-	TRUSTED_SHELL="$(dirname "$0")/lalshell"
+	# Shell to use to execute commands on all hosts. Use an absolute path here
+	# because this file is later sourced by gpinitsystem, where \$0 is no longer
+	# demo_cluster.sh.
+	TRUSTED_SHELL=$(pwd)/lalshell
 
 	ENCODING=UNICODE
 EOF

gpMgmt/bin/analyzedb

@@ -49,6 +49,12 @@ ANALYZE_ROOT_SQL = """analyze rootpartition %s"""
 REPORTS_ARE_STALE_AFTER_N_DAYS = 8
 NUM_REPORTS_TO_SAVE = 3
 
+
+def safe_log_string(value):
+    if isinstance(value, str):
+        return value.encode('ascii', 'backslashreplace').decode('ascii')
+    return str(value)
+
 GET_ALL_DATA_TABLES_SQL = """
 select n.nspname as schemaname, c.relname as tablename
 from pg_class c, pg_namespace n
@@ -430,7 +436,7 @@ class AnalyzeDb(Operation):
                     target = self._get_tablename_with_cols(can_schema, can_table, input_col_dict)
                 else:  # can in root_partition_col_dict
                     target = self._get_tablename_with_cols(can_schema, can_table, root_partition_col_dict)
-                logger.info(target)
+                logger.info(safe_log_string(target))
                 target_list.append(target)
             logger.info("---------------------------------------------------")
 
@@ -951,7 +957,10 @@ class AnalyzeDb(Operation):
 # Create a Command object that executes a query using psql.
 def create_psql_command(dbname, query):
     psql_cmd = """psql %s -c %s""" % (pipes.quote(dbname), pipes.quote(query))
-    return Command(query, psql_cmd)
+    # Keep the command text intact for execution, but make the display name
+    # ASCII-safe so logger/output paths do not choke on UTF-8 identifiers.
+    safe_query_display = query.encode('ascii', 'backslashreplace').decode('ascii')
+    return Command(safe_query_display, psql_cmd)
 
 
 def run_sql(conn, query):
@@ -1402,13 +1411,13 @@ class AnalyzeWorker(Worker):
                     self.cmd = None
                     return
                 elif self.pool.should_stop:
-                    self.logger.debug("[%s] got cmd and pool is stopped: %s" % (self.name, self.cmd))
+                    self.logger.debug("[%s] got cmd and pool is stopped: %s" % (self.name, safe_log_string(self.cmd)))
                     self.pool.markTaskDone()
                     self.cmd = None
                 else:
                     # run the command
                     # get rid of the gucs for displaying in the log
-                    cmd_display = re.sub(r'set .*;\s*', '', self.cmd.name)
+                    cmd_display = safe_log_string(re.sub(r'set .*;\s*', '', self.cmd.name))
                     self.logger.info("[%s] started  %s" % (self.name, cmd_display))
                     start_time = time.time()
                     self.cmd.run()
@@ -1425,7 +1434,7 @@ class AnalyzeWorker(Worker):
             except Exception as e:
                 self.logger.exception(e)
                 if self.cmd:
-                    self.logger.debug("[%s] finished cmd with exception: %s" % (self.name, self.cmd))
+                    self.logger.debug("[%s] finished cmd with exception: %s" % (self.name, safe_log_string(self.cmd)))
                     self.pool.addFinishedWorkItem(self.cmd)
                     self.cmd = None
 

gpMgmt/bin/gppylib/commands/base.py

@@ -37,6 +37,12 @@
 
 CMD_CACHE = {}
 
+
+def _safe_log_string(value):
+    if isinstance(value, str):
+        return value.encode('ascii', 'backslashreplace').decode('ascii')
+    return str(value)
+
 # Maximum retries if sshd rejects the connection due to too many
 # unauthenticated connections.
 SSH_MAX_RETRY = 10
@@ -86,7 +92,7 @@ def markTaskDone(self):
         self.work_queue.task_done()
 
     def addCommand(self, cmd):
-        self.logger.debug("Adding cmd to work_queue: %s" % cmd.cmdStr)
+        self.logger.debug("Adding cmd to work_queue: %s" % _safe_log_string(cmd.cmdStr))
         self.work_queue.put(cmd)
         self._assigned += 1
 
@@ -272,20 +278,20 @@ def run(self):
                     self.cmd = None
                     return
                 elif self.pool.should_stop:
-                    self.logger.debug("[%s] got cmd and pool is stopped: %s" % (self.name, self.cmd))
+                    self.logger.debug("[%s] got cmd and pool is stopped: %s" % (self.name, _safe_log_string(self.cmd)))
                     self.pool.markTaskDone()
                     self.cmd = None
                 else:
-                    self.logger.debug("[%s] got cmd: %s" % (self.name, self.cmd.cmdStr))
+                    self.logger.debug("[%s] got cmd: %s" % (self.name, _safe_log_string(self.cmd.cmdStr)))
                     self.cmd.run()
-                    self.logger.debug("[%s] finished cmd: %s" % (self.name, self.cmd))
+                    self.logger.debug("[%s] finished cmd: %s" % (self.name, _safe_log_string(self.cmd)))
                     self.pool.addFinishedWorkItem(self.cmd)
                     self.cmd = None
 
             except Exception as e:
                 self.logger.exception(e)
                 if self.cmd:
-                    self.logger.debug("[%s] finished cmd with exception: %s" % (self.name, self.cmd))
+                    self.logger.debug("[%s] finished cmd with exception: %s" % (self.name, _safe_log_string(self.cmd)))
                     self.pool.addFinishedWorkItem(self.cmd)
                     self.cmd = None
 
@@ -548,9 +554,9 @@ def __init__(self, name, cmdStr, ctxt=LOCAL, remoteHost=None, stdin=None, gphome
 
     def __str__(self):
         if self.results:
-            return "%s cmdStr='%s'  had result: %s" % (self.name, self.cmdStr, self.results)
+            return "%s cmdStr='%s'  had result: %s" % (self.name, _safe_log_string(self.cmdStr), self.results)
         else:
-            return "%s cmdStr='%s'" % (self.name, self.cmdStr)
+            return "%s cmdStr='%s'" % (self.name, _safe_log_string(self.cmdStr))
 
     # Start a process that will execute the command but don't wait for
     # it to complete.  Return the Popen object instead.
@@ -559,7 +565,7 @@ def runNoWait(self):
         return self.exec_context.proc
 
     def run(self, validateAfter=False):
-        self.logger.debug("Running Command: %s" % self.cmdStr)
+        self.logger.debug("Running Command: %s" % _safe_log_string(self.cmdStr))
         self.exec_context.execute(self, pickled=self.pickled, start_new_session=self.start_new_session)
 
         if validateAfter:

gpMgmt/bin/gppylib/test/unit/test_unit_analyzedb.py

@@ -0,0 +1,22 @@
+import imp
+import os
+
+from gppylib.test.unit.gp_unittest import GpTestCase, run_tests
+
+
+class AnalyzeDbTestCase(GpTestCase):
+    def setUp(self):
+        analyzedb_file = os.path.abspath(os.path.dirname(__file__) + "/../../../analyzedb")
+        self.subject = imp.load_source('analyzedb', analyzedb_file)
+
+    def test_create_psql_command_keeps_utf8_sql_but_uses_ascii_safe_display_name(self):
+        query = 'analyze "public"."spiegelungssätze"'
+
+        cmd = self.subject.create_psql_command('special_encoding_db', query)
+
+        self.assertEqual(cmd.name, 'analyze "public"."spiegelungss\\xe4tze"')
+        self.assertIn('spiegelungssätze', cmd.cmdStr)
+
+
+if __name__ == '__main__':
+    run_tests()

gpMgmt/test/behave/mgmt_utils/gpcheckcat.feature

@@ -311,14 +311,14 @@ Feature: gpcheckcat tests
         Then gpcheckcat should return a return code of 3
         And the user runs "dropdb fkey_ta"
 
+    @extended
     Scenario: gpcheckcat should report and repair extra entries with non-oid primary keys
         Given database "extra_pk_db" is dropped and recreated
         And the path "gpcheckcat.repair.*" is removed from current working directory
         And the user runs "psql extra_pk_db -c 'CREATE SCHEMA my_pk_schema' "
         And the user runs "psql extra_pk_db -f test/behave/mgmt_utils/steps/data/gpcheckcat/add_operator.sql "
         Then psql should return a return code of 0
-        And the user runs "psql extra_pk_db -c "set allow_system_table_mods=true;DELETE FROM pg_catalog.pg_operator where oprname='!#'" "
-        Then psql should return a return code of 0
+        Then The user runs sql "set allow_system_table_mods=true;DELETE FROM pg_catalog.pg_operator where oprname='!#'" in "extra_pk_db" on first primary segment
         When the user runs "gpcheckcat -R missing_extraneous extra_pk_db"
         Then gpcheckcat should return a return code of 3
         And the path "gpcheckcat.repair.*" is found in cwd "0" times
@@ -728,18 +728,13 @@ Feature: gpcheckcat tests
         And the user runs "dropdb all_good"
 
 
-    Scenario: validate session GUC passed with -x is set
+    Scenario: gpcheckcat accepts session GUC passed with -x in single node mode
         Given the database is not running
           And the user runs "gpstart -ma"
           And "gpstart -ma" should return a return code of 0
-         Then the user runs "gpcheckcat -R foreign_key"
-         Then gpcheckcat should return a return code of 1
-          And gpcheckcat should print ".* System was started in single node mode - only utility mode connections are allowed" to stdout
          Then the user runs "gpcheckcat -x gp_role=utility -R foreign_key"
          Then gpcheckcat should return a return code of 0
           And the user runs "gpstop -ma"
           And "gpstop -m" should return a return code of 0
           And the user runs "gpstart -a"
 
-
-

gpMgmt/test/behave/mgmt_utils/gpinitsystem.feature

@@ -70,6 +70,7 @@ Feature: gpinitsystem tests
         Given the user runs "gpstate"
          Then gpstate should return a return code of 0
 
+    @extended
     Scenario: gpinitsystem creates a backout file when gpinitsystem process terminated
         Given create demo cluster config
         And all files in gpAdminLogs directory are deleted
@@ -84,6 +85,7 @@ Feature: gpinitsystem tests
         And gpinitsystem should return a return code of 0
         And gpintsystem logs should not contain lines about running backout script
 
+    @extended
     Scenario: gpinitsystem creates a backout file when gpcreateseg process terminated
         Given create demo cluster config
         And all files in gpAdminLogs directory are deleted
@@ -97,6 +99,7 @@ Feature: gpinitsystem tests
         And gpinitsystem should return a return code of 0
         And gpintsystem logs should not contain lines about running backout script
 
+    @extended
     Scenario: gpinitsystem does not create or need backout file when user terminated very early
         Given create demo cluster config
         And all files in gpAdminLogs directory are deleted
@@ -333,4 +336,3 @@ Feature: gpinitsystem tests
         When the user runs command "grep -q '.*gpcreateseg\.sh.*Completed ssh.*' ~/gpAdminLogs/gpinitsystem*log"
         Then grep should return a return code of 0
         And the user runs command "mv ../gpAux/gpdemo/clusterConfigFile.bak ../gpAux/gpdemo/clusterConfigFile"
-

gpMgmt/test/behave/mgmt_utils/gpmovemirrors.feature

@@ -103,6 +103,7 @@ Feature: Tests for gpmovemirrors
                 assert that gp_seg_config wasn't updated
         """
 
+    @extended
     Scenario Outline: user can <correction> if <failed_count> mirrors failed to move initially
         Given the database is running
         And all the segments are running
@@ -146,6 +147,7 @@ Feature: Tests for gpmovemirrors
         | run gprecoverseg          | some         | 0                  | 1,2            | running in place full recovery for all failed contents |
         | run gprecoverseg          | all          | None               | 0,1,2          | running in place full recovery for all failed contents |
 
+    @extended
     @skip_cleanup
     Scenario: gpmovemirrors can move mirrors even if start fails for some mirrors
         Given the database is running
@@ -181,6 +183,7 @@ Feature: Tests for gpmovemirrors
         And user can start transactions
 
 
+  @extended
   @demo_cluster
   Scenario: gpmovemirrors -i creates recovery_progress.file if some mirrors are moved
     Given the database is running
@@ -204,6 +207,7 @@ Feature: Tests for gpmovemirrors
     And user can start transactions
     And all files in gpAdminLogs directory are deleted on all hosts in the cluster
 
+  @extended
   @demo_cluster
   Scenario: gpmovemirrors -i creates recovery_progress.file if all mirrors are moved
     Given the database is running

gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature

@@ -1,7 +1,7 @@
 @gprecoverseg
 Feature: gprecoverseg tests
 
-    Scenario: incremental recovery works with tablespaces
+    Scenario Outline: incremental recovery works with tablespaces
         Given the database is running
           And a tablespace is created with data
           And user stops all primary processes
@@ -117,6 +117,7 @@ Feature: gprecoverseg tests
         And all the segments are running
         And the segments are synchronized
 
+    @extended
     Scenario: gprecoverseg full recovery displays pg_controldata success info
         Given the database is running
         And all the segments are running
@@ -129,6 +130,7 @@ Feature: gprecoverseg tests
         And the segments are synchronized
         And check segment conf: postgresql.conf
 
+    @extended
     Scenario: gprecoverseg incremental recovery displays pg_controldata success info
         Given the database is running
         And all the segments are running
@@ -141,6 +143,7 @@ Feature: gprecoverseg tests
         And the segments are synchronized
         And check segment conf: postgresql.conf
 
+    @extended
     Scenario: gprecoverseg full recovery displays pg_controldata success info
         Given the database is running
         And all the segments are running
@@ -153,6 +156,7 @@ Feature: gprecoverseg tests
         And the segments are synchronized
         And check segment conf: postgresql.conf
 
+    @extended
     Scenario: gprecoverseg incremental recovery displays pg_controldata success info
         Given the database is running
         And all the segments are running
@@ -256,6 +260,7 @@ Feature: gprecoverseg tests
           And all the segments are running
           And the segments are synchronized
 
+    @extended
     Scenario: gprecoverseg differential recovery displays rsync progress to the user
         Given the database is running
         And all the segments are running
@@ -278,6 +283,7 @@ Feature: gprecoverseg tests
         And verify replication slot internal_wal_replication_slot is available on all the segments
         And check segment conf: postgresql.conf
 
+    @extended
     Scenario: gprecoverseg does not display rsync progress to the user when --no-progress option is specified
         Given the database is running
         And all the segments are running
@@ -299,6 +305,7 @@ Feature: gprecoverseg tests
         And the segments are synchronized
         And check segment conf: postgresql.conf
 
+    @extended
     Scenario: When gprecoverseg incremental recovery uses pg_rewind to recover and an existing postmaster.pid on the killed primary segment corresponds to a non postgres process
         Given the database is running
         And all the segments are running
@@ -323,6 +330,7 @@ Feature: gprecoverseg tests
         And the backup pid file is deleted on "primary" segment
         And the background pid is killed on "primary" segment
 
+    @extended
     Scenario: Pid does not correspond to any running process
         Given the database is running
         And all the segments are running
@@ -345,6 +353,7 @@ Feature: gprecoverseg tests
         And the segments are synchronized
         And the backup pid file is deleted on "primary" segment
 
+    @extended
     Scenario: pg_isready functions on recovered segments
         Given the database is running
           And all the segments are running
@@ -362,6 +371,7 @@ Feature: gprecoverseg tests
           And the segments are synchronized
           And pg_isready reports all primaries are accepting connections
 
+    @extended
     Scenario: gprecoverseg incremental recovery displays status for mirrors after pg_rewind call
         Given the database is running
         And all the segments are running
@@ -376,6 +386,7 @@ Feature: gprecoverseg tests
         And the segments are synchronized
         And the cluster is rebalanced
 
+    @extended
     @backup_restore_bashrc
     Scenario: gprecoverseg should not return error when banner configured on host
         Given the database is running
@@ -718,6 +729,7 @@ Feature: gprecoverseg tests
     And the cluster is recovered in full and rebalanced
     And the row count from table "test_recoverseg" in "postgres" is verified against the saved data
 
+  @extended
   @demo_cluster
   Scenario Outline: gprecoverseg differential recovery segments come up even if recovery for one segment fails
     Given the database is running
@@ -1813,6 +1825,7 @@ Feature: gprecoverseg tests
           And gprecoverseg should return a return code of 0
           And the cluster is rebalanced
 
+  @extended
   @demo_cluster
   Scenario: gprecoverseg recovers segment when config file contains hostname on demo cluster
     Given the database is running
@@ -1831,6 +1844,7 @@ Feature: gprecoverseg tests
     And the cluster configuration has no segments where "content=0 and status='d'"
     Then the cluster is rebalanced
 
+  @extended
   @demo_cluster
   Scenario: gprecoverseg skips recovery when config file contains invalid hostname on demo cluster
     Given the database is running