Sync with CTDS created with pytohn script

fence/blueprints/data/indexd.py

@@ -109,7 +109,7 @@ def get_signed_url_for_file(
     x_forwarded_headers = [
         f"{header}:{value}" for header, value in flask.request.headers if "X-" in header
     ]
-    user_agent = f"User-Agent:{flask.request.headers.get("User-Agent")}"
+    user_agent = f"User-Agent:{flask.request.headers.get('User-Agent')}"
     audit_headers = x_forwarded_headers + [user_agent]
     # add the user details to `flask.g.audit_data` first, so they are
     # included in the audit log if `IndexedFile(file_id)` raises a 404
@@ -988,12 +988,15 @@ def bucket_name(self):
         Return:
             Optional[str]: bucket name or None if not in config
         """
-
         s3_buckets = get_value(
             flask.current_app.config,
             "S3_BUCKETS",
             InternalError("S3_BUCKETS not configured"),
         )
+        explicit_match = self.parsed_url.netloc
+        if explicit_match in s3_buckets:
+            return explicit_match
+
         for bucket in s3_buckets:
             if re.match("^" + bucket + "$", self.parsed_url.netloc):
                 return bucket
@@ -1753,5 +1756,7 @@ def verify_data_upload_bucket_configuration(bucket):
     s3_buckets = flask.current_app.config["ALLOWED_DATA_UPLOAD_BUCKETS"]
     if bucket not in s3_buckets:
         logger.error(f"Bucket '{bucket}' not in ALLOWED_DATA_UPLOAD_BUCKETS config")
-        logger.debug(f"Buckets configgured in ALLOWED_DATA_UPLOAD_BUCKETS {s3_buckets}")
+        logger.debug(
+            f"Buckets configgured in ALLOWED_DATA_UPLOAD_BUCKETS: {s3_buckets}"
+        )
         raise Forbidden(f"Uploading to bucket '{bucket}' is not allowed")

fence/sync/sync_users.py

@@ -1934,7 +1934,7 @@ def _grant_arborist_policies(
             )
         except ArboristError as e:
             self.logger.error(
-                f"Could not get user {username} policies from Arborist: {e} Revoking all policies..."
+                f"Could not get user {username} policies from Arborist: {e}. Revoking all policies..."
             )
             # if getting existing policies fails, revoke all policies and re-apply
             is_revoke_all = True
@@ -2185,7 +2185,11 @@ def _update_authz_in_arborist(
                                 self._created_policies.add(policy_id)
                             policy_ids_to_grant.add(policy_id)
                 self._grant_arborist_policies(
-                    username, policy_ids_to_grant, user_yaml=None, expires=expires
+                    username,
+                    policy_ids_to_grant,
+                    user_yaml=None,
+                    expires=expires,
+                    remove_users_with_no_policies=False,
                 )
 
             if user_yaml:
@@ -2195,7 +2199,11 @@ def _update_authz_in_arborist(
                 )  # add policies from whitelist and useryaml
 
             self._grant_arborist_policies(
-                username, incoming_policies, user_yaml, expires=expires
+                username,
+                incoming_policies,
+                user_yaml,
+                expires=expires,
+                remove_users_with_no_policies=True,
             )
 
         if user_yaml:
@@ -2440,11 +2448,18 @@ def _grant_arborist_policy(self, username, policy_id, expires=None):
             bool: True if granting of policy was successful, False otherwise
         """
         try:
-            response_json = self.arborist_client.grant_user_policy(
+            resp = self.arborist_client.grant_user_policy(
                 username,
                 policy_id,
                 expires_at=expires,
             )
+            if not resp:
+                self.logger.error(
+                    "could not grant policy `{}` to user `{}`".format(
+                        policy_id, username
+                    )
+                )
+                return False
         except ArboristError as e:
             self.logger.error(
                 "could not grant policy `{}` to user `{}`: {}".format(
@@ -2471,12 +2486,17 @@ def _grant_bulk_user_policies(self, username, policy_ids, expires=None):
             bool: True if granting of policies was successful, False otherwise
         """
         try:
-            response_json = self.arborist_client.grant_bulk_user_policy(
+            resp = self.arborist_client.grant_bulk_user_policy(
                 username, policy_ids, expires
             )
+            if not resp:
+                self.logger.error(
+                    "could not grant bulk policies to user `{}`".format(username)
+                )
+                return False
         except ArboristError as e:
             self.logger.error(
-                "could not grant bulk policies  to user `{}`: {}".format(username, e)
+                "could not grant bulk policies to user `{}`: {}".format(username, e)
             )
             return False
         except ArboristTimeoutError as e: