diff --git a/src/borg/archiver.py b/src/borg/archiver.py
index fa02f037e2..5b8e16e1d9 100644
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -1200,7 +1200,7 @@ def hardlink_master_seen(item):
                 return 'source' not in item or not hardlinkable(item.mode) or item.source in hardlink_masters
 
             def is_hardlink_master(item):
-                return item.get('hardlink_master', True) and 'source' not in item
+                return item.get('hardlink_master', True) and 'source' not in item and hardlinkable(item.mode)
 
             def update_hardlink_masters(item1, item2):
                 if is_hardlink_master(item1) or is_hardlink_master(item2):