Merge release branch 4.7 to 4.8

* 4.7:
  Revert "Merge pull request #1482 from remibergsma/iptables-fix"

Author: swill

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -17,16 +17,27 @@
 # specific language governing permissions and limitations
 # under the License.
 import sys
+import os
 import base64
 
+from merge import DataBag
+from pprint import pprint
+import subprocess
+import logging
 import re
+import time
+import shutil
+import os.path
+import os
 from fcntl import flock, LOCK_EX, LOCK_UN
 
-from cs.CsDatabag import CsDataBag
+from cs.CsDatabag import CsDataBag, CsCmdLine
+import cs.CsHelper
 from cs.CsNetfilter import CsNetfilters
 from cs.CsDhcp import CsDhcp
 from cs.CsRedundant import *
 from cs.CsFile import CsFile
+from cs.CsApp import CsApache, CsDnsmasq
 from cs.CsMonitor import CsMonitor
 from cs.CsLoadBalancer import CsLoadBalancer
 from cs.CsConfig import CsConfig
@@ -197,23 +208,7 @@ def create(self):
 
         def process(self, direction, rule_list, base):
             count = base
-            rule_list_splitted = []
-            for rule in rule_list:
-                if ',' in rule['cidr']:
-                    cidrs = rule['cidr'].split(',')
-                    for cidr in cidrs:
-                        new_rule = {
-                            'cidr': cidr,
-                            'last_port': rule['last_port'],
-                            'type': rule['type'],
-                            'first_port': rule['first_port'],
-                            'allowed': rule['allowed']
-                        }
-                        rule_list_splitted.append(new_rule)
-                else:
-                    rule_list_splitted.append(rule)
-
-            for i in rule_list_splitted:
+            for i in rule_list:
                 r = self.AclRule(direction, self, i, self.config, count)
                 r.create()
                 count += 1
@@ -266,7 +261,7 @@ def create(self):
                     rstr = "%s -m icmp --icmp-type %s" % (rstr, self.icmp_type)
                 rstr = "%s %s -j %s" % (rstr, self.dport, self.action)
                 rstr = rstr.replace("  ", " ").lstrip()
-                self.fw.append([self.table, "", rstr])
+                self.fw.append([self.table, self.count, rstr])
 
     def process(self):
         for item in self.dbag:
@@ -480,7 +475,7 @@ def configure_iptables(self, dev, obj):
         self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
         self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
         self.fw.append(["", "front", "-A INPUT -i %s -p esp -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])])
-        self.fw.append(["nat", "front", "-A POSTROUTING -o %s -m mark --mark 0x525 -j ACCEPT" % dev])
+        self.fw.append(["nat", "front", "-A POSTROUTING -t nat -o %s -m mark --mark 0x525 -j ACCEPT" % dev])
         for net in obj['peer_guest_cidr_list'].lstrip().rstrip().split(','):
             self.fw.append(["mangle", "front",
                             "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)])
@@ -796,7 +791,7 @@ def forward_vr(self, rule):
                 rule['internal_ip'],
                 internal_fwports
               )
-        fw4 = "-A POSTROUTING -j SNAT --to-source %s -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \
+        fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \
               (
                 self.getGuestIp(),
                 self.getNetworkByIp(rule['internal_ip']),
@@ -991,7 +986,7 @@ def main(argv):
         lb.process()
 
         logging.debug("Configuring iptables rules")
-        nf = CsNetfilters(False)
+        nf = CsNetfilters()
         nf.compare(config.get_fw())
 
         logging.debug("Configuring iptables rules done ...saving rules")

systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py

@@ -15,8 +15,9 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from CsDatabag import CsDataBag
+from CsDatabag import CsDataBag, CsCmdLine
 from CsApp import CsApache, CsDnsmasq, CsPasswdSvc
+import CsHelper
 import logging
 from netaddr import IPAddress, IPNetwork
 import CsHelper
@@ -197,7 +198,7 @@ def is_added(self):
         return self.get_attr("add")
 
     def to_str(self):
-        print(self.address)
+        pprint(self.address)
 
 
 class CsDevice:
@@ -370,6 +371,8 @@ def fw_router(self):
             self.fw.append(["mangle", "front",
                             "-A FIREWALL_%s " % self.address['public_ip'] +
                             "-m state --state RELATED,ESTABLISHED -j ACCEPT"])
+            self.fw.append(["mangle", "",
+                            "-A FIREWALL_%s DROP" % self.address['public_ip']])
             self.fw.append(["mangle", "",
                             "-A VPN_%s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.address['public_ip']])
             self.fw.append(["mangle", "",
@@ -387,7 +390,8 @@ def fw_router(self):
 
         self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"])
-        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev])
+        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" %
+                        self.dev])
         self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"])
 
@@ -430,13 +434,6 @@ def fw_vpcrouter(self):
         self.fw.append(["mangle", "front", "-A PREROUTING " +
                         "-m state --state RELATED,ESTABLISHED " +
                         "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"])
-
-        self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"])
-        self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"])
-        self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"])
-
-        self.fw.append(["filter", "", "-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT"])
-
         if self.get_type() in ["guest"]:
             self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" %
                             (self.address['network'], self.dev, self.dev)])
@@ -475,14 +472,22 @@ def fw_vpcrouter(self):
                             ])
 
         if self.get_type() in ["public"]:
+            self.fw.append(["", "front",
+                            "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % (
+                                self.dev, self.address['network'], self.dev)
+                            ])
             self.fw.append(
                 ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev])
             self.fw.append(
                 ["mangle", "", "-A VPN_STATS_%s -o %s -m mark --mark 0x525/0xffffffff" % (self.dev, self.dev)])
             self.fw.append(
                 ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)])
             self.fw.append(
-                ["", "front", "-A FORWARD -j NETWORK_STATS_eth1"])
+                ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev])
+
+        self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"])
+        self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"])
+        self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"])
 
         self.fw.append(["", "", "-A NETWORK_STATS -i eth0 -o eth2 -p tcp"])
         self.fw.append(["", "", "-A NETWORK_STATS -i eth2 -o eth0 -p tcp"])
@@ -491,11 +496,9 @@ def fw_vpcrouter(self):
 
         self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"])
-        self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev])
-        self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"])
+
         self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"])
         self.fw.append(["filter", "", "-A INPUT -i eth0 -p tcp -m tcp --dport 3922 -m state --state NEW,ESTABLISHED -j ACCEPT"])
-        self.fw.append(["filter", "", "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT"])
 
         self.fw.append(["filter", "", "-P INPUT DROP"])
         self.fw.append(["filter", "", "-P FORWARD DROP"])

systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py

@@ -54,7 +54,7 @@ def process(self):
         self.cloud.commit()
 
         # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems.
-        CsHelper.execute2("service dnsmasq restart")
+        CsHelper.service("dnsmasq", "restart")
 
     def configure_server(self):
         # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS)

systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py

@@ -71,16 +71,14 @@ def _configure_firewall(self, add_rules, remove_rules, stat_rules):
             port = path[1]
             firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
 
-        for rules in stat_rules:
+        for rules in remove_rules:
             path = rules.split(':')
             ip = path[0]
             port = path[1]
-            firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
+            firewall.append(["filter", "", "-D INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
 
-        for rules in remove_rules:
+        for rules in stat_rules:
             path = rules.split(':')
             ip = path[0]
             port = path[1]
-            if ["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)] in firewall:
-                firewall.remove(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])
-
+            firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)])

systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py

@@ -15,12 +15,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import print_function
-
 import CsHelper
-from CsDatabag import CsCmdLine
+from pprint import pprint
+from CsDatabag import CsDataBag, CsCmdLine
 import logging
-from cs_iptables_save import Tables
 
 
 class CsChain(object):
@@ -83,7 +81,6 @@ class CsNetfilters(object):
 
     def __init__(self, load=True):
         self.rules = []
-        self.iptablerules = []
         self.table = CsTable()
         self.chain = CsChain()
         if load:
@@ -94,10 +91,7 @@ def get_all_rules(self):
             if i.startswith('*'):  # Table
                 self.table.add(i[1:])
             if i.startswith(':'):  # Chain
-                string = i[1:].split(' ')[0]
-                cmd = "iptables -t %s -N %s" % (self.table.last(), string)
-                self.iptablerules.append(cmd)
-                self.chain.add(self.table.last(), string)
+                self.chain.add(self.table.last(), i[1:].split(' ')[0])
             if i.startswith('-A'):  # Rule
                 self.chain.add_rule(i.split()[1])
                 rule = CsNetfilter()
@@ -131,7 +125,10 @@ def has_rule(self, new_rule):
     def get_unseen(self):
         del_list = [x for x in self.rules if x.unseen()]
         for r in del_list:
-            self.delete(r)
+            cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True))
+            logging.debug("unseen cmd:  %s ", cmd)
+            CsHelper.execute(cmd)
+            # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table())
             logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table())
 
     def compare(self, list):
@@ -140,16 +137,12 @@ def compare(self, list):
             # Ensure all inbound/outbound chains have a default drop rule
             if c.startswith("ACL_INBOUND") or c.startswith("ACL_OUTBOUND"):
                 list.append(["filter", "", "-A %s -j DROP" % c])
-        # PASS 1:  Ensure all chains are present and cleanup unused rules.
+        # PASS 1:  Ensure all chains are present
         for fw in list:
             new_rule = CsNetfilter()
             new_rule.parse(fw[2])
             new_rule.set_table(fw[0])
-            self.has_rule(new_rule)
-
-        self.del_standard()
-        self.get_unseen()
-
+            self.add_chain(new_rule)
         # PASS 2: Create rules
         for fw in list:
             new_rule = CsNetfilter()
@@ -158,33 +151,28 @@ def compare(self, list):
             if isinstance(fw[1], int):
                 new_rule.set_count(fw[1])
 
-            logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table())
-            # front means insert instead of append
-            cpy = fw[2]
-            if fw[1] == "front":
-                cpy = cpy.replace('-A', '-I')
-            if isinstance(fw[1], int):
-                cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1]))
-
-            self.iptablerules.append("iptables -t %s %s" % (new_rule.get_table(), cpy))
-        self.apply_rules()
-
-    def apply_rules(self):
-        s = []
-        for r in self.iptablerules:
-            r.replace('  ', ' ')  # Remove duplicate spaces
-            if r not in s:
-                s.append(r)
-
-        chains = Tables(s)
-        chains.table_printout()
+            logging.debug("Checking if the rule already exists: rule=%s table=%s chain=%s", new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain())
+            if self.has_rule(new_rule):
+                logging.debug("Exists: rule=%s table=%s", fw[2], new_rule.get_table())
+            else:
+                # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table())
+                logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table())
+                # front means insert instead of append
+                cpy = fw[2]
+                if fw[1] == "front":
+                    cpy = cpy.replace('-A', '-I')
+                if isinstance(fw[1], int):
+                    cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1]))
+
+                CsHelper.execute("iptables -t %s %s" % (new_rule.get_table(), cpy))
+        self.del_standard()
+        self.get_unseen()
 
-        # COMMIT all rules.
-        result = CsHelper.execute("iptables-restore < /tmp/rules.save")
-        if result:
-            logging.info("iptables-restore result: %s", result)
-        else:
-            logging.info("iptables-restore result: success!")
+    def add_chain(self, rule):
+        """ Add the given chain if it is not already present """
+        if not self.has_chain(rule.get_table(), rule.get_chain()):
+            CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain()))
+            self.chain.add(rule.get_table(), rule.get_chain())
 
     def del_standard(self):
         """ Del rules that are there but should not be deleted