diff --git a/serveradmin/serverdb/forms.py b/serveradmin/serverdb/forms.py index b450e395..03c065f8 100644 --- a/serveradmin/serverdb/forms.py +++ b/serveradmin/serverdb/forms.py @@ -24,7 +24,7 @@ def clean(self): # It makes no sense to add inet or supernet attributes to hosts of # ip_addr_type null because they would have to be empty anyways. inet_attribute = ( - self.cleaned_data['attribute'].type in ('inet', 'supernet') and + self.cleaned_data['attribute'].type in ('inet', 'inet4', 'inet6', 'supernet') and self.instance.servertype.ip_addr_type == 'null' ) if inet_attribute: diff --git a/serveradmin/serverdb/models.py b/serveradmin/serverdb/models.py index 7a625521..72c0fec6 100644 --- a/serveradmin/serverdb/models.py +++ b/serveradmin/serverdb/models.py @@ -4,6 +4,7 @@ """ import re +from collections import defaultdict from distutils.util import strtobool from ipaddress import ( IPv4Address, @@ -13,7 +14,7 @@ IPv6Interface, ip_network, IPv4Network, - IPv6Network, + IPv6Network, AddressValueError, NetmaskValueError, ) from typing import Union @@ -38,6 +39,8 @@ 'reverse': str, 'number': lambda x: float(x) if '.' in str(x) else int(x), 'inet': lambda x: inet_to_python(x), + 'inet4': lambda x: inet4_to_python(x), + 'inet6': lambda x: inet6_to_python(x), 'macaddr': EUI, 'date': str, 'datetime': str, @@ -77,6 +80,47 @@ def get_choices(types): return zip(*([types] * 2)) +def is_supernet_consistent(ip_address, server): + """Check if requested IP address is consistent with supernets for all other IP address of a server.""" + + if ip_address.version == 4: + server_attribute_cls = ServerInet4Attribute + else: + server_attribute_cls = ServerInet6Attribute + + # A server belongs to different supernets depending on network servertype + supernet_attributes = Attribute.objects.filter( + type='supernet', + servertype_attributes__servertype_id=server.servertype_id, + ) + supernet_servertypes = [x.target_servertype for x in supernet_attributes] + + for supernet_servertype in supernet_servertypes: + supernet_1 = server.get_supernet(supernet_servertype) # The result is guaranteed to be consistent + + if supernet_1 is None: + continue + + # Check the supernet of the new IP address + try: + supernet_2 = server_attribute_cls.objects.get( + value__net_contains_or_equals=ip_address, + server__servertype__ip_addr_type='network', + server__servertype_id=supernet_servertype, + ).server + except server_attribute_cls.DoesNotExist: + print(f'Zupa: {supernet_1} DoesNotExist') + # There are some provider networks which are valid only for one address family. + # TODO: Enforce validation once the "local" network is gone. + continue + + if supernet_1 != supernet_2: + raise ValidationError( + f'Non-matching {supernet_servertype} {supernet_2} for IP address {ip_address}, ' + f'other IP addresses of {server.hostname} are in {supernet_1}' + ) + + # TODO: Make validators out of the methods is_ip_address, is_unique and # is_network and attach them to the model fields validators. def is_ip_address(ip_interface: Union[IPv4Interface, IPv6Interface]) -> None: @@ -106,16 +150,30 @@ def is_unique_ip(ip_interface: Union[IPv4Interface, IPv6Interface], :return: """ + if ip_interface.version == 4: + server_attribute_cls = ServerInet4Attribute + else: + server_attribute_cls = ServerInet6Attribute + # We avoid querying the duplicate hosts here and giving the user # detailed information because checking with exists is cheaper than # querying the server and this is a validation and should be fast. has_duplicates = ( + # TODO: Remove "intern_ip" support. Server.objects.filter(intern_ip=ip_interface).exclude( Q(servertype__ip_addr_type='network') | Q(server_id=object_id) ).exists() or + # TODO: Remove "primary_ip6" support. ServerInetAttribute.objects.filter(value=ip_interface).exclude( - server__servertype__ip_addr_type='network').exists()) + Q(server__servertype__ip_addr_type='network') | + Q(server_id=object_id) + ).exists() or + server_attribute_cls.objects.filter(value=ip_interface).exclude( + Q(server__servertype__ip_addr_type='network') | + Q(server_id=object_id) + ).exists() + ) if has_duplicates: raise ValidationError( 'An object with {0} already exists'.format(str(ip_interface))) @@ -151,6 +209,20 @@ def inet_to_python(obj: object) -> Union[IPv4Interface, IPv6Interface]: except ValueError as error: raise ValidationError(str(error)) +# WARNING: called only for edit->commit, not for commit! +def inet4_to_python(obj: object) -> IPv4Interface: + try: + return IPv4Interface(obj) + except (AddressValueError, NetmaskValueError): + raise ValidationError(f'{obj} does not appear to be an IPv4 interface') + + +def inet6_to_python(obj: object) -> IPv4Interface: + try: + return IPv6Interface(obj) + except (AddressValueError, NetmaskValueError): + raise ValidationError(f'{obj} does not appear to be an IPv6 interface') + def network_overlaps(ip_interface: Union[IPv4Interface, IPv6Interface], servertype_id: str, object_id: int) -> None: @@ -226,6 +298,7 @@ def __init__(self, *args, **kwargs): max_length=32, choices=get_choices(ATTRIBUTE_TYPES.keys()), ) + supernet = models.BooleanField(null=False, default=False) multi = models.BooleanField(null=False, default=False) hovertext = models.TextField(null=False, blank=True, default='') group = models.CharField( @@ -438,11 +511,52 @@ def __str__(self): return self.hostname def get_supernet(self, servertype): - return Server.objects.get( + """Get a supernet of given servertype for the current server. + + This function will check all IP addresses of a server which have the "supernet" feature enabled. + If data is inconsistent, an exception is raised. + No matching network for just some of addresses does not mean inconsitency. + """ + + supernet_1 = None + + # TODO: Remove "intern_ip" support. + supernet_1 = Server.objects.get( servertype=servertype, + # It should probably match on ip_addr_type too, but we will remove this soon anyway. intern_ip__net_contains_or_equals=self.intern_ip, ) + for server_attribute_cls in (ServerInet4Attribute, ServerInet6Attribute): + ip_addresses = server_attribute_cls.objects.filter( + server_id=self.server_id, + attribute__supernet=True, + ) + + # TODO: How to net_contains_or_equals for iterable? + for ip_address in ip_addresses: + try: + attr = server_attribute_cls.objects.get( + value__net_contains_or_equals=ip_address.value, + server__servertype__ip_addr_type='network', + server__servertype_id=servertype.servertype_id, + ) + if not attr.attribute.supernet: + raise ValidationError(f'Not a supernet: {servertype}!') + supernet_2 = attr.server + # TODO: Shouldn't we check that the requested servertype really point + except server_attribute_cls.DoesNotExist: + continue + else: + # Always trust the 1st found network + if supernet_1 is None: + supernet_1 = supernet_2 + # Verify that all found networks match the 1st found one. + elif supernet_1 != supernet_2: + raise ValidationError(f'Can\'t determine {servertype} for {self.hostname}!') + + return supernet_1 + def clean(self): super(Server, self).clean() @@ -474,6 +588,9 @@ def clean(self): network_overlaps(self.intern_ip, self.servertype.servertype_id, self.server_id) + if ip_addr_type != 'null': + is_supernet_consistent(self.intern_ip, self.server) + def get_attributes(self, attribute): model = ServerAttribute.get_model(attribute.type) return model.objects.filter(server=self, attribute=attribute) @@ -520,6 +637,10 @@ def get_model(attribute_type): return ServerNumberAttribute if attribute_type == 'inet': return ServerInetAttribute + if attribute_type == 'inet4': + return ServerInet4Attribute + if attribute_type == 'inet6': + return ServerInet6Attribute if attribute_type == 'macaddr': return ServerMACAddressAttribute if attribute_type == 'date': @@ -695,6 +816,96 @@ def clean(self): network_overlaps(self.value, self.server.servertype_id, self.server.server_id) + is_supernet_consistent(self.value, self.server) + + +class ServerInet4Attribute(ServerAttribute): + attribute = models.ForeignKey( + Attribute, + db_index=False, + on_delete=models.CASCADE, + limit_choices_to=dict(type='inet4'), + ) + value = netfields.InetAddressField() + + class Meta: + app_label = 'serverdb' + db_table = 'server_inet4_attribute' + unique_together = [['server', 'attribute', 'value']] + index_together = [['attribute', 'value']] + + def clean(self): + super(ServerAttribute, self).clean() + + if type(self.value) != IPv4Interface: + self.value = inet4_to_python(self.value) + + # Get the ip_addr_type of the servertype + ip_addr_type = self.server.servertype.ip_addr_type + + if ip_addr_type == 'null': + # A Servertype with ip_addr_type "null" and attributes of type + # inet must be denied per configuration. This is just a safety net + # in case e.g. somebody creates them programmatically. + raise ValidationError( + _('%(attribute_id)s must be null'), code='invalid value', + params={'attribute_id': self.attribute_id}) + elif ip_addr_type == 'host': + is_ip_address(self.value) + is_unique_ip(self.value, self.server.server_id) + elif ip_addr_type == 'loadbalancer': + is_ip_address(self.value) + elif ip_addr_type == 'network': + is_network(self.value) + network_overlaps(self.value, self.server.servertype_id, + self.server.server_id) + + is_supernet_consistent(self.value, self.server) + + +class ServerInet6Attribute(ServerAttribute): + attribute = models.ForeignKey( + Attribute, + db_index=False, + on_delete=models.CASCADE, + limit_choices_to=dict(type='inet6'), + ) + value = netfields.InetAddressField() + + class Meta: + app_label = 'serverdb' + db_table = 'server_inet6_attribute' + unique_together = [['server', 'attribute', 'value']] + index_together = [['attribute', 'value']] + + def clean(self): + super(ServerAttribute, self).clean() + + if type(self.value) != IPv6Interface: + self.value = inet6_to_python(self.value) + + # Get the ip_addr_type of the servertype + ip_addr_type = self.server.servertype.ip_addr_type + + if ip_addr_type == 'null': + # A Servertype with ip_addr_type "null" and attributes of type + # inet must be denied per configuration. This is just a safety net + # in case e.g. somebody creates them programmatically. + raise ValidationError( + _('%(attribute_id)s must be null'), code='invalid value', + params={'attribute_id': self.attribute_id}) + elif ip_addr_type == 'host': + is_ip_address(self.value) + is_unique_ip(self.value, self.server.server_id) + elif ip_addr_type == 'loadbalancer': + is_ip_address(self.value) + elif ip_addr_type == 'network': + is_network(self.value) + network_overlaps(self.value, self.server.servertype_id, + self.server.server_id) + + is_supernet_consistent(self.value, self.server) + class ServerMACAddressAttribute(ServerAttribute): attribute = models.ForeignKey( diff --git a/serveradmin/serverdb/query_materializer.py b/serveradmin/serverdb/query_materializer.py index 54232b8e..f3c9996b 100644 --- a/serveradmin/serverdb/query_materializer.py +++ b/serveradmin/serverdb/query_materializer.py @@ -9,6 +9,8 @@ from ipaddress import IPv4Address, IPv6Address +from django.core.exceptions import ValidationError + from adminapi.dataset import DatasetObject from serveradmin.serverdb.models import ( Servertype, @@ -16,7 +18,7 @@ ServertypeAttribute, Server, ServerAttribute, - ServerRelationAttribute, + ServerRelationAttribute, ServerInet4Attribute, ServerInet6Attribute, ) @@ -116,12 +118,8 @@ def _add_attributes(self, servers_by_type): """Add the attributes to the results""" for key, attributes in self._attributes_by_type.items(): if key == 'supernet': - for attribute in attributes: - self._add_supernet_attribute(attribute, ( - s - for st in self._servertype_ids_by_attribute[attribute] - for s in servers_by_type[st] - )) + # Add supernets only after all other attributes. + continue elif key == 'domain': for attribute in attributes: self._add_domain_attribute(attribute, [ @@ -160,6 +158,15 @@ def _add_attributes(self, servers_by_type): sa.get_value(), ) + for key, attributes in self._attributes_by_type.items(): + if key == 'supernet': + for attribute in attributes: + self._add_supernet_attribute(attribute, ( + s + for st in self._servertype_ids_by_attribute[attribute] + for s in servers_by_type[st] + )) + def _add_related_attributes(self, servers_by_type): for attribute, sa in self._related_servertype_attributes: self._add_related_attribute(attribute, sa, servers_by_type) @@ -185,24 +192,93 @@ def _add_supernet_attribute(self, attribute, servers): This function takes advantage of networks in the same servertype not overlapping with each other. """ - target = None + supernets = {} for source in sorted(servers, key=lambda s: _sort_key(s.intern_ip)): - # Check the previous target - if target is not None: - network = target.intern_ip.network - if network.version != source.intern_ip.version: - target = None - elif network.broadcast_address < source.intern_ip.ip: - target = None - elif source.intern_ip not in network: + # TODO: We have a working get_supernet() now, why not use it instead? + # TODO: We can't depend on servers being sorted anymore. + # TODO: Remove "intern_ip" support. + # There never was nor is "primary_ip6" support. + supernets['intern_ip'] = { + 'source': source.intern_ip, + 'target': None, + } + + for net_attribute in self._server_attributes[source]: + # Find server attributes which can be used for calculating supernets. + # This allows for having "primary" ip addresses used for supernet calculation + # and "additional" ip address which won't be used here. + # FIXME: We never check if there's more than 1 attribute per address family. + if net_attribute.supernet == False: continue - # Check for a new target - if target is None and source.intern_ip: - try: - target = source.get_supernet(attribute.target_servertype) - except Server.DoesNotExist: + net_attribute_value = self._server_attributes.get(source).get(net_attribute) + if net_attribute_value is None: continue - self._server_attributes[source][attribute] = target + source_address = self._server_attributes[source][net_attribute] + supernets[net_attribute] = { + 'source': source_address, + 'target': None, + } + + for supernet_k, supernet_v in supernets.items(): + # Check the previous target + if supernet_v['target'] is not None: + network = supernet_v['source'].network + if network.version != supernet_v['source'].version: + supernet_v['target'] = None + elif network.broadcast_address < supernet_v['source'].ip: + supernet_v['target'] = None + elif supernet_v['source'] not in network: + continue + # Check for a new target + if supernet_v['target'] is None and supernet_v['source']: + if supernet_k == 'intern_ip': + try: + supernet_v['target'] = source.get_supernet(attribute.target_servertype) + except Server.DoesNotExist: + continue + else: + if supernet_v['source'].version == 4: + server_attribute_cls = ServerInet4Attribute + else: + server_attribute_cls = ServerInet6Attribute + + try: + network = server_attribute_cls.objects.get( + value__net_contains_or_equals=supernet_v['source'], + server__servertype__ip_addr_type='network', + server__servertype=attribute.target_servertype, + ) + supernet_v['target'] = Server.objects.get(server_id=network.server_id) + except ( + Server.DoesNotExist, + ServerInet4Attribute.DoesNotExist, + ServerInet6Attribute.DoesNotExist, + ): + continue + + # Verify that the networks are consistent. Refuse to return data if they are not. + supernet_temp_k = None + supernet_temp_v = None + for supernet_k, supernet_v in supernets.items(): + # Ignore if target supernet is not found by given method. + # We might not have all data in yet, and we will be fine trusting "intern_ip". + if supernet_v['target'] is None: + continue + + # Trust the 1st found supernet. + if supernet_temp_v == None: + supernet_temp_k = supernet_k + supernet_temp_v = supernet_v + continue + + # Verify the 2nd and later supernets against the 1st one. + if supernet_v['target'] != supernet_temp_v['target']: + raise ValidationError( + f'Inconsistent {attribute} for {source}: ' + f'{supernet_temp_v["target"]} from {supernet_temp_k} vs ' + f'{supernet_v["target"]} from {supernet_k}!' + ) + self._server_attributes[source][attribute] = supernet_temp_v['target'] def _add_related_attribute( self, attribute, servertype_attribute, servers_by_type @@ -271,7 +347,7 @@ def _get_attributes(self, server, join_results): # NOQA: C901 if attribute not in self._joined_attributes: continue - if attribute.type == 'inet': + if attribute.type in ['inet', 'inet4', 'inet6']: if value is None: yield attribute.attribute_id, None else: diff --git a/serveradmin/serverdb/sql_generator.py b/serveradmin/serverdb/sql_generator.py index a91918df..661015c5 100644 --- a/serveradmin/serverdb/sql_generator.py +++ b/serveradmin/serverdb/sql_generator.py @@ -173,7 +173,7 @@ def _containment_filter_template(attribute, filt): template = None # To be formatted 2 times value = filt.value - if attribute.type == 'inet': + if attribute.type in ['inet', 'inet4', 'inet6']: if isinstance(filt, StartsWith): template = "{{0}} >>= {0} AND host({{0}}) = host(0{})" elif isinstance(filt, Contains):