From 4b791be9f6806b666c64b75d1aea6662c1271a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Sat, 6 Feb 2016 14:24:26 +0100 Subject: [PATCH] Add full IPV6 support --- data/Makefile | 4 +- data/build_c_array.py | 225 ++++++++++++++++++++++++++---------------- src/ip_to_geo.c | 4 +- 3 files changed, 146 insertions(+), 87 deletions(-) diff --git a/data/Makefile b/data/Makefile index 476364a..571b7ce 100644 --- a/data/Makefile +++ b/data/Makefile @@ -1,6 +1,6 @@ IP_DATA = ../src/ip_data.c PROVIDERS = afrinic arin apnic lacnic ripencc -DEST = prefix_res_ipv4 +DEST = prefix_res MD5 = $(addsuffix .md5,$(PROVIDERS)) all: clean_md5 $(MD5) $(IP_DATA) @@ -21,7 +21,7 @@ $(PROVIDERS): @md5sum -c $@ || wget "ftp://ftp.ripe.net/pub/stats/$(basename $@)/delegated-$(basename $@)-extended-latest" -O $(basename $@) ; true $(DEST): $(PROVIDERS) - @cat $(PROVIDERS) | grep -v asn | grep -v summary | grep -v '#' | grep ipv4 | sort -n -k 4 -t '|' > $(DEST) + @cat $(PROVIDERS) | grep -v asn | grep -v summary | grep -v '#' | grep ipv[46] | sort -n -k 4 -t '|' > $(DEST) $(IP_DATA): $(DEST) @echo "Rebuild ip_data.c" diff --git a/data/build_c_array.py b/data/build_c_array.py index 4514b96..52ec32c 100755 --- a/data/build_c_array.py +++ b/data/build_c_array.py @@ -8,7 +8,7 @@ IP_TYPE_INDEX=2 IP_INDEX=3 IP_SIZE_INDEX=4 -class IP_ELEMENT(object): +class IP_ELEMENT(object): def __init__(self, start, end=None, size=0, country_code=None, level=0): self._start = start self._end = end @@ -22,39 +22,38 @@ class IP_ELEMENT(object): if not self._end: self._compute_last_ip() - self._splitted_start = IP_ELEMENT.split_ip(self._start) - self._splitted_end = IP_ELEMENT.split_ip(self._end) + self._splitted_start = self.split_ip(self._start) + self._splitted_end = self.split_ip(self._end) + + def split_ip(self, ip): + return [int(x, self._base) for x in ip.split(self._separator)] + + def ip_to_str(self, int_ip): + res = [] + for i in range(0, self.get_ip_len()): + res.insert(0, self._format % int((int_ip >> (i*8)) & 0xFF)) + return self._separator.join(res) + + def ip_array_to_int(self, array): + val = 0 + for i in range(0, len(array)): + val += array[len(array)-i-1] << (i*8) + return val + + def ip_to_int(self, str_ip): + return self.ip_array_to_int(self.split_ip(str_ip)) + + def make_group(self): + ip_val = self._splitted_start[::] + for i in range(self._level+1, self.get_ip_len()): + ip_val[i] = 0 + return self._separator.join([self._format % x for x in ip_val]) + + def name(self): + return 'ip__%s__%s' %(self._start.replace(self._separator, '_'), self._end.replace(self._separator, '_')) def _compute_last_ip(self): - size = self._size - end_ip = IP_ELEMENT.ip_to_int(self._start) - for i in range(0,4): - if not size: break # We can have _size == 0 - end_ip += (((size % 256)-1) & 0xFF) << (i*8) - size = int(size/256) - self._end = IP_ELEMENT.ip_to_str(end_ip) - # print '%s + %d -> %s' % (self._start, self._size, self._end) - - @staticmethod - def split_ip(ip): - return [int(x) for x in ip.split('.')] - - @staticmethod - def ip_to_int(str_ip): - splitted_ip = IP_ELEMENT.split_ip(str_ip) - val = splitted_ip[0] << 24 - val += splitted_ip[1] << 16 - val += splitted_ip[2] << 8 - val += splitted_ip[3] << 0 - return val - - @staticmethod - def ip_to_str(int_ip): - val = '%d.' % (int((int_ip >> 24) & 0xFF)) - val += '%d.' % (int((int_ip >> 16) & 0xFF)) - val += '%d.' % (int((int_ip >> 8) & 0xFF)) - val += '%d' % (int((int_ip >> 0) & 0xFF)) - return val + raise NotImplementedError() def set_next(self, ip): self._next = ip @@ -71,9 +70,6 @@ class IP_ELEMENT(object): def set_level(self, level): self._level = level - def name(self): - return 'ip__%s__%s' %(self._start.replace('.', '_'), self._end.replace('.', '_')) - def printme(self): print 'static const ip_level %s = {' % (self.name()) print '\t.prev = %s,' % (self._prev and '&%s' % (self._prev.name()) or 'NULL') @@ -85,13 +81,76 @@ class IP_ELEMENT(object): print '\t.code = %d,' % (self._country_code and self._country_code or 0) print '};' -countries = [] -ip_idx = [0] * 255 -cur_ip_prefix = 1 -cur_idx = 0 + def get_ip_len(self): + raise NotImplementedError() -f = open("prefix_res_ipv4") -array_vals = {} +class IP_ELEMENT4(IP_ELEMENT): + + def __init__(self, start, end=None, size=0, country_code=None, level=0): + self._separator = '.' + self._base = 10 + self._format = '%d' + super(IP_ELEMENT4, self).__init__(start, end, size, country_code, level) + + def get_ip_len(self): + return 4 + + def _compute_last_ip(self): + size = self._size + end_ip = self.ip_to_int(self._start) + i=0 + while size > 0: + end_ip += (((size % 256)-1) & 0xFF) << (i*8) + size = int(size/256) + i += 1 + self._end = self.ip_to_str(end_ip) + # print '%s + %d -> %s' % (self._start, self._size, self._end) + +class IP_ELEMENT6(IP_ELEMENT): + + def __init__(self, start, end=None, size=0, country_code=None, level=0): + self._separator = ':' + self._base = 16 + self._format = '%02x' + super(IP_ELEMENT6, self).__init__(start, end, size, country_code, level) + + def get_ip_len(self): + return 16 + + def _get_mask(self): + mask = 0 + for i in range(0, self._size): + mask += 1 << i + mask <<= 128-self._size + return mask + + def _compute_last_ip(self): + if self._size == 0: + self._end = self._start[:] + else: + mask = self._get_mask() + self._end = self.ip_to_str(self.ip_to_int(self._start) | ~mask) + +def extend_ipv6(ipv6): + tmp = '' + for s in ipv6.split(':'): + if not s: break + while len(s) != 4: + s = '0' + s + tmp += s + while len(tmp) < 16*2: + tmp += '0' + res = '' + for i in range(0, 15*2, 2): + res += tmp[i] + tmp[i+1] + ':' + res += tmp[30] + tmp[31] + return res + +countries = [] + +f = open("prefix_res") +array_vals_ipv4 = {} +array_vals_ipv6 = {} while True: l = f.readline() # l = sys.stdin.readline() @@ -108,27 +167,23 @@ while True: countries.append(country) ip = information[IP_INDEX] - splitted_ip = ip.split('.') - int_ip = int(splitted_ip[0]) << 24 - int_ip += int(splitted_ip[1]) << 16 - int_ip += int(splitted_ip[2]) << 8 - int_ip += int(splitted_ip[3]) << 0 - - interval_size = int(information[IP_SIZE_INDEX]) - array_vals[ip] = IP_ELEMENT(ip, None, interval_size, country_idx) + if information[IP_TYPE_INDEX] == 'ipv4': + array_vals_ipv4[ip] = IP_ELEMENT4(ip, None, int(information[IP_SIZE_INDEX]), country_idx) + elif information[IP_TYPE_INDEX] == 'ipv6': + ip = extend_ipv6(ip) + array_vals_ipv6[ip] = IP_ELEMENT6(ip, None, int(information[IP_SIZE_INDEX]), country_idx) + else: + sys.stderr.write('Unknown IP type %s\n' % (information[IP_TYPE_INDEX])) print '/* This file was automatically generated, do not edit it ! */' print '#include \n\n' def ip_sort(a, b): - for i in range(0, 4): + for i in range(0, len(a._splitted_start)): if a._splitted_start[i] != b._splitted_start[i]: return a._splitted_start[i] - b._splitted_start[i] return 0 -ip_list = array_vals.values() -ip_list.sort(ip_sort) - def get_interval(root, intervals, level): new_intervals = [] for ip in intervals: @@ -183,10 +238,7 @@ def manage_root(root, intervals, level): cur_ip.set_level(level+1) for ip in sub_interval: ip.set_level(level+1) - ip_val = IP_ELEMENT.ip_to_int(cur_ip._start) - for i in range(level, 3): - ip_val &= ~(0xFF << ((2-i)*8)) & 0xFFFFFFFF - new_group = IP_ELEMENT(IP_ELEMENT.ip_to_str(ip_val), level=level) + new_group = cur_ip.__class__(cur_ip.make_group(), level=level) sub_interval.insert(0, cur_ip) child = manage_root(cur_ip._splitted_start[level+1], sub_interval, level+1) new_group.set_childs(cur_ip) @@ -215,35 +267,40 @@ def print_ip(ip): while cur_ip: cur_ip.printme() cur_ip = cur_ip._next - -start_idx = 0 -end_idx = start_idx+1 -cur_interval = [ip_list[start_idx]] -root = ip_list[start_idx]._splitted_start[0] -root_ips = [None] * 256 -while True: - if end_idx >= len(ip_list): break - if ip_list[end_idx]._splitted_start[0] != root: - start_idx = end_idx - res = manage_root(root, cur_interval, 1) - print_ip(res) - root_ips[res._splitted_start[0]] = res - cur_interval = [ip_list[end_idx]] - root = ip_list[start_idx]._splitted_start[0] - else: - cur_interval.append(ip_list[end_idx]) - end_idx += 1 -res = manage_root(root, cur_interval, 1) -print_ip(res) +def build_array(ip_list, array_name): + ip_list.sort(ip_sort) + start_idx = 0 + end_idx = start_idx+1 + cur_interval = [ip_list[start_idx]] + root = ip_list[start_idx]._splitted_start[0] + root_ips = [None] * 256 -print 'static const ip_level* s_root_ip[256] = {' -for i in range(0, 256): - if root_ips[i]: - print '\t&%s,' % (root_ips[i].name()) - else: - print '\tNULL, // %d' % (i) -print '};\n' + while True: + if end_idx >= len(ip_list): break + if ip_list[end_idx]._splitted_start[0] != root: + start_idx = end_idx + res = manage_root(root, cur_interval, 1) + print_ip(res) + root_ips[res._splitted_start[0]] = res + cur_interval = [ip_list[end_idx]] + root = ip_list[start_idx]._splitted_start[0] + else: + cur_interval.append(ip_list[end_idx]) + end_idx += 1 + res = manage_root(root, cur_interval, 1) + print_ip(res) + + print '\nstatic const ip_level* %s[256] = {' % (array_name) + for i in range(0, 256): + if root_ips[i]: + print '\t&%s,' % (root_ips[i].name()) + else: + print '\tNULL, // %d' % (i) + print '};\n' + +build_array(array_vals_ipv4.values(), 's_root_ipv4') +build_array(array_vals_ipv6.values(), 's_root_ipv6') print 'static const uint8_t country_codes[][3] = {' for cc in countries: diff --git a/src/ip_to_geo.c b/src/ip_to_geo.c index a009e1b..0906624 100644 --- a/src/ip_to_geo.c +++ b/src/ip_to_geo.c @@ -70,7 +70,9 @@ const uint8_t* ip_to_geo(uint8_t* ip, unsigned ip_size) const ip_level* first_level; if (ip_size == 4) - first_level = s_root_ip[ip[0]]; + first_level = s_root_ipv4[ip[0]]; + else if (ip_size == 16) + first_level = s_root_ipv6[ip[0]]; else return NULL;