diff --git a/pythonlibs/mt_api/__init__.py b/pythonlibs/mt_api/__init__.py new file mode 100644 index 0000000..877e416 --- /dev/null +++ b/pythonlibs/mt_api/__init__.py @@ -0,0 +1,409 @@ +from __future__ import unicode_literals + +import binascii +import hashlib +import logging +import socket +import ssl +import sys + +from .retryloop import RetryError +from .retryloop import retryloop +from .socket_utils import set_keepalive + +PY2 = sys.version_info[0] < 3 +logger = logging.getLogger(__name__) + + +class RosAPIError(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + if isinstance(self.value, dict) and self.value.get('message'): + return self.value['message'] + elif isinstance(self.value, list): + elements = ( + '%s: %s' % + (element.__class__, str(element)) for element in self.value + ) + return '[%s]' % (', '.join(element for element in elements)) + else: + return str(self.value) + + +class RosAPIConnectionError(RosAPIError): + pass + + +class RosAPIFatalError(RosAPIError): + pass + + +class RosApiLengthUtils(object): + def __init__(self, api): + self.api = api + + def write_lenght(self, length): + self.api.write_bytes(self.length_to_bytes(length)) + + def length_to_bytes(self, length): + if length < 0x80: + return self.to_bytes(length) + elif length < 0x4000: + length |= 0x8000 + return self.to_bytes(length, 2) + elif length < 0x200000: + length |= 0xC00000 + return self.to_bytes(length, 3) + elif length < 0x10000000: + length |= 0xE0000000 + return self.to_bytes(length, 4) + else: + return self.to_bytes(0xF0) + self.to_bytes(length, 4) + + def read_length(self): + b = self.api.read_bytes(1) + i = self.from_bytes(b) + if (i & 0x80) == 0x00: + return i + elif (i & 0xC0) == 0x80: + return self._unpack(1, i & ~0xC0) + elif (i & 0xE0) == 0xC0: + return self._unpack(2, i & ~0xE0) + elif (i & 0xF0) == 0xE0: + return self._unpack(3, i & ~0xF0) + elif (i & 0xF8) == 0xF0: + return self.from_bytes(self.api.read_bytes(1)) + else: + raise RosAPIFatalError('Unknown value: %x' % i) + + def _unpack(self, times, i): + temp1 = self.to_bytes(i) + temp2 = self.api.read_bytes(times) + try: + temp3 = temp2.decode('utf-8') + except: + try: + temp3 = temp2.decode('windows-1252') + except Exception: + print("Cannot decode response properly:", temp2) + print(Exception) + exit(1) + + res = temp1 + temp3 + return self.from_bytes(res) + + if PY2: + def from_bytes(self, data): + data_values = [ord(char) for char in data] + value = 0 + for byte_value in data_values: + value <<= 8 + value += byte_value + return value + + def to_bytes(self, i, size=1): + data = [] + for _ in xrange(size): + data.append(chr(i & 0xff)) + i >>= 8 + return ''.join(reversed(data)) + else: + def from_bytes(self, data): + return int.from_bytes(data, 'big') + + def to_bytes(self, i, size=1): + return i.to_bytes(size, 'big') + + +class RosAPI(object): + """Routeros api""" + + def __init__(self, socket): + self.socket = socket + self.length_utils = RosApiLengthUtils(self) + + def login(self, username, pwd): + for _, attrs in self.talk([b'/login']): + token = binascii.unhexlify(attrs[b'ret']) + hasher = hashlib.md5() + hasher.update(b'\x00') + hasher.update(pwd) + hasher.update(token) + self.talk([b'/login', b'=name=' + username, + b'=response=00' + hasher.hexdigest().encode('ascii')]) + + def talk(self, words): + if self.write_sentence(words) == 0: + return + output = [] + while True: + input_sentence = self.read_sentence() + if not len(input_sentence): + continue + attrs = {} + reply = input_sentence.pop(0) + for line in input_sentence: + try: + second_eq_pos = line.index(b'=', 1) + except IndexError: + attrs[line[1:]] = b'' + else: + attrs[line[1:second_eq_pos]] = line[second_eq_pos + 1:] + output.append((reply, attrs)) + if reply == b'!done': + if output[0][0] == b'!trap': + raise RosAPIError(output[0][1]) + if output[0][0] == b'!fatal': + self.socket.close() + raise RosAPIFatalError(output[0][1]) + return output + + def write_sentence(self, words): + words_written = 0 + for word in words: + self.write_word(word) + words_written += 1 + self.write_word(b'') + return words_written + + def read_sentence(self): + sentence = [] + while True: + word = self.read_word() + if not len(word): + return sentence + sentence.append(word) + + def write_word(self, word): + logger.debug('>>> %s' % word) + self.length_utils.write_lenght(len(word)) + self.write_bytes(word) + + def read_word(self): + word = self.read_bytes(self.length_utils.read_length()) + logger.debug('<<< %s' % word) + return word + + def write_bytes(self, data): + sent_overal = 0 + while sent_overal < len(data): + try: + sent = self.socket.send(data[sent_overal:]) + except socket.error as e: + raise RosAPIConnectionError(str(e)) + if sent == 0: + raise RosAPIConnectionError('Connection closed by remote end.') + sent_overal += sent + + def read_bytes(self, length): + received_overal = b'' + while len(received_overal) < length: + try: + received = self.socket.recv( + length - len(received_overal)) + except socket.error as e: + raise RosAPIConnectionError(str(e)) + if len(received) == 0: + raise RosAPIConnectionError('Connection closed by remote end.') + received_overal += received + return received_overal + + + + +class BaseRouterboardResource(object): + def __init__(self, api, namespace): + self.api = api + self.namespace = namespace + + def call(self, command, set_kwargs, query_kwargs=None): + query_kwargs = query_kwargs or {} + query_arguments = self._prepare_arguments(True, **query_kwargs) + set_arguments = self._prepare_arguments(False, **set_kwargs) + query = ([('%s/%s' % (self.namespace, command)).encode('ascii')] + + query_arguments + set_arguments) + response = self.api.api_client.talk(query) + + output = [] + for response_type, attributes in response: + if response_type == b'!re': + output.append(self._remove_first_char_from_keys(attributes)) + + return output + + @staticmethod + def _prepare_arguments(is_query, **kwargs): + command_arguments = [] + for key, value in kwargs.items(): + if key in ['id', 'proplist']: + key = '.%s' % key + key = key.replace('_', '-') + selector_char = '?' if is_query else '=' + command_arguments.append( + ('%s%s=' % (selector_char, key)).encode('ascii') + value) + + return command_arguments + + @staticmethod + def _remove_first_char_from_keys(dictionary): + elements = [] + for key, value in dictionary.items(): + key = key.decode('ascii') + if key in ['.id', '.proplist']: + key = key[1:] + elements.append((key, value)) + return dict(elements) + + def get(self, **kwargs): + return self.call('print', {}, kwargs) + + def detailed_get(self, **kwargs): + return self.call('print', {'detail': b''}, kwargs) + + def set(self, **kwargs): + return self.call('set', kwargs) + + def add(self, **kwargs): + return self.call('add', kwargs) + + def remove(self, **kwargs): + return self.call('remove', kwargs) + + +class RouterboardResource(BaseRouterboardResource): + def detailed_get(self, **kwargs): + return self.call('print', {'detail': ''}, kwargs) + + def call(self, command, set_kwargs, query_kwargs=None): + query_kwargs = query_kwargs or {} + result = super(RouterboardResource, self).call( + command, self._encode_kwargs(set_kwargs), + self._encode_kwargs(query_kwargs)) + for item in result: + for k in item: + item[k] = item[k].decode('ascii') + return result + + def _encode_kwargs(self, kwargs): + return dict((k, v.encode('ascii')) for k, v in kwargs.items()) + + +class RouterboardAPI(object): + def __init__(self, host, username='api', password='', port=8728, ssl=False): + self.host = host + self.username = username + self.password = password + self.socket = None + self.port = port + self.ssl = ssl + self.reconnect() + + def __enter__(self): + return self + + def __exit__(self, _, __, ___): + self.close_connection() + + def reconnect(self): + if self.socket: + self.close_connection() + try: + for retry in retryloop(10, delay=0.1, timeout=30): + try: + self.connect() + self.login() + except socket.error: + retry() + except (socket.error, RetryError) as e: + raise RosAPIConnectionError(str(e)) + + def connect(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(15.0) + sock.connect((self.host, self.port)) + set_keepalive(sock, after_idle_sec=10) + if self.ssl: + try: + self.socket = ssl.wrap_socket(sock) + except ssl.SSLError as e: + raise RosAPIConnectionError(str(e)) + else: + self.socket = sock + self.api_client = RosAPI(self.socket) + + def login(self): + self.api_client.login(self.username.encode('ascii'), + self.password.encode('ascii')) + + def get_resource(self, namespace): + return RouterboardResource(self, namespace) + + def get_base_resource(self, namespace): + return BaseRouterboardResource(self, namespace) + + def close_connection(self): + self.socket.close() + + +class Mikrotik(object): + + def __init__(self, hostname, username, password): + self.hostname = hostname + self.username = username + self.password = password + + def login(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((self.hostname, 8728)) + mt = RosAPI(s) + mt.login(self.username, self.password) + return mt + + def talk(self, talk_command): + r = self.login() + response = r.talk(talk_command) + return(response) + + def api_print(self, base_path, params=None): + command = [base_path + '/print'] + if params is not None: + for key, value in params.iteritems(): + item = b'=' + key + '=' + str(value) + command.append(item) + + return self.talk(command) + + def api_add(self, base_path, params): + command = [base_path + '/add'] + for key, value in params.iteritems(): + item = b'=' + key + '=' + str(value) + command.append(item) + + return self.talk(command) + + def api_edit(self, base_path, params): + command = [base_path + '/set'] + for key, value in params.iteritems(): + item = b'=' + key + '=' + str(value) + command.append(item) + + return self.talk(command) + + def api_remove(self, base_path, remove_id): + command = [ + base_path + '/remove', + b'=.id=' + remove_id + ] + + return self.talk(command) + + def api_command(self, base_path, params=None): + command = [base_path] + if params is not None: + for key, value in params.iteritems(): + item = b'=' + key + '=' + str(value) + command.append(item) + + return self.talk(command) diff --git a/pythonlibs/mt_api/retryloop.py b/pythonlibs/mt_api/retryloop.py new file mode 100644 index 0000000..43061dc --- /dev/null +++ b/pythonlibs/mt_api/retryloop.py @@ -0,0 +1,38 @@ +# retry loop from http://code.activestate.com/recipes/578163-retry-loop/ +import time +import sys + + +class RetryError(Exception): + pass + + +def retryloop(attempts, timeout=None, delay=0, backoff=1): + starttime = time.time() + success = set() + for i in range(attempts): + success.add(True) + yield success.clear + if success: + return + duration = time.time() - starttime + if timeout is not None and duration > timeout: + break + if delay: + time.sleep(delay) + delay *= backoff + + e = sys.exc_info()[1] + + # No pending exception? Make one + if e is None: + try: + raise RetryError + except RetryError as exc: + e = exc + + # Decorate exception with retry information: + e.args = e.args + ("on attempt {0} of {1} after {2:.3f} seconds".format( + i + 1, attempts, duration), ) + + raise e diff --git a/pythonlibs/mt_api/socket_utils.py b/pythonlibs/mt_api/socket_utils.py new file mode 100644 index 0000000..9d1709b --- /dev/null +++ b/pythonlibs/mt_api/socket_utils.py @@ -0,0 +1,15 @@ +import socket + + +# http://stackoverflow.com/a/14855726 +def set_keepalive(sock, after_idle_sec=1, interval_sec=3, max_fails=5): + """Set TCP keepalive on an open socket. + + It activates after 1 second (after_idle_sec) of idleness, + then sends a keepalive ping once every 3 seconds (interval_sec), + and closes the connection after 5 failed ping (max_fails), or 15 seconds + """ + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, after_idle_sec) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, max_fails) diff --git a/pythonlibs/mt_common.py b/pythonlibs/mt_common.py new file mode 100644 index 0000000..672f45f --- /dev/null +++ b/pythonlibs/mt_common.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +import mt_api +import re + + +def clean_params(params): + ''' + remove keys with empty values + modify keys with '_' to match mikrotik parameters + convert yes/no to true/false + ''' + if isinstance(params, dict): + for key in list(params): + if params[key] is None: + del params[key] + continue + + new_key = re.sub('_', '-', key) + if new_key != key: + params[new_key] = str(params[key]) + del params[key] + continue + + if params[key] == "yes": + params[key] = "true" + if params[key] == "no": + params[key] = "false" + else: + print("Must be a dictionary") + + +class MikrotikIdempotent(): + ''' + MikrotikIdempotent Class + - A helper class for Ansible modules to abstract common functions. + + Example Usage: + mt_obj = MikrotikIdempotent( + hostname = params['hostname'], + username = params['username'], + password = params['password'], + state = None, + desired_params = params['settings'], + idempotent_param= 'name', + api_path = '/interface/ethernet', + ) + + mt_obj.sync_state() + ''' + + def __init__( + self, hostname, username, password, desired_params, api_path, + state, idempotent_param, check_mode=False): + + self.hostname = hostname + self.username = username + self.password = password + self.state = state + self.desired_params = desired_params + self.idempotent_param = idempotent_param + self.current_params = {} + self.api_path = api_path + self.check_mode = check_mode + + self.login_success = False + self.changed = False + self.changed_msg = [] + self.failed = False + self.failed_msg = [] + + self.login() + + def login(self): + self.mk = mt_api.Mikrotik( + self.hostname, + self.username, + self.password, + ) + + try: + self.mk.login() + self.login_success = True + except: + self.failed_msg = "Could not log into Mikrotik device." + " Check the username and password.", + + def get_current_params(self): + clean_params(self.desired_params) + self.param_id = None + self.current_param = None + self.current_params = self.mk.api_print(base_path=self.api_path) + + # When state and idempotent_param is None we are working + # on editable params only and we are grabbing the only item from response + if self.state is None and self.idempotent_param is None: + self.current_param = self.current_params[0][1] + + # Else we iterate over every item in the list until we find the matching + # params + # We also set the param_id here to reference later for editing or removal + else: + for current_param in self.current_params: + if self.idempotent_param in current_param[1]: + if self.desired_params[self.idempotent_param] == current_param[1][self.idempotent_param]: + self.current_param = current_param[1] + self.param_id = current_param[1]['.id'] + # current_param now is a dict, something like: + # { + # ".id": "*1", + # "full-duplex": "true", + # "mac-address": "08:00:27:6F:4C:22", + # "mtu": "1500", + # "name": "ether1", + # ... + # } + + def add(self): + # When current_param is empty we need to call api_add method to add + # all the parameters in the desired_params + if self.current_param is None: + self.new_params = self.desired_params + self.old_params = "" + if not self.check_mode: + self.mk.api_add( + base_path = self.api_path, + params = self.desired_params, + ) + self.changed = True + + # Else we need to determing what the difference between the currently + # and the desired + else: + self.edit() + + def rem(self): + # if param_id is set this means there is currently a matching item + # which we will remove + if self.param_id: + self.new_params = "item removed" + self.old_params = self.desired_params + if not self.check_mode: + self.mk.api_remove( + base_path=self.api_path, + remove_id=self.param_id, + ) + self.changed = True + + def edit(self): + out_params = {} + old_params = {} #used to store values of params we change + + # iterate over items in desired params and match against items in current_param + # to figure out the difference + for desired_param in self.desired_params: + self.desired_params[desired_param] = str(self.desired_params[desired_param]) + if desired_param in self.current_param: + if self.current_param[desired_param] != self.desired_params[desired_param]: + out_params[desired_param] = self.desired_params[desired_param] + old_params[desired_param] = self.current_param[desired_param] + else: + out_params[desired_param] = self.desired_params[desired_param] + if desired_param in self.current_param: + old_params[desired_param] = self.current_param[desired_param] + + # When out_params has been set it means we found our diff + # and will set it on the mikrotik + if out_params: + if self.param_id is not None: + out_params['.id'] = self.current_param['.id'] + + if not self.check_mode: + self.mk.api_edit( + base_path = self.api_path, + params = out_params, + ) + + # we don't need to show the .id in the changed message + if '.id' in out_params: + del out_params['.id'] + + self.changed_msg.append({ + "new_params": out_params, + "old_params": old_params, + }) + + self.new_params = out_params + self.old_params = old_params + self.changed = True + + def sync_state(self): + self.get_current_params() + + # When state and idempotent_param are not set we are working + # on editable parameters only that we can't add or remove + if self.state is None and self.idempotent_param is None: + self.edit() + elif self.state == "absent": + self.rem() + elif self.state == "present" or self.idempotent_param: + self.add()