add mt_api library

pythonlibs/mt_api/__init__.py

@@ -0,0 +1,409 @@
+from __future__ import unicode_literals
+
+import binascii
+import hashlib
+import logging
+import socket
+import ssl
+import sys
+
+from .retryloop import RetryError
+from .retryloop import retryloop
+from .socket_utils import set_keepalive
+
+PY2 = sys.version_info[0] < 3
+logger = logging.getLogger(__name__)
+
+
+class RosAPIError(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        if isinstance(self.value, dict) and self.value.get('message'):
+            return self.value['message']
+        elif isinstance(self.value, list):
+            elements = (
+                '%s: %s' %
+                (element.__class__, str(element)) for element in self.value
+            )
+            return '[%s]' % (', '.join(element for element in elements))
+        else:
+            return str(self.value)
+
+
+class RosAPIConnectionError(RosAPIError):
+    pass
+
+
+class RosAPIFatalError(RosAPIError):
+    pass
+
+
+class RosApiLengthUtils(object):
+    def __init__(self, api):
+        self.api = api
+
+    def write_lenght(self, length):
+        self.api.write_bytes(self.length_to_bytes(length))
+
+    def length_to_bytes(self, length):
+        if length < 0x80:
+            return self.to_bytes(length)
+        elif length < 0x4000:
+            length |= 0x8000
+            return self.to_bytes(length, 2)
+        elif length < 0x200000:
+            length |= 0xC00000
+            return self.to_bytes(length, 3)
+        elif length < 0x10000000:
+            length |= 0xE0000000
+            return self.to_bytes(length, 4)
+        else:
+            return self.to_bytes(0xF0) + self.to_bytes(length, 4)
+
+    def read_length(self):
+        b = self.api.read_bytes(1)
+        i = self.from_bytes(b)
+        if (i & 0x80) == 0x00:
+            return i
+        elif (i & 0xC0) == 0x80:
+            return self._unpack(1, i & ~0xC0)
+        elif (i & 0xE0) == 0xC0:
+            return self._unpack(2, i & ~0xE0)
+        elif (i & 0xF0) == 0xE0:
+            return self._unpack(3, i & ~0xF0)
+        elif (i & 0xF8) == 0xF0:
+            return self.from_bytes(self.api.read_bytes(1))
+        else:
+            raise RosAPIFatalError('Unknown value: %x' % i)
+
+    def _unpack(self, times, i):
+        temp1 = self.to_bytes(i)
+        temp2 = self.api.read_bytes(times)
+        try:
+            temp3 = temp2.decode('utf-8')
+        except:
+            try:
+                temp3 = temp2.decode('windows-1252')
+            except Exception:
+                print("Cannot decode response properly:", temp2)
+                print(Exception)
+                exit(1)
+
+        res = temp1 + temp3
+        return self.from_bytes(res)
+
+    if PY2:
+        def from_bytes(self, data):
+            data_values = [ord(char) for char in data]
+            value = 0
+            for byte_value in data_values:
+                value <<= 8
+                value += byte_value
+            return value
+
+        def to_bytes(self, i, size=1):
+            data = []
+            for _ in xrange(size):
+                data.append(chr(i & 0xff))
+                i >>= 8
+            return ''.join(reversed(data))
+    else:
+        def from_bytes(self, data):
+            return int.from_bytes(data, 'big')
+
+        def to_bytes(self, i, size=1):
+            return i.to_bytes(size, 'big')
+
+
+class RosAPI(object):
+    """Routeros api"""
+
+    def __init__(self, socket):
+        self.socket = socket
+        self.length_utils = RosApiLengthUtils(self)
+
+    def login(self, username, pwd):
+        for _, attrs in self.talk([b'/login']):
+            token = binascii.unhexlify(attrs[b'ret'])
+        hasher = hashlib.md5()
+        hasher.update(b'\x00')
+        hasher.update(pwd)
+        hasher.update(token)
+        self.talk([b'/login', b'=name=' + username,
+                   b'=response=00' + hasher.hexdigest().encode('ascii')])
+
+    def talk(self, words):
+        if self.write_sentence(words) == 0:
+            return
+        output = []
+        while True:
+            input_sentence = self.read_sentence()
+            if not len(input_sentence):
+                continue
+            attrs = {}
+            reply = input_sentence.pop(0)
+            for line in input_sentence:
+                try:
+                    second_eq_pos = line.index(b'=', 1)
+                except IndexError:
+                    attrs[line[1:]] = b''
+                else:
+                    attrs[line[1:second_eq_pos]] = line[second_eq_pos + 1:]
+            output.append((reply, attrs))
+            if reply == b'!done':
+                if output[0][0] == b'!trap':
+                    raise RosAPIError(output[0][1])
+                if output[0][0] == b'!fatal':
+                    self.socket.close()
+                    raise RosAPIFatalError(output[0][1])
+                return output
+
+    def write_sentence(self, words):
+        words_written = 0
+        for word in words:
+            self.write_word(word)
+            words_written += 1
+        self.write_word(b'')
+        return words_written
+
+    def read_sentence(self):
+        sentence = []
+        while True:
+            word = self.read_word()
+            if not len(word):
+                return sentence
+            sentence.append(word)
+
+    def write_word(self, word):
+        logger.debug('>>> %s' % word)
+        self.length_utils.write_lenght(len(word))
+        self.write_bytes(word)
+
+    def read_word(self):
+        word = self.read_bytes(self.length_utils.read_length())
+        logger.debug('<<< %s' % word)
+        return word
+
+    def write_bytes(self, data):
+        sent_overal = 0
+        while sent_overal < len(data):
+            try:
+                sent = self.socket.send(data[sent_overal:])
+            except socket.error as e:
+                raise RosAPIConnectionError(str(e))
+            if sent == 0:
+                raise RosAPIConnectionError('Connection closed by remote end.')
+            sent_overal += sent
+
+    def read_bytes(self, length):
+        received_overal = b''
+        while len(received_overal) < length:
+            try:
+                received = self.socket.recv(
+                    length - len(received_overal))
+            except socket.error as e:
+                raise RosAPIConnectionError(str(e))
+            if len(received) == 0:
+                raise RosAPIConnectionError('Connection closed by remote end.')
+            received_overal += received
+        return received_overal
+
+
+
+
+class BaseRouterboardResource(object):
+    def __init__(self, api, namespace):
+        self.api = api
+        self.namespace = namespace
+
+    def call(self, command, set_kwargs, query_kwargs=None):
+        query_kwargs = query_kwargs or {}
+        query_arguments = self._prepare_arguments(True, **query_kwargs)
+        set_arguments = self._prepare_arguments(False, **set_kwargs)
+        query = ([('%s/%s' % (self.namespace, command)).encode('ascii')] +
+                 query_arguments + set_arguments)
+        response = self.api.api_client.talk(query)
+
+        output = []
+        for response_type, attributes in response:
+            if response_type == b'!re':
+                output.append(self._remove_first_char_from_keys(attributes))
+
+        return output
+
+    @staticmethod
+    def _prepare_arguments(is_query, **kwargs):
+        command_arguments = []
+        for key, value in kwargs.items():
+            if key in ['id', 'proplist']:
+                key = '.%s' % key
+            key = key.replace('_', '-')
+            selector_char = '?' if is_query else '='
+            command_arguments.append(
+                ('%s%s=' % (selector_char, key)).encode('ascii') + value)
+
+        return command_arguments
+
+    @staticmethod
+    def _remove_first_char_from_keys(dictionary):
+        elements = []
+        for key, value in dictionary.items():
+            key = key.decode('ascii')
+            if key in ['.id', '.proplist']:
+                key = key[1:]
+            elements.append((key, value))
+        return dict(elements)
+
+    def get(self, **kwargs):
+        return self.call('print', {}, kwargs)
+
+    def detailed_get(self, **kwargs):
+        return self.call('print', {'detail': b''}, kwargs)
+
+    def set(self, **kwargs):
+        return self.call('set', kwargs)
+
+    def add(self, **kwargs):
+        return self.call('add', kwargs)
+
+    def remove(self, **kwargs):
+        return self.call('remove', kwargs)
+
+
+class RouterboardResource(BaseRouterboardResource):
+    def detailed_get(self, **kwargs):
+        return self.call('print', {'detail': ''}, kwargs)
+
+    def call(self, command, set_kwargs, query_kwargs=None):
+        query_kwargs = query_kwargs or {}
+        result = super(RouterboardResource, self).call(
+            command, self._encode_kwargs(set_kwargs),
+            self._encode_kwargs(query_kwargs))
+        for item in result:
+            for k in item:
+                item[k] = item[k].decode('ascii')
+        return result
+
+    def _encode_kwargs(self, kwargs):
+        return dict((k, v.encode('ascii')) for k, v in kwargs.items())
+
+
+class RouterboardAPI(object):
+    def __init__(self, host, username='api', password='', port=8728, ssl=False):
+        self.host = host
+        self.username = username
+        self.password = password
+        self.socket = None
+        self.port = port
+        self.ssl = ssl
+        self.reconnect()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, _, __, ___):
+        self.close_connection()
+
+    def reconnect(self):
+        if self.socket:
+            self.close_connection()
+        try:
+            for retry in retryloop(10, delay=0.1, timeout=30):
+                try:
+                    self.connect()
+                    self.login()
+                except socket.error:
+                    retry()
+        except (socket.error, RetryError) as e:
+            raise RosAPIConnectionError(str(e))
+
+    def connect(self):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(15.0)
+        sock.connect((self.host, self.port))
+        set_keepalive(sock, after_idle_sec=10)
+        if self.ssl:
+            try:
+                self.socket = ssl.wrap_socket(sock)
+            except ssl.SSLError as e:
+                raise RosAPIConnectionError(str(e))
+        else:
+            self.socket = sock
+        self.api_client = RosAPI(self.socket)
+
+    def login(self):
+        self.api_client.login(self.username.encode('ascii'),
+                              self.password.encode('ascii'))
+
+    def get_resource(self, namespace):
+        return RouterboardResource(self, namespace)
+
+    def get_base_resource(self, namespace):
+        return BaseRouterboardResource(self, namespace)
+
+    def close_connection(self):
+        self.socket.close()
+
+
+class Mikrotik(object):
+
+  def __init__(self, hostname, username, password):
+    self.hostname = hostname
+    self.username = username
+    self.password = password
+
+  def login(self):
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.connect((self.hostname, 8728))
+    mt = RosAPI(s)
+    mt.login(self.username, self.password)
+    return mt
+
+  def talk(self, talk_command):
+    r = self.login()
+    response = r.talk(talk_command)
+    return(response)
+
+  def api_print(self, base_path, params=None):
+    command = [base_path + '/print']
+    if params is not None:
+      for key, value in params.iteritems():
+        item = b'=' + key + '=' + str(value)
+        command.append(item)
+
+    return self.talk(command)
+
+  def api_add(self, base_path, params):
+    command = [base_path + '/add']
+    for key, value in params.iteritems():
+      item = b'=' + key + '=' + str(value)
+      command.append(item)
+
+    return self.talk(command)
+
+  def api_edit(self, base_path, params):
+    command = [base_path + '/set']
+    for key, value in params.iteritems():
+      item = b'=' + key + '=' + str(value)
+      command.append(item)
+
+    return self.talk(command)
+
+  def api_remove(self, base_path, remove_id):
+    command = [
+        base_path + '/remove',
+        b'=.id=' + remove_id
+    ]
+
+    return self.talk(command)
+
+  def api_command(self, base_path, params=None):
+    command = [base_path]
+    if params is not None:
+      for key, value in params.iteritems():
+        item = b'=' + key + '=' + str(value)
+        command.append(item)
+
+    return self.talk(command)

pythonlibs/mt_api/retryloop.py

@@ -0,0 +1,38 @@
+# retry loop from http://code.activestate.com/recipes/578163-retry-loop/
+import time
+import sys
+
+
+class RetryError(Exception):
+    pass
+
+
+def retryloop(attempts, timeout=None, delay=0, backoff=1):
+    starttime = time.time()
+    success = set()
+    for i in range(attempts): 
+        success.add(True)
+        yield success.clear
+        if success:
+            return
+        duration = time.time() - starttime
+        if timeout is not None and duration > timeout:
+            break
+        if delay:
+            time.sleep(delay)
+            delay *= backoff
+
+    e = sys.exc_info()[1]
+
+    # No pending exception? Make one
+    if e is None:
+        try:
+            raise RetryError
+        except RetryError as exc:
+            e = exc
+
+    # Decorate exception with retry information:
+    e.args = e.args + ("on attempt {0} of {1} after {2:.3f} seconds".format(
+        i + 1, attempts, duration), )
+
+    raise e

pythonlibs/mt_api/socket_utils.py

@@ -0,0 +1,15 @@
+import socket
+
+
+# http://stackoverflow.com/a/14855726
+def set_keepalive(sock, after_idle_sec=1, interval_sec=3, max_fails=5):
+    """Set TCP keepalive on an open socket.
+
+    It activates after 1 second (after_idle_sec) of idleness,
+    then sends a keepalive ping once every 3 seconds (interval_sec),
+    and closes the connection after 5 failed ping (max_fails), or 15 seconds
+    """
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, after_idle_sec)
+    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec)
+    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, max_fails)

pythonlibs/mt_common.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+import mt_api
+import re
+
+
+def clean_params(params):
+  '''
+  remove keys with empty values
+  modify keys with '_' to match mikrotik parameters
+  convert yes/no to true/false
+  '''
+  if isinstance(params, dict):
+    for key in list(params):
+      if params[key] is None:
+        del params[key]
+        continue
+
+      new_key = re.sub('_', '-', key)
+      if new_key != key:
+        params[new_key] = str(params[key])
+        del params[key]
+        continue
+
+      if params[key] == "yes":
+        params[key] = "true"
+      if params[key] == "no":
+        params[key] = "false"
+  else:
+    print("Must be a dictionary")
+
+
+class MikrotikIdempotent():
+  '''
+  MikrotikIdempotent Class
+    - A helper class for Ansible modules to abstract common functions.
+
+  Example Usage:
+    mt_obj = MikrotikIdempotent(
+      hostname        = params['hostname'],
+      username        = params['username'],
+      password        = params['password'],
+      state           = None,
+      desired_params  = params['settings'],
+      idempotent_param= 'name',
+      api_path        = '/interface/ethernet',
+    )
+
+    mt_obj.sync_state()
+  '''
+
+  def __init__(
+   self, hostname, username, password, desired_params, api_path,
+   state, idempotent_param, check_mode=False):
+
+    self.hostname         = hostname
+    self.username         = username
+    self.password         = password
+    self.state            = state
+    self.desired_params   = desired_params
+    self.idempotent_param = idempotent_param
+    self.current_params   = {}
+    self.api_path         = api_path
+    self.check_mode       = check_mode
+
+    self.login_success    = False
+    self.changed          = False
+    self.changed_msg      = []
+    self.failed           = False
+    self.failed_msg       = []
+
+    self.login()
+
+  def login(self):
+    self.mk = mt_api.Mikrotik(
+      self.hostname,
+      self.username,
+      self.password,
+    )
+
+    try:
+      self.mk.login()
+      self.login_success = True
+    except:
+      self.failed_msg = "Could not log into Mikrotik device." + " Check the username and password.",
+
+  def get_current_params(self):
+    clean_params(self.desired_params)
+    self.param_id = None
+    self.current_param = None
+    self.current_params = self.mk.api_print(base_path=self.api_path)
+
+    # When state and idempotent_param is None we are working
+    # on editable params only and we are grabbing the only item from response
+    if self.state is None and self.idempotent_param is None:
+      self.current_param = self.current_params[0][1]
+
+    # Else we iterate over every item in the list until we find the matching
+    # params
+    # We also set the param_id here to reference later for editing or removal
+    else:
+      for current_param in self.current_params:
+        if self.idempotent_param in current_param[1]:
+          if self.desired_params[self.idempotent_param] == current_param[1][self.idempotent_param]:
+            self.current_param = current_param[1]
+            self.param_id = current_param[1]['.id']
+            # current_param now is a dict, something like:
+            # {
+            #    ".id": "*1",
+            #    "full-duplex": "true",
+            #    "mac-address": "08:00:27:6F:4C:22",
+            #    "mtu": "1500",
+            #    "name": "ether1",
+            #    ...
+            # }
+
+  def add(self):
+    # When current_param is empty we need to call api_add method to add
+    # all the parameters in the desired_params
+    if self.current_param is None:
+      self.new_params = self.desired_params
+      self.old_params = ""
+      if not self.check_mode:
+          self.mk.api_add(
+              base_path = self.api_path,
+              params    = self.desired_params,
+          )
+      self.changed = True
+
+    # Else we need to determing what the difference between the currently
+    # and the desired
+    else:
+      self.edit()
+
+  def rem(self):
+    # if param_id is set this means there is currently a matching item
+    # which we will remove
+    if self.param_id:
+      self.new_params = "item removed"
+      self.old_params = self.desired_params
+      if not self.check_mode:
+        self.mk.api_remove(
+            base_path=self.api_path,
+            remove_id=self.param_id,
+        )
+      self.changed = True
+
+  def edit(self):
+    out_params = {}
+    old_params = {} #used to store values of params we change
+
+    # iterate over items in desired params and match against items in current_param
+    # to figure out the difference
+    for desired_param in self.desired_params:
+      self.desired_params[desired_param] = str(self.desired_params[desired_param])
+      if desired_param in self.current_param:
+        if self.current_param[desired_param] != self.desired_params[desired_param]:
+          out_params[desired_param] = self.desired_params[desired_param]
+          old_params[desired_param] = self.current_param[desired_param]
+      else:
+        out_params[desired_param] = self.desired_params[desired_param]
+        if desired_param in self.current_param:
+          old_params[desired_param] = self.current_param[desired_param]
+
+    # When out_params has been set it means we found our diff
+    # and will set it on the mikrotik
+    if out_params:
+      if self.param_id is not None:
+        out_params['.id'] = self.current_param['.id']
+
+      if not self.check_mode:
+        self.mk.api_edit(
+            base_path = self.api_path,
+            params    = out_params,
+        )
+
+      # we don't need to show the .id in the changed message
+      if '.id' in out_params:
+        del out_params['.id']
+
+      self.changed_msg.append({
+          "new_params":   out_params,
+          "old_params":   old_params,
+      })
+
+      self.new_params = out_params
+      self.old_params = old_params
+      self.changed = True
+
+  def sync_state(self):
+    self.get_current_params()
+
+    # When state and idempotent_param are not set we are working
+    # on editable parameters only that we can't add or remove
+    if self.state is None and self.idempotent_param is None:
+      self.edit()
+    elif self.state == "absent":
+      self.rem()
+    elif self.state == "present" or self.idempotent_param:
+      self.add()