diff --git a/.gitignore b/.gitignore index c7789f1..a49f4f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ env +env3 *.pyc dist *.egg-info/ diff --git a/.travis.yml b/.travis.yml index 8fd312e..7d86a98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ os: linux python: - "2.7" + - "3.6" -install: make env +install: pip install --upgrade -r requirements.txt && python setup.py install script: make test lint diff --git a/Makefile b/Makefile index 2783d5e..58bda7c 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,15 @@ -test: env +test: + ./tests/rss-ladder + ./tests/server-info-show + ./tests/link_rate_units.sh + pytest netutils_linux_*/ + +env: + rm -rf env + virtualenv env && \ . env/bin/activate && \ - ./tests/rss-ladder && \ - ./tests/rx_buffers_test.py && \ - ./tests/softnet_stat_test.py && \ - ./tests/server-info-show && \ - ./tests/softirq_top_test.py && \ - ./tests/assessor_test.py & \ - ./tests/link_rate_units.sh + pip install --upgrade -r requirements.txt && \ + python setup.py install help: @echo " env create a development environment using virtualenv" @@ -16,12 +19,35 @@ help: @echo " coverage run tests with code coverage" @echo " test run tests" -env: - rm -rf env - virtualenv env && \ - . env/bin/activate && \ - pip install --upgrade -r requirements.txt && \ - python setup.py install +# only for localhost MacOS testing. +test2: + . env2/bin/activate && \ + ./tests/rss-ladder && \ + ./tests/server-info-show && \ + ./tests/link_rate_units.sh + pytest netutils_linux_*/ + +env2: + rm -rf env2 + virtualenv --python=python2 env2 && \ + . env2/bin/activate && \ + pip install --upgrade -r requirements.txt && \ + python setup.py install + +# only for localhost MacOS testing. +test3: + . env3/bin/activate && \ + ./tests/rss-ladder && \ + ./tests/server-info-show && \ + ./tests/link_rate_units.sh + pytest netutils_linux_*/ + +env3: + rm -rf env3 + virtualenv --python=python3 env3 && \ + . env3/bin/activate && \ + pip install --upgrade -r requirements.txt && \ + python setup.py install clean: rm -fr env @@ -32,7 +58,6 @@ clean: find . -name '*~' -exec rm -f {} \; lint: - . env/bin/activate && \ flake8 netutils_linux_monitoring netutils_linux_tuning netutils_linux_hardware coverage: diff --git a/netutils_linux_hardware/assessor.py b/netutils_linux_hardware/assessor.py index 0457be4..722782d 100644 --- a/netutils_linux_hardware/assessor.py +++ b/netutils_linux_hardware/assessor.py @@ -1,7 +1,13 @@ import re +import math import yaml +def round_(x, d=0): + p = 10 ** d + return float(math.floor((x * p) + math.copysign(0.5, x)))/p + + def extract(dictionary, key_sequence): key_sequence.reverse() while dictionary and key_sequence: @@ -35,9 +41,10 @@ def any2int(value): return int(value) return 0 - def grade_int(self, value, _min, _max, scale=10): - value = self.any2int(value) - return min(scale, max(1, int(1 + round((value - _min) * (scale - 1.) / (_max - _min))))) + @staticmethod + def grade_int(value, _min, _max, scale=10): + value = Assessor.any2int(value) + return min(scale, max(1, int(1 + round_((value - _min) * (scale - 1.) / (_max - _min)+.001)))) @staticmethod def grade_str(value, good=None, bad=None): diff --git a/tests/assessor_test.py b/netutils_linux_hardware/assessor_test.py similarity index 97% rename from tests/assessor_test.py rename to netutils_linux_hardware/assessor_test.py index 5f45c39..86c09d5 100755 --- a/tests/assessor_test.py +++ b/netutils_linux_hardware/assessor_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from unittest import TestCase, main +from six import iteritems from netutils_linux_hardware import Assessor @@ -30,7 +31,7 @@ def test_grade_str(self): "Dlink": 1, "Realtek": 1, } - for k, v in expected.iteritems(): + for k, v in iteritems(expected): self.assertEqual(Assessor.grade_str(k, good, bad), v) def test_grade_fact(self): @@ -59,5 +60,6 @@ def test_grade_list(self): self.assertEqual(assessor.grade_list([1], 1, 4), 1) self.assertEqual(assessor.grade_list([1, 2, 3, 4], 1, 4), 10) + if __name__ == '__main__': main() diff --git a/netutils_linux_hardware/interrupts.py b/netutils_linux_hardware/interrupts.py index b38e6f0..2894b78 100644 --- a/netutils_linux_hardware/interrupts.py +++ b/netutils_linux_hardware/interrupts.py @@ -1,7 +1,8 @@ # coding: utf-8 # pylint: disable=C0111, C0103 -from parsers import Parser +from six import print_ +from netutils_linux_hardware.parsers import Parser class NICQueues(object): @@ -28,7 +29,7 @@ def netdev_queue_relationship(queue, dev): return kind if kind.isdigit() or kind == 'TxRx': return 'rxtx' - print queue + print_(queue) if queue.count(',') > 0 and dev in queue: return 'shared' return 'unknown' @@ -47,7 +48,7 @@ class IRQQueueCounter(Parser): @staticmethod def all_netdev_queues(text, netdevs): for line in text.strip().split('\n'): - for netdev in netdevs.iterkeys(): + for netdev in netdevs: if netdev in line: yield line.strip() continue @@ -65,5 +66,5 @@ def parse(self, text, **kwargs): netdevs = kwargs['netdevs'] cpu_count = self.irq2cpucount(text) queues_names = self.irq2queues(text, cpu_count, netdevs) - for netdev in netdevs.iterkeys(): + for netdev in netdevs: netdevs[netdev]['queues'] = NICQueues().parse(queues_names, netdev) diff --git a/netutils_linux_hardware/netdev.py b/netutils_linux_hardware/netdev.py index b8145b4..31b088b 100644 --- a/netutils_linux_hardware/netdev.py +++ b/netutils_linux_hardware/netdev.py @@ -2,8 +2,9 @@ # pylint: disable=C0111, C0103 import os -from interrupts import IRQQueueCounter -from parsers import EthtoolBuffers, ReductorMirror, BrctlOutput, YAMLLike +from six import iteritems +from netutils_linux_hardware.interrupts import IRQQueueCounter +from netutils_linux_hardware.parsers import EthtoolBuffers, ReductorMirror, BrctlOutput, YAMLLike class ReaderNet(object): @@ -38,7 +39,7 @@ def net_dev_list_drivers(self): driverfile = os.path.join(self.datadir, 'ethtool/i', netdev) driverdata = YAMLLike().parse_file_safe(driverfile) if driverdata: - driverdata = dict((k, v) for k, v in driverdata.iteritems() if k in keys_required) + driverdata = dict((k, v) for k, v in iteritems(driverdata) if k in keys_required) else: driverdata = dict() self.netdevs[netdev]['driver'] = driverdata diff --git a/netutils_linux_hardware/parsers.py b/netutils_linux_hardware/parsers.py index e639fd4..73e8011 100644 --- a/netutils_linux_hardware/parsers.py +++ b/netutils_linux_hardware/parsers.py @@ -3,6 +3,7 @@ import os import yaml +from six import print_, iteritems class Parser(object): @@ -29,7 +30,7 @@ class ReductorMirror(Parser): @staticmethod def parse(text): lines = dict((line.split(' ', 1)) for line in text.strip().split('\n')) - for netdev, conf in lines.iteritems(): + for netdev, conf in iteritems(lines): output = dict() output['conf'] = dict() output['conf']['vlan'], output['conf']['ip'] = conf.split() @@ -53,8 +54,8 @@ def invert_dict_nesting(d): } """ d2 = dict() - for k, v in d.iteritems(): - for k2, v2 in v.iteritems(): + for k, v in iteritems(d): + for k2, v2 in iteritems(v): if not d2.get(k2): d2[k2] = dict() d2[k2][k] = v2 @@ -78,11 +79,11 @@ def parse(text): types = ['SSD', 'HDD'] if not text: return dict() - return dict((k, types[v]) for k, v in yaml.load(text - .replace(":", ": ") - .replace("/sys/block/", "") - .replace("/queue/rotational", "")) - .iteritems()) + return dict((k, types[v]) for k, v in iteritems(yaml.load(text + .replace(":", ": ") + .replace("/sys/block/", "") + .replace("/queue/rotational", "")) + )) class DiskSizeInfo(Parser): @@ -116,7 +117,7 @@ class MemInfo(YAMLLike): ) def parse(self, text): - return dict((k, int(v.replace(' kB', ''))) for k, v in yaml.load(text).iteritems() if k in self.keys_required) + return dict((k, int(v.replace(' kB', ''))) for k, v in iteritems(yaml.load(text)) if k in self.keys_required) class CPULayout(Parser): @@ -141,7 +142,7 @@ def parse(text): elif key.count('.') == 0: dev = key else: - print 'QinQ not supported yet. Device: {0}'.format(key) + print_('QinQ not supported yet. Device: {0}'.format(key)) raise NotImplementedError netdevs[dev] = dict() diff --git a/netutils_linux_hardware/reader.py b/netutils_linux_hardware/reader.py index a2bb1e8..ac30949 100644 --- a/netutils_linux_hardware/reader.py +++ b/netutils_linux_hardware/reader.py @@ -3,8 +3,8 @@ import os import yaml -from parsers import YAMLLike, CPULayout, DiskInfo, MemInfo -from netdev import ReaderNet +from netutils_linux_hardware.parsers import YAMLLike, CPULayout, DiskInfo, MemInfo +from netutils_linux_hardware.netdev import ReaderNet class Reader(object): diff --git a/netutils_linux_monitoring/base_top.py b/netutils_linux_monitoring/base_top.py index 4b52df5..52a5b68 100644 --- a/netutils_linux_monitoring/base_top.py +++ b/netutils_linux_monitoring/base_top.py @@ -3,7 +3,8 @@ from random import randint from optparse import Option, OptionParser, OptionConflictError from colorama import Fore -from colors import wrap +from six import print_, iteritems +from netutils_linux_monitoring.colors import wrap class BaseTop(object): @@ -55,7 +56,7 @@ def parse_options(self, options=None): pass self.options, _ = parser.parse_args() if options: - for name, value in options.iteritems(): + for name, value in iteritems(options): setattr(self.options, name, value) if hasattr(self, 'post_optparse'): # pylint: disable=E1101 @@ -84,9 +85,9 @@ def run(self): if self.options.clear: system('clear') if self.diff: - print self + print_(self) except KeyboardInterrupt: - print + print_() exit(0) def repr_source(self): diff --git a/netutils_linux_monitoring/irqtop.py b/netutils_linux_monitoring/irqtop.py index 48e2dd2..3bf7582 100644 --- a/netutils_linux_monitoring/irqtop.py +++ b/netutils_linux_monitoring/irqtop.py @@ -3,6 +3,7 @@ from random import randint from copy import deepcopy from optparse import Option +from six.moves import xrange from netutils_linux_monitoring.base_top import BaseTop from netutils_linux_monitoring.colors import colorize_cpu_list, colorize from netutils_linux_monitoring.numa import Numa diff --git a/netutils_linux_monitoring/layout.py b/netutils_linux_monitoring/layout.py index a4af362..03ea927 100644 --- a/netutils_linux_monitoring/layout.py +++ b/netutils_linux_monitoring/layout.py @@ -2,6 +2,7 @@ """ Everything about console output's layout """ +from six import print_ from prettytable import PrettyTable @@ -20,8 +21,8 @@ def make_table(header, align_map=None, rows=None): try: t.add_row(row) except Exception as err: - print 'fields:', t.field_names - print 'row:', row - print 'rows:', rows + print_('fields:', t.field_names) + print_('row:', row) + print_('rows:', rows) raise err return t diff --git a/netutils_linux_monitoring/link_rate.py b/netutils_linux_monitoring/link_rate.py index 20293ce..20abd84 100644 --- a/netutils_linux_monitoring/link_rate.py +++ b/netutils_linux_monitoring/link_rate.py @@ -4,6 +4,7 @@ from random import randint from optparse import Option from collections import namedtuple +from six import print_, iteritems from netutils_linux_monitoring.base_top import BaseTop from netutils_linux_monitoring.layout import make_table from netutils_linux_monitoring.numa import Numa @@ -57,7 +58,7 @@ def parse(self): def eval(self): self.diff = deepcopy(self.current) - for dev, data in self.current.iteritems(): + for dev, data in iteritems(self.current): for stat in self.stats: if self.options.random: self.diff[dev][stat] = randint(0, 10000) @@ -130,7 +131,7 @@ def devices_list(self): devices = self.options.devices.split(',') else: devices = self.devices_list_regex() - return filter(self.devices_list_validate, devices) + return list(filter(self.devices_list_validate, devices)) def post_optparse(self): """ Asserting and applying parsing options """ @@ -167,4 +168,4 @@ def unit_change(self): if __name__ == '__main__': - print LinkRateTop().run() + print_(LinkRateTop().run()) diff --git a/netutils_linux_monitoring/network_top.py b/netutils_linux_monitoring/network_top.py index f4748b3..fdad73b 100644 --- a/netutils_linux_monitoring/network_top.py +++ b/netutils_linux_monitoring/network_top.py @@ -2,6 +2,7 @@ from colorama import Style from optparse import OptionParser, OptionConflictError +from six import iteritems, itervalues from netutils_linux_monitoring import IrqTop, Softirqs, SoftnetStatTop, LinkRateTop from netutils_linux_monitoring.numa import Numa from netutils_linux_monitoring.base_top import BaseTop @@ -25,16 +26,16 @@ def __init__(self): fake=self.options.random) def parse(self): - return dict((top_name, _top.parse()) for top_name, _top in self.tops.iteritems()) + return dict((top_name, _top.parse()) for top_name, _top in iteritems(self.tops)) def eval(self): if all((self.current, self.previous)): - self.diff = dict((top_name, _top.diff) for top_name, _top in self.tops.iteritems()) + self.diff = dict((top_name, _top.diff) for top_name, _top in iteritems(self.tops)) def tick(self): self.previous = self.current self.current = self.parse() - for _top in self.tops.itervalues(): + for _top in itervalues(self.tops): _top.tick() self.eval() @@ -103,14 +104,14 @@ def __repr__(self): def parse_options(self): """ Tricky way to gather all options in one util without conflicts, parse them and do some logic after parse """ parser = OptionParser() - for top in self.tops.itervalues(): + for top in itervalues(self.tops): for opt in top.specific_options: try: parser.add_option(opt) except OptionConflictError: pass # I don't know how to make a set of options self.options, _ = parser.parse_args() - for top in self.tops.itervalues(): + for top in itervalues(self.tops): top.options = self.options if hasattr(top, 'post_optparse'): top.post_optparse() diff --git a/netutils_linux_monitoring/numa.py b/netutils_linux_monitoring/numa.py index ff01084..91d531b 100644 --- a/netutils_linux_monitoring/numa.py +++ b/netutils_linux_monitoring/numa.py @@ -4,6 +4,7 @@ import os from subprocess import Popen, PIPE +from six import print_ class Numa(object): @@ -62,7 +63,7 @@ def detect_layouts(self): if process.returncode != 0: return None rows = stdout.strip().split('\n') - layouts = [map(int, row.split()[1:3]) for row in rows if 'NODE' not in row] + layouts = [list(map(int, row.split()[1:3])) for row in rows if 'NODE' not in row] numa_layout, socket_layout = zip(*layouts) self.numa_layout = dict(enumerate(numa_layout)) self.socket_layout = dict(enumerate(socket_layout)) @@ -70,5 +71,5 @@ def detect_layouts(self): if __name__ == '__main__': numa = Numa() - print 'SOCKET', numa.socket_layout - print 'NUMA', numa.numa_layout + print_('SOCKET', numa.socket_layout) + print_('NUMA', numa.numa_layout) diff --git a/netutils_linux_monitoring/softirqs.py b/netutils_linux_monitoring/softirqs.py index d149b24..5c5afe5 100644 --- a/netutils_linux_monitoring/softirqs.py +++ b/netutils_linux_monitoring/softirqs.py @@ -1,4 +1,5 @@ from optparse import Option +from six import iteritems from netutils_linux_monitoring.base_top import BaseTop from netutils_linux_monitoring.layout import make_table from netutils_linux_monitoring.numa import Numa @@ -30,7 +31,7 @@ def parse(self): with open(self.options.softirqs_file) as fd: metrics = [line.strip().split(':') for line in fd.readlines() if ':' in line] - return dict((k, map(int, v.strip().split())) for k, v in metrics) + return dict((k, list(map(int, v.strip().split()))) for k, v in metrics) @staticmethod def __active_cpu_count__(data): @@ -38,17 +39,15 @@ def __active_cpu_count__(data): def eval(self): self.diff = dict((key, self.list_diff( - data, self.previous[key])) for key, data in self.current.iteritems()) + data, self.previous[key])) for key, data in iteritems(self.current)) def __repr__(self): active_cpu_count = self.__active_cpu_count__(self.current) header = ["CPU", "NET_RX", "NET_TX"] + net_rx = self.repr_source().get('NET_RX')[:active_cpu_count] + net_tx = self.repr_source().get('NET_TX')[:active_cpu_count] rows = [ - [wrap("CPU{0}".format(n), cpu_color(n, self.numa)), v[0], v[1]] for n, v in - enumerate(zip( - self.repr_source().get('NET_RX')[:active_cpu_count], - self.repr_source().get('NET_TX')[:active_cpu_count] - )) + [wrap("CPU{0}".format(n), cpu_color(n, self.numa)), v[0], v[1]] for n, v in enumerate(zip(net_rx, net_tx)) ] table = make_table(header, ['l', 'r', 'r'], rows) return self.__repr_table__(table) diff --git a/tests/softirq_top_test.py b/netutils_linux_monitoring/softirqs_test.py similarity index 94% rename from tests/softirq_top_test.py rename to netutils_linux_monitoring/softirqs_test.py index 3b3f96c..0097bbc 100755 --- a/tests/softirq_top_test.py +++ b/netutils_linux_monitoring/softirqs_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import unittest +from six.moves import xrange from netutils_linux_monitoring.softirqs import Softirqs @@ -14,5 +15,6 @@ def test_file2data(self): top.options.softirqs_file = 'tests/softirqs/{0}/softirqs{1}'.format(cpu, i) self.assertIn('NET_RX', top.parse()) + if __name__ == '__main__': unittest.main() diff --git a/netutils_linux_monitoring/softnet_stat.py b/netutils_linux_monitoring/softnet_stat.py index 21d5ff1..5a18b67 100644 --- a/netutils_linux_monitoring/softnet_stat.py +++ b/netutils_linux_monitoring/softnet_stat.py @@ -80,7 +80,8 @@ def parse(self): def eval(self): self.diff = [data - self.previous[cpu] for cpu, data in enumerate(self.current)] - def make_header(self): + @staticmethod + def make_header(): return ["CPU", "total", "dropped", "time_squeeze", "cpu_collision", "received_rps"] def make_rows(self): diff --git a/tests/softnet_stat_test.py b/netutils_linux_monitoring/softnet_stat_test.py similarity index 100% rename from tests/softnet_stat_test.py rename to netutils_linux_monitoring/softnet_stat_test.py diff --git a/netutils_linux_tuning/rx_buffers.py b/netutils_linux_tuning/rx_buffers.py index 32b8c57..a5a63f2 100644 --- a/netutils_linux_tuning/rx_buffers.py +++ b/netutils_linux_tuning/rx_buffers.py @@ -1,4 +1,5 @@ from os import system, path +from six import print_ from subprocess import Popen, PIPE @@ -30,8 +31,8 @@ def extract_value(s): ns = '/etc/sysconfig/network-scripts/' with open(path.join(ns, 'ifcfg-' + self.dev)) as config: - if any(line for line in config.xreadlines() if 'ETHTOOL_OPTS' in line): - print "{0}'s RX ring buffer already manually tuned.".format(self.dev) + if any(line for line in config.readlines() if 'ETHTOOL_OPTS' in line): + print_("{0}'s RX ring buffer already manually tuned.".format(self.dev)) exit(0) process = Popen(['ethtool', '-i', self.dev], stdout=PIPE, stderr=PIPE) _, _ = process.communicate() @@ -63,9 +64,9 @@ def apply(self): self.investigate() self.prefered = self.determine() if self.prefered == self.current: - print "{0}'s RX ring buffer already has fine size.".format(self.dev) + print_("{0}'s RX ring buffer already has fine size.".format(self.dev)) return assert self.prefered, "Can't eval prefered RX ring buffer size." command = 'ethtool -G {0} rx {1}'.format(self.dev, self.prefered) - print 'run:', command + print_('run:', command) system(command) diff --git a/tests/rx_buffers_test.py b/netutils_linux_tuning/rx_buffers_test.py similarity index 95% rename from tests/rx_buffers_test.py rename to netutils_linux_tuning/rx_buffers_test.py index f4f4052..1d9b3ee 100755 --- a/tests/rx_buffers_test.py +++ b/netutils_linux_tuning/rx_buffers_test.py @@ -12,7 +12,7 @@ class RxBuffersIncreaserTest(unittest.TestCase): """ def setUp(self): - self.rxbi = RxBuffersIncreaser(upper_bound=2048) + self.rxbi = RxBuffersIncreaser() def test_4096(self): self.assertEqual(self.rxbi.determine(256, 4096), 2048) diff --git a/requirements.txt b/requirements.txt index fbe4209..5c6494a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ ipaddress six colorama flake8 +pytest diff --git a/utils/server-info-show b/utils/server-info-show index 812b77b..e64496a 100755 --- a/utils/server-info-show +++ b/utils/server-info-show @@ -3,6 +3,7 @@ # pylint: disable=C0111, C0103 import os +from six import print_ from netutils_linux_hardware.reader import Reader @@ -14,7 +15,7 @@ def main(): # test = '2xE5530.82576_and_82574L.l2_mixed.manual' test = '2xE5-2640.i350_and_82599ES.l2_mixed.masterconf' datadir = os.getenv('DATADIR', os.path.join(test_dir, test)) - print Reader(datadir) + print_(Reader(datadir)) if __name__ == '__main__': main()