From 26650f6fe4019020c4b78ff92a7745bbe23d64f5 Mon Sep 17 00:00:00 2001 From: David Beniamine <david.beniamine@tetras-libre.fr> Date: Tue, 12 Jan 2021 14:23:25 +0100 Subject: [PATCH] Initial commit --- .gitignore | 8 ++++ .gitlab-ci.yml | 14 ++++++ Readme.md | 91 +++++++++++++++++++++++++++++++++++++++ __init__.py | 0 pyTiming/__init__.py | 0 pyTiming/pyTiming.py | 68 +++++++++++++++++++++++++++++ pyTiming/tests/test_pt.py | 56 ++++++++++++++++++++++++ setup.cfg | 2 + setup.py | 22 ++++++++++ 9 files changed, 261 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 Readme.md create mode 100644 __init__.py create mode 100644 pyTiming/__init__.py create mode 100644 pyTiming/pyTiming.py create mode 100644 pyTiming/tests/test_pt.py create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2485c22 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.egg-info +.eggs +__pycache__ +.ropeproject +.pytest_cache +*.pyc +*.sw? +.cache diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..ddc23d2 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,14 @@ +image: python:3.6 + +stages: + - build + - test + - deploy + +before_script: + - pip install setuptools + - pip install . + +pytest: + stage: test + script: python setup.py test diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..dda026a --- /dev/null +++ b/Readme.md @@ -0,0 +1,91 @@ +# Python timing + +This is a simple libraire to do fine timing of python application and measure parallelism + +## Install + +### pip + +```bash +pip3 install git+https://gitlab.tetras-libre.fr/tetras-libre/performance-analysis/pyTiming#egg=pyTiming +``` + +### For developpement + +```bash +git clone https://gitlab.tetras-libre.fr/tetras-libre/performance-analysis/pyTiming +pip3 install -e . +``` + +### As a dependency in your setup.py + +```python +from setuptools.command.install import install +from setuptools.command.develop import develop +from setuptools import setup, find_packages +import pip + +... + +gitDeps = [ +'git+https://gitlab.tetras-libre.fr/tetras-libre/performance-analysis/pyTiming#egg=pyTiming', + ] + + +def customRun(command_subclass): + orig_run = command_subclass.run + + def modified_run(self): + for dep in gitDeps: + pip.main(['install', dep]) + + orig_run(self) + + command_subclass.run = modified_run + return command_subclass + +@customRun +class actionsOnInstall(install): + pass + + +@customRun +class actionsOnDevelop(develop): + pass + +... + +setup( +... + cmdclass={'install': actionsOnInstall, + 'develop': actionsOnDevelop, +... +) + +``` + +## Usage + +```python +from pyTiming.pyTiming import timer +timer = timer() # starts measuring time +. . . + +timer.start_step('step1') +. . . +timer.start_step('step1.1') +. . . +timer.end_step('step1.1') +timer.start_step('step1.2') +. . . +timer.end_step('step1.2') +timer.end_step('step1') + +timer.start_step('step2') +. . . +timer.end_step('step2') + +timer.end() # end all measurements + +print(timer.get_analysis()) +``` diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyTiming/__init__.py b/pyTiming/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyTiming/pyTiming.py b/pyTiming/pyTiming.py new file mode 100644 index 0000000..44a1c25 --- /dev/null +++ b/pyTiming/pyTiming.py @@ -0,0 +1,68 @@ +from time import time +from resource import getrusage, RUSAGE_CHILDREN, RUSAGE_SELF +from multiprocessing import cpu_count + + +class timer: + METRICS = ['cpu', 'total'] + + def __init__(self): + """Initialize the main timer + """ + self.starts = {} + self.ends = {} + self.start_step('main') + + def start_step(self, name): + """ Start a new timer or replace start time for the given timer + Keyword arguments: + name -- name of the timer + """ + self.starts[name] = self._get_timers() + + def end_step(self, name): + """ Ends timer or replace end time for the given timer + Keyword arguments: + name -- name of the timer + """ + self.ends[name] = self._get_timers() + + def end(self): + self.end_step('main') + + def get_analysis(self): + """ Return all the metrics + """ + out = {} + for key in self.ends: + out[key] = self._get_times(key) + if 'main' in out: + for key in self.ends: + out[key]['exec_percent'] = 100 * out[key]['total'] / out['main']['total'] + return out + + def _get_times(self, name): + """ Return the metrics for step name + Keyword arguments: + name -- name of the timer + """ + out = {m: self.ends[name][m] - self.starts[name][m] for m in self.METRICS} + out['parallelism'] = out['cpu'] / out['total'] + return out + + def _get_timers(self): + """ Returns a dictionnary of mesured times corresponding to now + """ + return {m: getattr(self, f'_get_{m}_time')() for m in self.METRICS} + + @staticmethod + def _get_cpu_time(): + """ Get the cpu time metric + """ + return getrusage(RUSAGE_CHILDREN).ru_utime + getrusage(RUSAGE_SELF).ru_utime + + @staticmethod + def _get_total_time(): + """ Get the total time metric + """ + return time() diff --git a/pyTiming/tests/test_pt.py b/pyTiming/tests/test_pt.py new file mode 100644 index 0000000..dccdd8c --- /dev/null +++ b/pyTiming/tests/test_pt.py @@ -0,0 +1,56 @@ +import random +from pyTiming import pyTiming +from multiprocessing import Pool, cpu_count +from multiprocessing.pool import ThreadPool + + +MAX_PROC = int(cpu_count() / 2) + + +def stress(arg): + size = 50000 + sorted([random.randint(0, size) for i in range(size)]) + + +def processes(): + with Pool(MAX_PROC) as p: + p.map(stress, range(MAX_PROC)) + + +def threads(): + with ThreadPool(MAX_PROC) as p: + p.map(stress, range(MAX_PROC)) + + +def seq(): + for i in range(MAX_PROC): + stress(i) + +def test_timer(): + timer = pyTiming.timer() + + timer.start_step('processes') + processes() + timer.end_step('processes') + + timer.start_step('threads') + threads() + timer.end_step('threads') + + timer.start_step('seq') + seq() + timer.end_step('seq') + + timer.end() + + analysis = timer.get_analysis() + assert analysis['processes']['parallelism'] > 1.1 + assert analysis['processes']['parallelism'] < MAX_PROC + assert analysis['threads']['parallelism'] > .8 + assert analysis['threads']['parallelism'] < 1.1 + assert analysis['seq']['parallelism'] > .8 + assert analysis['seq']['parallelism'] < 1.1 + + + + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b7e4789 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..dd02938 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +from setuptools import setup, find_packages +from os import path + +here = path.abspath(path.dirname(__file__)) + +with open(path.join(here, 'Readme.md'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name="pyTiming", + url="https://gitlab.tetras-libre.fr/tetras-libre/performance-analysis/pyTiming", + version="1.0", + license="Gpl v3+", + author="Tetras Libre", + author_email="Contact@Tetras-Libre.fr", + description="Simple Performance analysis library", + long_description=long_description, + packages=find_packages(), + install_requires=[""], + setup_requires=["pytest-runner"], + tests_require=["pytest"], +) -- GitLab