diff --git a/.gitignore b/.gitignore index 91767b1..7c5c7fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ *py[cod] +*.c +__pycache__/ example/ -*job +*.job *log +tmp/ +build/ +test/ +dist/ +*.egg-info/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6948f2e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2022~2023 yodeng + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/runjob/__init__.py b/runjob/__init__.py new file mode 100644 index 0000000..8dee4bf --- /dev/null +++ b/runjob/__init__.py @@ -0,0 +1 @@ +from ._version import __version__ diff --git a/setup.py b/setup.py index b0a9870..0c112a0 100644 --- a/setup.py +++ b/setup.py @@ -1,46 +1,124 @@ import os +import sys +import sysconfig from setuptools import setup -from src.version import __version__ - - -def getdes(): - des = "" - with open(os.path.join(os.getcwd(), "README.md")) as fi: - des = fi.read() - return des - - -def listdir(path): - df = [] - for a, b, c in os.walk(path): - df.append((a, [os.path.join(a, i) for i in c])) - return df - - -setup( - name="runjob", - version=__version__, - packages=["runjob"], - package_data={"runjob": ["*config", "*.json"]}, - package_dir={"runjob": "src"}, - author="Deng Yong", - author_email="yodeng@tju.edu.cn", - url="https://github.com/yodeng/runjob", - license="BSD", - install_requires=["psutil", "ratelimiter", - "batchcompute", "prettytable", "configparser"], - python_requires='>=2.7.10, <3.10', - long_description=getdes(), - long_description_content_type='text/markdown', - entry_points={ - 'console_scripts': [ - 'runjob = runjob.run:main', - 'qs = runjob.jobstat:main', - 'qcs = runjob.jobstat:batchStat', - 'runsge = runjob.sge_run:main', - 'runbatch = runjob.sge_run:main', - 'runsge0 = runjob.sge:main', +from functools import partial +from setuptools.extension import Extension + + +class Packages(object): + + def __init__(self, pkg_name=""): + self.name = pkg_name + self.base_dir = os.path.dirname(__file__) + basejoin = partial(self._join, self.base_dir) + self.source_dir = basejoin("src") + self.version_file = basejoin("src/_version.py") + self.des_file = basejoin("README.md") + self.req_file = basejoin("requirements.txt") + + def _join(self, *args): + return os.path.join(*args) + + @property + def listdir(self): + df, nc = [], [] + if os.path.isdir(self.name): + nc = os.listdir(self.name) + for a, b, c in os.walk(self.source_dir): + if os.path.basename(a).startswith("__"): + continue + for i in c: + if i.startswith("__") or i.endswith(".ini") or not i.endswith(".py") or i in nc: + continue + p = os.path.join(a[len(self.source_dir)+1:], i) + df.append(p) + return df + + @property + def description(self): + des = "" + with open(self.des_file) as fi: + des = fi.read() + return des + + @property + def version(self): + v = {} + with open(self.version_file) as fi: + c = fi.read() + exec(compile(c, self.version_file, "exec"), v) + return v["__version__"] + + @property + def requirements(self): + requires = [] + with open(self.req_file) as fi: + for line in fi: + line = line.strip() + requires.append(line) + return requires + + @property + def _extensions(self): + exts = [] + for f in self.listdir: + e = Extension(self.name + "." + os.path.splitext(f)[0].replace("/", "."), + [os.path.join(self.source_dir, f), ], extra_compile_args=["-O3", ],) + e.cython_directives = { + 'language_level': sysconfig._PY_VERSION_SHORT_NO_DOT[0]} + exts.append(e) + return exts + + @property + def _package_dir(self): + pd = {} + for a, b, v in os.walk(self.source_dir): + p = a.replace(self.source_dir, self.name).replace("/", ".") + pd[p] = a.replace(self.source_dir, "src") + return pd + + def install(self, ext=False): + kwargs = {} + kwargs.update( + name=self.name, + version=self.version, + license="MIT", + packages=list(self._package_dir.keys()), + package_dir=self._package_dir, + package_data={self.name: ["*config", "*.json"], }, + install_requires=self.requirements, + python_requires='>=2.7.10, <3.10', + long_description=self.description, + long_description_content_type='text/markdown', + entry_points={'console_scripts': self._entrys}, + author="yodeng", + author_email="yodeng@tju.edu.cn", + url="https://github.com/yodeng/runjob", + ) + if ext: + kwargs.pop("package_dir") + kwargs["ext_modules"] = self._extensions + setup(**kwargs) + + @property + def _entrys(self): + eps = [ + '%s = %s.run:main' % (self.name, self.name), + '%s = %s.jobstat:main' % ("qs", self.name), + '%s = %s.jobstat:batchStat' % ("qcs", self.name), + '%s = %s.sge_run:main' % ("runsge", self.name), + '%s = %s.sge_run:main' % ("runbatch", self.name), + '%s = %s.sge:main' % ("runsge0", self.name), ] - } -) + return eps + + +def main(): + pkgs = Packages("runjob") + pkgs.install(ext=False) + + +if __name__ == "__main__": + main() diff --git a/src/__init__.py b/src/__init__.py index 58f3ace..8dee4bf 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1 +1 @@ -from .version import __version__ +from ._version import __version__ diff --git a/src/version.py b/src/_version.py similarity index 100% rename from src/version.py rename to src/_version.py diff --git a/src/cluster.py b/src/cluster.py index a0e2241..9e5b773 100644 --- a/src/cluster.py +++ b/src/cluster.py @@ -130,7 +130,7 @@ def AddOneTask(self, job=None, outdir=""): self.client = self.cluster.client job.client = self.client - def modifyTaskOutMapping(self, mapping=""): + def modifyTaskOutMapping(self, job=None, mapping=""): if mapping: if self.cluster.oss_mount_entry["Destination"] in mapping: # if not mapping.endswith("/"): @@ -142,7 +142,7 @@ def modifyTaskOutMapping(self, mapping=""): "^"+des, src, mapping) else: self.cluster.conf.logger.debug( - "Only oss mount path support(%s), %s", outdir, job.name) + "Only oss mount path support(%s), %s", mapping, job.name) def get_instance_type(self, cpu=2, mem=4): e = [] diff --git a/src/jobstat.py b/src/jobstat.py index 79a939c..9bb4dea 100644 --- a/src/jobstat.py +++ b/src/jobstat.py @@ -22,7 +22,7 @@ from .bc_stat import * from .cluster import * from .job import Jobfile -from .version import __version__ +from ._version import __version__ from .config import load_config, print_config diff --git a/src/run.py b/src/run.py index 420413c..b38d0b0 100644 --- a/src/run.py +++ b/src/run.py @@ -16,7 +16,7 @@ from .utils import * from .qsub import qsub -from .version import __version__ +from ._version import __version__ class ParseSingal(Thread): diff --git a/src/sge_run.py b/src/sge_run.py index cef868a..b1572af 100644 --- a/src/sge_run.py +++ b/src/sge_run.py @@ -16,7 +16,7 @@ from .utils import * from .cluster import * from .sge import ParseSingal -from .version import __version__ +from ._version import __version__ from .qsub import myQueue, QsubError from .config import load_config, print_config @@ -368,7 +368,7 @@ def submit(self, job): task.AddOneTask( job=job, outdir=self.conf.get("args", "out_maping")) if job.out_maping: - task.modifyTaskOutMapping(mapping=job.out_maping) + task.modifyTaskOutMapping(job=job, mapping=job.out_maping) task.Submit() info = "Your job (%s) has been submitted in batchcompute (%s) %d times\n" % ( task.name, task.id, job.subtimes+1) diff --git a/src/utils.py b/src/utils.py index e2865c3..da45316 100644 --- a/src/utils.py +++ b/src/utils.py @@ -10,7 +10,7 @@ from subprocess import call, PIPE from ratelimiter import RateLimiter -from .version import __version__ +from ._version import __version__ if sys.version_info[0] < 3: from Queue import Queue