diff --git a/cli/database/create.py b/cli/database/create.py index 734e8265..cbc3e3d4 100644 --- a/cli/database/create.py +++ b/cli/database/create.py @@ -48,17 +48,65 @@ def memory_statistics(): logging.info(f"final -Xmx is : {max(total_memory - 1, 6):.2f} {size_units[unit_index]}") +def is_valid_regex(pattern): + try: + re.compile(pattern) + return True + except re.error: + return False + + def conf_option_deal(args): options = dict() if args.extraction_config_file: try: with open(args.extraction_config_file, "r") as f: - options = json.load(f) + extract_options = json.load(f) + for conf in extract_options: + language = conf["extractor"] + # all 先不处理 + if language == "all": + continue + if language not in args.language: + logging.error("%s language will not be extracted and the configuration is invalid", language) + continue + for option in conf["extractor_options"]: + if "name" not in option: + logging.error("option language error: please check name not in this conf : %s", + json.dumps(option)) + return -1 + key = option["name"] + if "value" not in option: + logging.error("option value error: value not in this conf : %s", json.dumps(option)) + return -1 + if "config" not in option["value"]: + logging.error("option config error:config not in this conf[\"value\"]: %s", + json.dumps(option)) + return -1 + value = option["value"]["config"] + if "pattern" in option["value"]: + pattern = option["value"]["pattern"] + if is_valid_regex(pattern): + if re.search(pattern, value): + logging.warning("option pattern error: this conf will be ignore: %s", + json.dumps(option)) + continue + else: + logging.warning("option pattern error: this conf will be ignore: %s", + json.dumps(option)) + continue + if language not in options: + options[language] = dict() + if key in options[language]: + logging.error("in %s extract, %s redefine", language, key) + return -1 + options[language][key] = value except Exception as e: logging.error(e) return -1 for language in args.language: - options[language] = dict() + if language not in options: + options[language] = dict() if args.extraction_config: # 要求option必须是a.b=c的形式，a为语言名，若不是报错 pattern = r'^(.+)\.(.+)\=(.+)$' @@ -72,6 +120,9 @@ def conf_option_deal(args): if language not in args.language: logging.error("option language error: %s does not need to be extracted", language) return -1 + if key in options[language]: + logging.error("in %s extract, %s redefine", language, key) + return -1 options[language][key] = value else: logging.error("option format error: %s, it need like java.a=b", tmp) @@ -87,7 +138,6 @@ def database_create(args): if options == -1: logging.error("configuration error, Please check conf") raise ValueError("configuration error") - memory_statistics() timeout = args.timeout extractor_fail = list() for language in args.language: diff --git a/cli/extractor/extractor.py b/cli/extractor/extractor.py index d2b9b641..ff67e26f 100644 --- a/cli/extractor/extractor.py +++ b/cli/extractor/extractor.py @@ -1,7 +1,7 @@ import logging import psutil - +import shlex from run.runner import Runner from sparrow_schema.schema import sparrow @@ -17,6 +17,7 @@ class Extractor: sql_extractor = "" swift_extractor = "" xml_extractor = "" + arkts_extractor = "" def __init__(self): Extractor.cfamily_extractor = sparrow.home / "language" / "cfamily" / "extractor" / "usr" / "bin" / "coref-cfamily-src-extractor" @@ -28,6 +29,7 @@ def __init__(self): Extractor.sql_extractor = sparrow.home / "language" / "sql" / "extractor" / "coref-sql-src-extractor_deploy.jar" Extractor.swift_extractor = sparrow.home / "language" / "swift" / "extractor" / "usr" / "bin" / "coref-swift-src-extractor" Extractor.xml_extractor = sparrow.home / "language" / "xml" / "extractor" / "coref-xml-extractor_deploy.jar" + Extractor.arkts_extractor = sparrow.home / "language" / "arkts" / "extractor" / "coref-arkts-src-extractor" def cfamily_extractor_cmd(source_root, database, options): @@ -58,15 +60,19 @@ def go_extractor_cmd(source_root, database, options): def java_extractor_cmd(source_root, database, options): cmd = list() - cmd += jar_extractor_cmd(Extractor.java_extractor, source_root, database) + cmd += jar_extractor_cmd(Extractor.java_extractor, source_root, database, options) if options: + if "white-list" in options and "whiteList" in options: + logging.error("white-list and whiteList cannot be configured at the same time") + return -1 + if "cp" in options and "classpath" in options: + logging.error("cp and classpath cannot be configured at the same time") + return -1 for (key, value) in options.items(): if key == "white-list" or key == "whiteList": - cmd += ["-w=", value] - elif key == "cp": - cmd += ["-cp=", value] - elif key == "classpath": - cmd += ["--classpath=", value] + cmd += ["-w=" + value] + elif key == "cp" or key == "classpath": + cmd += ["-cp=" + value] elif key == "incremental": if value == "true": cmd += ["--incremental"] @@ -80,8 +86,9 @@ def java_extractor_cmd(source_root, database, options): logging.warning("java.incremental does not take effect, please use java.incremental=true") else: if key != "cache-dir" and key != "commit" and key != "remote-cache-type" and \ - key != "oss-bucket" and key != "oss-config-file" and key != "oss-path-prefix": - logging.warning("unsupported config name:%s for java extractor.", key) + key != "oss-bucket" and key != "oss-config-file" and key != "oss-path-prefix" and \ + key != "jvm_opts": + logging.warning("unsupported config name: %s for java extractor.", key) if "incremental" not in options or options["incremental"] != "true": cmd += ["--parallel"] return cmd @@ -124,7 +131,7 @@ def javascript_extractor_cmd(source_root, database, options): def properties_extractor_cmd(source_root, database, options): - cmd = jar_extractor_cmd(Extractor.properties_extractor, source_root, database) + cmd = jar_extractor_cmd(Extractor.properties_extractor, source_root, database, options) return cmd @@ -136,13 +143,13 @@ def python_extractor_cmd(source_root, database, options): def sql_extractor_cmd(source_root, database, options): cmd = list() - cmd += jar_extractor_cmd(Extractor.sql_extractor, source_root, database) + cmd += jar_extractor_cmd(Extractor.sql_extractor, source_root, database, options) if "sql-dialect-type" in options: cmd += ["--sql-dialect-type", options["sql-dialect-type"]] return cmd -def swift_extractor(source_root, database, options): +def swift_extractor_cmd(source_root, database, options): cmd = list() cmd += [str(Extractor.swift_extractor), str(source_root), str(database)] if options: @@ -156,23 +163,59 @@ def swift_extractor(source_root, database, options): def xml_extractor_cmd(source_root, database, options): - cmd = jar_extractor_cmd(Extractor.xml_extractor, source_root, database) + cmd = jar_extractor_cmd(Extractor.xml_extractor, source_root, database, options) return cmd -def jar_extractor_cmd(extractor_path, source_root, database): - # 获取内存信息 - mem = psutil.virtual_memory() - total_memory = mem.total - pod_memory_limit = get_pod_memory_limit() - if pod_memory_limit != 0: - total_memory = pod_memory_limit - total_memory_gb = round(total_memory / (1024 ** 3)) - logging.info("current memory is : %s GB", total_memory_gb) - xmx = max(total_memory_gb - 1, 6) - logging.info("final -Xmx is: %s GB", xmx) +def arkts_extractor_cmd(source_root, database, options): cmd = list() - cmd += ["java", "-jar", "-Xmx" + str(xmx) + "g", str(extractor_path)] + cmd += [str(Extractor.arkts_extractor), "extract"] + \ + ["--extract-text", "-s", str(source_root)] + \ + ["-d", str(database / "coref_arkts_src.db")] + if options: + for (key, value) in options.items(): + if key == "blacklist" or key == "b": + cmd += ["--blacklist"] + value.split(",") + elif key == "use-gitignore": + cmd += ["--use-gitignore"] + elif key == "extract-text": + cmd += ["--extract-text"] + elif key == "extract-deps": + cmd += ["--extract-deps"] + elif key == "file-size-limit": + cmd += ["--file-size-limit", value] + elif key == "paths": + cmd += ["--paths", value] + else: + logging.warning("unsupported config name:%s for arkts extractor.", key) + return cmd + + +def jar_extractor_cmd(extractor_path, source_root, database, options): + jvm_opts = None + if options: + for (key, value) in options.items(): + if key == "jvm_opts": + # jvm_opts from user specified extract config + jvm_opts = value + + # if no jvm_opts from extract config, calculate xmx according to current memory. + if not jvm_opts: + mem = psutil.virtual_memory() + total_memory = mem.total + pod_memory_limit = get_pod_memory_limit() + if pod_memory_limit != 0: + total_memory = pod_memory_limit + total_memory_gb = round(total_memory / (1024 ** 3)) + total_memory_gb = min(total_memory_gb, 32) # limit to 32G + xmx = max(total_memory_gb - 1, 6) + logging.info("current memory is: %s GB, will use xmx: %s GB.", total_memory_gb, xmx) + jvm_opts = f"-Xmx{xmx}g" + + logging.info("extract jvm_opts is: %s .", jvm_opts) + + cmd = list() + cmd += ["java"] + shlex.split(jvm_opts) + ["-jar", str(extractor_path)] cmd += [str(source_root), str(database)] return cmd @@ -190,10 +233,9 @@ def extractor_run(language, source_root, database, timeout, options): tmp = Runner(cmd, timeout) return tmp.subrun() else: - logging.error("Not supported language: %s", language) + logging.error("Failed to obtain the %s extractor", language) return -1 - def get_pod_memory_limit(): # cgroup 文件系统路径 memory_limit_path = "/sys/fs/cgroup/memory/memory.limit_in_bytes" @@ -209,7 +251,4 @@ def get_pod_memory_limit(): logging.error(f"IO error occurred when accessing cgroup files: {e}") except Exception as e: logging.error(f"An unexpected error occurred: {e}") - return memory_limit - - - + return memory_limit \ No newline at end of file diff --git a/cli/godel/godel_compiler.py b/cli/godel/godel_compiler.py index b8e853a4..78195db4 100644 --- a/cli/godel/godel_compiler.py +++ b/cli/godel/godel_compiler.py @@ -1,20 +1,25 @@ import logging -import tempfile -import time -from pathlib import Path +import re +import chardet from run.runner import Runner from sparrow_schema.schema import sparrow +def get_encoding(file_path): + with open(file_path, 'rb') as f: + result = chardet.detect(f.read()) + return result['encoding'] + + def godel_version_judge(path) -> str: # 判断脚本对应的godel编译器版本 - result = "script" + result = "0.3" try: - with open(path, "r") as f: + with open(path, "r", encoding=get_encoding(path)) as f: tmp = f.readline() - if "1.0" in tmp: - result = "1.0" + if re.match(r'//[ \t]*script', tmp): + result = "script" except Exception as e: logging.error(f"godel version judge error: {str(e)}") return result @@ -23,8 +28,8 @@ def godel_version_judge(path) -> str: def get_godel_compile(path): version = godel_version_judge(path) godel = "" - if version == "1.0": - godel = sparrow.godel_1_0 + if version == "0.3": + godel = sparrow.godel_0_3 elif version == "script": godel = sparrow.godel_script return godel @@ -35,7 +40,8 @@ def backend_execute(path, database, output, timeout, output_format, verbose): version = godel_version_judge(path) cmd = list() cmd += [str(godel), str(path), "--run-souffle-directly", "--package-path"] - cmd += [str(sparrow.lib_1_0)] + if version == "0.3": + cmd += [str(sparrow.lib_03)] if database is not None: cmd += ["--souffle-fact-dir", database] cmd += ["--souffle-output-format", output_format, "--souffle-output-path", output] @@ -45,23 +51,33 @@ def backend_execute(path, database, output, timeout, output_format, verbose): return tmp.subrun() +def precompiled(path, timeout): + cmd = [str(sparrow.godel_script), "-p", str(sparrow.lib_script), "--semantic-only", str(path)] + tmp = Runner(cmd, timeout) + status = tmp.subrun() + if status != 0: + return False + return True + + def execute(path, database, output, timeout, output_format, verbose): godel = get_godel_compile(path) version = godel_version_judge(path) cmd = list() if version == "script": - # godel-script两步编译，实际执行后端为1.0 - with tempfile.NamedTemporaryFile(suffix='.gdl') as temp_file: - cmd += [str(godel), str(path), "-p", str(sparrow.lib_1_0), "-o", temp_file.name] - if verbose: - cmd += ["--verbose"] - tmp = Runner(cmd, timeout) - start_time = time.time() - return_code = tmp.subrun() - if return_code != 0: - logging.error("%s compile error, please check it yourself", str(path)) - return -1 - logging.info("godel-script compile time: %.2fs", time.time() - start_time) - return backend_execute(Path(temp_file.name), database, output, timeout, output_format, verbose) + # godel-script 直接执行 + cmd += [str(godel), "-p", str(sparrow.lib_script), "-f", database] + cmd += ["-Of", "-r", str(path)] + if output_format == "sqlite": + cmd += ["--output-sqlite"] + elif output_format == "csv": + cmd += ["--output-csv"] + else: + cmd += ["--output-json"] + cmd += [output] + if verbose: + cmd += ["--verbose"] + tmp = Runner(cmd, timeout) + return tmp.subrun() else: return backend_execute(path, database, output, timeout, output_format, verbose) diff --git a/cli/package/__init__.py b/cli/package/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cli/package/compile_and_run.py b/cli/package/compile_and_run.py new file mode 100644 index 00000000..0b2a770b --- /dev/null +++ b/cli/package/compile_and_run.py @@ -0,0 +1,134 @@ +import os +import subprocess + +from query.run import get_files +from sparrow_schema.schema import * + + +def compile_query_and_get_reference(compiler, library_directory, query, output_directory): + logging.info("Compile " + str(query) + " to " + output_directory + "/out.gdl") + arg = [ + compiler, "-p", library_directory, + str(query), "-u", "-o", output_directory + "/out.gdl" + ] + result = subprocess.run(arg, stdout=subprocess.PIPE) + + if result.returncode != 0: + logging.error("Compile " + str(query) + " failed") + return + + reference = result.stdout.decode() + reference = reference.split("\n") + reference.pop() + return reference + + +def create_compiled_library_directory(directory_path) -> bool: + if not os.path.exists(directory_path): + os.mkdir(directory_path) + return True + if not os.path.isdir(directory_path): + logging.error("\"" + directory_path + "\" is not a directory, failed to create") + return False + return True + + +def compile_referenced_library_files(compiler_path, library_directory, reference_list, output_directory): + compile_fail_count = 0 + for reference in reference_list: + # module name is the first word of the line + module_name = reference[:reference.find(" ")] + # library file path is after the space + library_file_path = reference[reference.find(" ") + 1:] + # output path is the module name with "::" replaced by "." + output_path = output_directory + "/lib/" + output_path += module_name.replace("::", ".") + ".gdl" + # compile the library file to the output path + result = subprocess.run([ + compiler_path, "-p", library_directory, library_file_path, "-o", output_path + ]) + # check if compilation failed + if result.returncode != 0: + compile_fail_count += 1 + logging.error("Failed to compile " + output_path) + else: + logging.info("Compile " + library_file_path + " to " + output_path) + + if compile_fail_count != 0: + return False + return True + + +def compile_and_run(compiler_path, + library_path, + query_file, + executable_path, + database_path="", + verbose_mode=False): + output_directory = "out" + output_file = output_directory + "/" + query_file.stem + ".json" + logging.info("Start executing " + str(query_file)) + if not create_compiled_library_directory(output_directory): + return + if not create_compiled_library_directory(output_directory + "/lib"): + return + reference_list = compile_query_and_get_reference( + compiler_path, + library_path, + query_file, + output_directory + ) + if type(reference_list) != list: + return + if not compile_referenced_library_files( + compiler_path, + library_path, + reference_list, + output_directory + ): + return + args = [ + executable_path, + output_directory + "/out.gdl", + "--package-path", + output_directory + "/lib", + "--run-souffle-directly", + "--souffle-output-format", + "json", + "--souffle-output-path", + output_file + ] + if type(database_path) == str and len(database_path) > 0: + args.append("--souffle-fact-dir") + args.append(database_path) + if verbose_mode: + args.append("--verbose") + result = subprocess.run(args) + logging.info("Excution exited with code " + str(result.returncode) + ". Check result in " + output_file) + + +def package_run(args): + verbose_flag = args.verbose + database_path = args.database + godel_path_list = [] + for godel_dir in args.godel_dir: + godel_path_list += get_files(godel_dir, ".gdl") + godel_path_list += get_files(godel_dir, ".gs") + if not godel_path_list: + logging.error("There is no godel script(ends with .gs or .gdl) in the specified directory," + "Please redefine the directory or file by --gdl or -gdl") + raise ValueError("path lib error") + if not os.path.exists("./godel_modules"): + logging.error("Expect godel_modules directory here, " + "use package create --install to install packages into godel_modules") + return + for godel_path in godel_path_list: + compile_and_run( + sparrow.godel_script, + "./godel_modules", + godel_path, + sparrow.godel_1_0, + database_path, + verbose_flag + ) + return diff --git a/cli/package/create.py b/cli/package/create.py new file mode 100644 index 00000000..e917f139 --- /dev/null +++ b/cli/package/create.py @@ -0,0 +1,11 @@ +import package.manifest as manifest +import package.pack as pack +import package.compile_and_run as cr + +def package_create(args): + if args.manifest: + manifest.manifest_create() + elif args.pack: + pack.pack_to_zip() + elif args.install: + pack.unpack_from_zip(args.install) \ No newline at end of file diff --git a/cli/package/manifest.py b/cli/package/manifest.py new file mode 100644 index 00000000..161dc9fc --- /dev/null +++ b/cli/package/manifest.py @@ -0,0 +1,58 @@ +import json +import os +import logging +from sparrow_schema.schema import * + +# manifest.json format example: +# { +# "name": "package_name", +# "version": "package_version", +# "compiler-version": "godel_compiler_version", +# "workspace": [ +# "path/of/godel/file.gdl", +# "path/of/godel/file.gdl" +# ], +# "dependency": { +# "package_name": "version_number", +# "package_name": "version_number", +# } +# } + +def manifest_create(): + if os.path.exists("manifest.json"): + logging.error("manifest.json exists in this directory") + return + manifest = {} + manifest["name"] = "" + manifest["version"] = "0.0.1" + manifest["compiler-version"] = "" + manifest["workspace"] = [] + manifest["dependency"] = {} + json.dump(manifest, open("manifest.json", "w")) + logging.info("Successfully generate manifest.json") + +def manifest_read(): + if not os.path.exists("manifest.json"): + return + return json.load(open("manifest.json")) + +def manifest_check(manifest_object) -> bool: + if type(manifest_object) != dict: + logging.error("There is no manifest.json file in this directory") + return False + if not "name" in manifest_object: + logging.error("There is no name field in the manifest.json file") + return False + if not "version" in manifest_object: + logging.error("There is no version field in the manifest.json file") + return False + if not "compiler-version" in manifest_object: + logging.error("There is no compiler-version field in the manifest.json file") + return False + if not "workspace" in manifest_object: + logging.error("There is no workspace field in the manifest.json file") + return False + if not "dependency" in manifest_object: + logging.error("There is no dependency field in the manifest.json file") + return False + return True diff --git a/cli/package/pack.py b/cli/package/pack.py new file mode 100644 index 00000000..41df74d4 --- /dev/null +++ b/cli/package/pack.py @@ -0,0 +1,35 @@ +import zipfile +import package.manifest as manifest +import os +import logging +from pathlib import Path + +def pack_to_zip(): + project = manifest.manifest_read() + if not manifest.manifest_check(project): + return + if len(project["name"]) == 0: + logging.error("Failed to pack, project name is empty.") + return + zipfile_name = project["name"] + ".zip" + output = zipfile.ZipFile(zipfile_name, "w") + for file in project["workspace"]: + if os.path.isdir(file): + for root, dirs, files in os.walk(file): + for f in files: + logging.info("Pack " + os.path.join(root, f)) + output.write(os.path.join(root, f)) + elif os.path.isfile(file): + logging.info("Pack " + file) + output.write(file) + output.write("manifest.json") + output.close() + logging.info("Successfully generate " + zipfile_name) + +def unpack_from_zip(zip_path): + if not os.path.exists(zip_path): + logging.error("Cannot find zip file \"" + zip_path + "\"") + return + name = Path(zip_path).stem + logging.info("Extract " + zip_path + " to godel_modules/" + name) + zipfile.ZipFile(zip_path).extractall("godel_modules/" + name) \ No newline at end of file diff --git a/cli/query/run.py b/cli/query/run.py index 392cc321..c474386e 100644 --- a/cli/query/run.py +++ b/cli/query/run.py @@ -1,8 +1,15 @@ +import json import logging +import re +import shutil +import sqlite3 +import tempfile import time from pathlib import Path -from godel.godel_compiler import execute +from run.runner import Runner +from sparrow_schema.schema import sparrow +from godel.godel_compiler import execute, precompiled def get_files(target_path, suffix) -> list: @@ -22,21 +29,50 @@ def get_files(target_path, suffix) -> list: # 结果是否为空判断 def check_is_empty_query_result(output_path, output_format): query_result = "EMPTY" - try: - with open(output_path, "r") as f: - content = f.read() - if output_format == "json": - if not content.startswith("[]"): - query_result = "NOT-EMPTY" - elif output_format == "csv": - if content is not None: - query_result = "NOT-EMPTY" - except FileNotFoundError: - logging.warning("%s not exist, Please check the script to see if there is output", output_path) - except PermissionError as pe: - logging.warning(f"can not open output file: {str(pe)}") - except Exception as e: - logging.warning(f"can not get output: {str(e)}") + if output_format == "sqlite": + query_result = "NOT-EMPTY" + # # 连接到 SQLite 数据库 + # conn = sqlite3.connect(output_path) + # cursor = conn.cursor() + # # 构建 SQL 查询语句 + # try: + # # 执行查询 + # query = "SELECT name FROM sqlite_master WHERE type='table'" + # cursor.execute(query) + # tables = cursor.fetchall() + # cursor.execute(query) + # for table_name in tables: + # query = f"SELECT COUNT(*) FROM \"{table_name}\"" + # cursor.execute(query) + # result = cursor.fetchone() + # row_count = result[0] if result else 0 + # + # # 返回是否为空的布尔值 + # if row_count != 0: + # query_result = "NOT-EMPTY" + # except sqlite3.Error as e: + # print(f"An error occurred: {e}") + # return False # 如果表不存在，也可以返回 False 或处理异常 + # finally: + # # 关闭连接 + # cursor.close() + # conn.close() + else: + try: + with open(output_path, "r") as f: + content = f.read() + if output_format == "json": + if not content.startswith("[]"): + query_result = "NOT-EMPTY" + elif output_format == "csv": + if content is not None: + query_result = "NOT-EMPTY" + except FileNotFoundError: + logging.warning("%s not exist, Please check the script to see if there is output", output_path) + except PermissionError as pe: + logging.warning(f"can not open output file: {str(pe)}") + except Exception as e: + logging.warning(f"can not get output: {str(e)}") return query_result @@ -49,9 +85,13 @@ def conf_check(args): return False output_path = Path(args.output).expanduser().resolve() if not output_path.exists(): - logging.error("conf error: output directory %s does not exist, " - "Please redefine the directory or file by --output or -o", str(output_path)) - return False + logging.warning("%s not exists, it will be created", str(output_path)) + try: + output_path.mkdir(parents=True) + logging.info("%s success build", str(output_path)) + except Exception as e: + logging.error("can not to create output directory %s: %s", str(output_path), e) + return False if not output_path.is_dir(): logging.error("conf error: output need to be dir," "Please redefine the directory by --output or -o") @@ -89,6 +129,152 @@ def conf_check(args): return True +def get_lines_from_file(file_name, start_line, end_line): + content = "" + with open(file_name, 'r') as file: + for line_num, line in enumerate(file, start=1): + if start_line <= line_num <= end_line: + content += line + elif line_num > end_line: + break + return content + + +def compare_strings_ignore_whitespace(str1, str2): + # 移除空格、制表符和换行符 + pattern = r'\s+' + str1_clean = re.sub(pattern, '', str1) + str2_clean = re.sub(pattern, '', str2) + + # 进行比较 + return str1_clean == str2_clean + + +# 多脚本合并执行 +def merge_execute(args, gdl_list, timeout): + start_time = time.time() + # 各个脚本预编译 + for gdl in gdl_list: + if not precompiled(gdl, timeout - (time.time() - start_time)): + raise ValueError("precompiled error, Please check the script to determine syntax correctness and whether " + "it is a godel-script script") + # 脚本合并, 头文件和输出合并,重名函数,重名类仅会保留一份，若类接口函数不一致也仅会保留一份。不一致部分去除 + func_name = dict() + schema = dict() + import_set = set() + output = set() + out_map = dict() + with tempfile.NamedTemporaryFile(suffix='.gdl', mode="w+") as merge_file: + merge_file.writelines("// script\n") + merge_file_content = "" + for gdl in gdl_list: + with open(str(gdl), "r") as f: + pattern = r'use coref::(\w+)::\*' + for line in f: + if re.search(pattern, line): + if line not in import_set: + import_set.add(line) + with tempfile.NamedTemporaryFile(suffix='.json', mode="w+") as tmp_loc: + cmd = [str(sparrow.godel_script), str(gdl), "-l", str(tmp_loc.name)] + tmp = Runner(cmd, timeout - (time.time() - start_time)) + status = tmp.subrun() + if status != 0: + raise ValueError(gdl + " loc get error") + with open(str(tmp_loc.name), "r") as f: + loc_data = json.load(f) + for tmp in loc_data["schema"]: + schema_name = tmp["name"] + content = get_lines_from_file(gdl, tmp["location"]["begin_line"], + tmp["location"]["end_line"]) + "\n" + if schema_name not in schema: + schema[schema_name] = content + merge_file_content += content + else: + if not compare_strings_ignore_whitespace(content, schema[schema_name]): + logging.error("%s schema have different behaviors, please modify each function with the " + "same name yourself.", schema_name) + raise ValueError("merge error: same schema") + for tmp in loc_data["funcs"]: + if tmp["name"] == "main": + output_pattern = r'output\((\w+)\(\)\)' + main_content = get_lines_from_file(gdl, tmp["location"]["begin_line"], + tmp["location"]["end_line"]) + # 在 main 中查找 output(...) + output_matches = re.findall(output_pattern, main_content) + out_map[str(gdl)] = output_matches + for out in output_matches: + if out not in output: + output.add(out) + continue + hash_name = tmp["name"] + "#" + tmp["schema"] + content = get_lines_from_file(gdl, tmp["location"]["begin_line"], tmp["location"]["end_line"]) + if hash_name not in func_name: + func_name[hash_name] = content + if tmp["schema"] != "": + content = "impl " + tmp["schema"] + " {\n" + content + "\n}\n" + merge_file_content += content + else: + if not compare_strings_ignore_whitespace(content, func_name[hash_name]): + logging.error("%s function have different behaviors, please modify each function with the " + "same name yourself.", tmp["name"]) + raise ValueError("merge error: same function") + # 头文件合并 + for tmp in import_set: + merge_file_content = tmp + "\n" + merge_file_content + # 输出合并 + merge_file_content += "\nfn main() {\n" + for tmp in output: + merge_file_content += " output(" + tmp + "())\n" + merge_file_content += "}\n" + merge_file.write(merge_file_content) + merge_file.seek(0) + # 执行 + with tempfile.NamedTemporaryFile(suffix='.json', mode="w+") as output_file: + output = str(output_file.name) + if args.database: + database = str(Path(args.database).expanduser().resolve()) + return_code = execute(merge_file.name, database, output, args.timeout - (time.time() - start_time), + args.format, args.verbose) + else: + return_code = execute(merge_file.name, None, output, args.timeout - (time.time() - start_time), + args.format, args.verbose) + if return_code == 0: + query_result = check_is_empty_query_result(output, args.format) + logging.info("Task %s is %s, result is %s, execution time is %.2fs.", + "merge", "success", query_result, time.time() - start_time) + else: + logging.error("Task %s is %s, result is %s, execution time is %.2fs.", + "merge", "fail", "null", time.time() - start_time) + logging.error("%s execute error, please check by log", str(merge_file)) + if return_code == 0: + logging.info("run success") + else: + logging.error("run fail, There are scripts that cannot be executed among the scripts that need to be " + "executed. Please locate them based on the logs.") + raise RuntimeError("run fail") + # 输出处理: 根据各个文件的output进行分割,若后续添加注解进行输出此步骤需要修改。 + output_json = json.load(output_file) + if isinstance(output_json, list): + # 说明此时单文件单输出,gdl_list中只有一个文件 + for gdl in gdl_list: + if len(out_map[str(gdl)]) != 0: + output = str(Path(args.output).expanduser().resolve() / (gdl.stem + "." + args.format)) + shutil.copy2(output_file.name, output) + else: + # 单文件多输出或者多文件多输出: + for gdl in gdl_list: + if len(out_map[str(gdl)]) != 0: + output = str(Path(args.output).expanduser().resolve() / (gdl.stem + "." + args.format)) + with open(output, "w") as f: + if len(out_map[str(gdl)]) == 1: + content = output_json[out_map[str(gdl)][0]] + else: + content = dict() + for tmp in out_map[str(gdl)]: + content[tmp] = output_json[tmp] + f.write(json.dumps(content, indent=4)) + + def query_run(args): # conf 检查 if not conf_check(args): @@ -103,11 +289,18 @@ def query_run(args): logging.error("There is no godel script(ends with .gs or .gdl) in the specified directory," "Please redefine the directory or file by --gdl or -gdl") raise ValueError("path lib error") + + # 不同执行选项: + if args.merge: + logging.warning("When merging execution, please make sure that single reservation of functions or classes " + "with the same name will not affect the execution results.") + merge_execute(args, godel_path_list, args.timeout) + return status = 1 # 目前先各自执行: start_time = time.time() for godel_query_script in godel_path_list: - output = str(Path(args.output).expanduser().resolve()/(godel_query_script.stem + "." + args.format)) + output = str(Path(args.output).expanduser().resolve() / (godel_query_script.stem + "." + args.format)) if args.database: database = str(Path(args.database).expanduser().resolve()) return_code = execute(godel_query_script, database, output, args.timeout - (time.time() - start_time), @@ -117,16 +310,17 @@ def query_run(args): args.format, args.verbose) if return_code == 0: query_result = check_is_empty_query_result(output, args.format) - logging.info("Task %s is %s, result is %s, execution time is %.2fs.", + logging.info("Task %s is %s, result is %s, execution time is %.2fs.", str(godel_query_script), "success", query_result, time.time() - start_time) else: status = 0 - logging.error("Task %s is %s, result is %s, execution time is %.2fs.", + logging.error("Task %s is %s, result is %s, execution time is %.2fs.", str(godel_query_script), "fail", "null", time.time() - start_time) logging.error("%s execute error, please check by log", str(godel_query_script)) if status == 1: logging.info("run success") else: - logging.error("run fail, please check by log") + logging.error("run fail, There are scripts that cannot be executed among the scripts that need to be " + "executed. Please locate them based on the logs.") raise RuntimeError("run fail") return diff --git a/cli/rebuild/lib.py b/cli/rebuild/lib.py index c8d9313f..9c1da42b 100644 --- a/cli/rebuild/lib.py +++ b/cli/rebuild/lib.py @@ -3,6 +3,7 @@ import shutil import tempfile import time +from pathlib import Path from query.run import get_files from run.runner import Runner @@ -11,9 +12,10 @@ file_list = dict() -# 目前仅开放下列5种语言的godel-script写法,后续根据开放的语言进行修改 +# lib库与抽取器分开,逻辑上可以有一个库只有lib库无抽取器。 def open_lib(): - return ["java", "javascript", "go", "xml", "python"] + lib = ["java", "javascript", "go", "xml", "python", "sql", "cfamily", "properties", "swift", "arkts"] + return lib def get_rebuild_lang(args): @@ -41,39 +43,51 @@ def line_replace(output, line, base): return output.replace(line, " " * (len(line) - len(str(actual_line))) + str(actual_line), 1) -# 日志输出修改:目前将lib和一进行编译，此处out通过行号将错误信息对应会原文件 -def log_out(stream): - global file_list - for line in iter(stream.readline, b''): - output = line.strip() - if output: - if "-->" in output: - match = re.search(r'-->\s*([^:]+):(\d+)', output) - if match: - file_name = match.group(1) - line = match.group(2) - num = 0 - for file in file_list: - num += file["lines"] - if int(line) <= num: - output = output.replace(file_name, str(file["name"]), 1) - output = line_replace(output, line, num - file["lines"]) - break - else: - line = extract_first_number(output) - if line is not None: - num = 0 - for file in file_list: - num += file["lines"] - if int(line) <= num: - output = line_replace(output, line, num - file["lines"]) - break - print(output) +def log_mapping(output): + result = output + if "-->" in output: + match = re.search(r'-->\s*([^:]+):(\d+)', output) + if match: + file_name = match.group(1) + line = match.group(2) + num = 0 + for file in file_list: + num += file["lines"] + if int(line) <= num: + result = output.replace(file_name, str(file["name"]), 1) + result = line_replace(result, line, num - file["lines"]) + break + else: + line = extract_first_number(output) + if line is not None: + num = 0 + for file in file_list: + num += file["lines"] + if int(line) <= num: + result = line_replace(output, line, num - file["lines"]) + break + return result + + +# 日志输出修改:目前将lib和一进行编译，此处out通过行号将错误信息对应回原文件 +def log_info(output): + if output: + output = log_mapping(output) + logging.info(output.strip()) + + +def log_error(output): + if output: + output = log_mapping(output) + logging.error(output.strip()) def rebuild_lang(lib, verbose): global file_list lib_path = sparrow.lib_script / "coref" / lib + if not Path(lib_path).exists(): + logging.warning("lib %s not exist, please check", str(lib_path)) + return file_list = list() gdl_list = get_files(lib_path, ".gs") gdl_list += get_files(lib_path, ".gdl") @@ -95,18 +109,17 @@ def rebuild_lang(lib, verbose): num += len(lines) output_file.seek(0) cmd = list() - with tempfile.NamedTemporaryFile(suffix='.gdl') as tmp_out: - cmd += [str(sparrow.godel_script), str(output_file.name), "-p", str(sparrow.lib_1_0), "-o", tmp_out.name] - if verbose: - cmd += ["--verbose"] - tmp = Runner(cmd, 3600) - start_time = time.time() - return_code = tmp.subrun(log_out) - if return_code != 0: - logging.error("%s lib compile error, please check it yourself", lib_path) - return -1 - shutil.copy2(tmp_out.name, sparrow.lib_1_0 / ("coref." + lib + ".gdl")) - logging.info("%s lib compile success time: %.2fs", lib_path, time.time() - start_time) + cmd += [str(sparrow.godel_script), "--semantic-only", str(output_file.name)] + if verbose: + cmd += ["--verbose"] + tmp = Runner(cmd, 3600) + start_time = time.time() + return_code = tmp.subrun(log_info, log_error) + if return_code != 0: + logging.error("%s lib compile error, please check it yourself", lib_path) + return -1 + shutil.copy2(output_file.name, sparrow.lib_script / "coref" / (lib + ".gdl")) + logging.info("%s lib compile success time: %.2fs", lib_path, time.time() - start_time) def rebuild_lib(args): diff --git a/cli/requirements.txt b/cli/requirements.txt index 63c468f0..47bde349 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1 +1,3 @@ -psutil>=5.9.5 \ No newline at end of file +psutil==5.9.6 +pyinstaller==6.3.0 +chardet==5.2.0 \ No newline at end of file diff --git a/cli/run/__init__.py b/cli/run/__init__.py index 8b137891..e69de29b 100644 --- a/cli/run/__init__.py +++ b/cli/run/__init__.py @@ -1 +0,0 @@ - diff --git a/cli/run/runner.py b/cli/run/runner.py index 0226d7e3..fcbc98f8 100644 --- a/cli/run/runner.py +++ b/cli/run/runner.py @@ -1,14 +1,38 @@ +# -*-coding:utf-8-*- + import logging +import shlex import subprocess import threading -import shlex -def output_stream(stream): - for line in iter(stream.readline, b''): - output = line.strip() - if output: - print(output) +def stream_reader(stream, handler): + for line in iter(stream.readline, ''): + handler(line) + stream.close() + + +def log_info(message): + logging.info(message.strip()) + + +def log_error(message): + logging.error(message.strip()) + + +def terminate_process(proc, timeout, timers): + """终止进程并取消所有超时器""" + if proc.poll() is None: # 如果进程没有结束 + proc.kill() + proc.terminate() # 尝试终端 + logger = logging.getLogger() + for hdlr in logger.handlers: + hdlr.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) + logging.error("Execution timeout, You can add -t option to adjust timeout") + + # 取消所有定时器 + for timer in timers: + timer.cancel() class Runner: @@ -16,26 +40,53 @@ def __init__(self, cmd, timeout_seconds): self.cmd = cmd self.timeout_seconds = timeout_seconds - def subrun(self, output=None): + def subrun(self, info_out=None, error_out=None): logging.info("execute : %s", shlex.join(self.cmd)) + process = None + return_code = -1 try: - process = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - encoding="utf-8") - if output is None: - output = output_stream - output_thread = threading.Thread(target=output, args=(process.stdout,)) - output_thread.daemon = True # 设置为守护线程 - output_thread.start() - - process.wait(timeout=self.timeout_seconds) - - if process.returncode is None: - # 超时处理 - process.kill() - logging.error("execute time > %s s, time out, You can add -t option to adjust timeout", - self.timeout_seconds) + process = subprocess.Popen( + self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + encoding="utf-8", bufsize=1, text=True + ) + + logger = logging.getLogger() + for hdlr in logger.handlers: + hdlr.setFormatter(logging.Formatter('%(message)s')) + if not info_out: + info_out = log_info + if not error_out: + error_out = log_error + + # 为超时创建一个Timer并开始计时 + timeout_timer = threading.Timer(self.timeout_seconds, terminate_process, + [process, self.timeout_seconds, []]) + + stdout_thread = threading.Thread(target=stream_reader, args=(process.stdout, info_out)) + stderr_thread = threading.Thread(target=stream_reader, args=(process.stderr, error_out)) + + stdout_thread.start() + stderr_thread.start() + + timeout_timer.start() # 启动超时计时器 + + # 主线程等待子进程结束 + return_code = process.wait() + + # 子进程结束后取消超时计时器 + timeout_timer.cancel() + + # 等待读取子进程的输出/错误的线程可以正确结束 + stdout_thread.join() + stderr_thread.join() return_code = process.wait() - return return_code except Exception as e: logging.error("execute error: %s", e) return -1 + finally: + logger = logging.getLogger() + for hdlr in logger.handlers: + hdlr.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) + if process: + process.kill() + return return_code diff --git a/cli/sparrow b/cli/sparrow index 50c5f70d..a3311c2f 100644 --- a/cli/sparrow +++ b/cli/sparrow @@ -12,16 +12,4 @@ function get_realpath() { echo "${previous}" } -PYTHON="" -if command -v python > /dev/null 2>&1; then - PYTHON="python" -elif command -v python2 > /dev/null 2>&1; then - PYTHON="python2" -elif command -v python3 > /dev/null 2>&1; then - PYTHON="python3" -else - echo "No python interpreter founded." - exit -1 -fi - -exec $PYTHON "$(dirname $(get_realpath "$0"))"/cli/sparrow-cli.zip --sparrow-cli-internal "$(dirname $(get_realpath "$0"))" "$@" +exec "$(dirname $(get_realpath "$0"))"/cli/sparrow-cli --sparrow-cli-internal "$(dirname $(get_realpath "$0"))" "$@" diff --git a/cli/sparrow-cli.py b/cli/sparrow-cli.py index 08fa6f2e..e9eae56b 100644 --- a/cli/sparrow-cli.py +++ b/cli/sparrow-cli.py @@ -9,10 +9,27 @@ from query.run import * from extractor.extractor import Extractor from database.create import database_create +from package.create import package_create +from package.compile_and_run import package_run SPARROW_HOME = os.path.abspath(os.path.join(os.path.join(sys.path[0], ".."))) +class ErrorFilter(logging.Filter): + def filter(self, record): + return record.levelno == logging.ERROR + + +class InfoFilter(logging.Filter): + def filter(self, record): + return record.levelno == logging.INFO + + +class WarningFilter(logging.Filter): + def filter(self, record): + return record.levelno == logging.WARNING + + def extractor_choices(): extractor_list = list() for name, value in vars(Extractor).items(): @@ -21,6 +38,18 @@ def extractor_choices(): return extractor_list +def sparrow_check(): + for name, value in vars(Extractor).items(): + if isinstance(value, Path): + if not value.exists(): + return False + for name, value in vars(sparrow).items(): + if isinstance(value, Path): + if not value.exists(): + return False + return True + + def parse_args(): global SPARROW_HOME parser = argparse.ArgumentParser(description='sparrow-cli', @@ -29,7 +58,6 @@ def parse_args(): parser.add_argument("--sparrow-home", "-sparrow-home", dest='sparrow_home', help='sparrow home, You can specify the sparrow location yourself') parser.add_argument('--sparrow-cli-internal', dest='sparrow_home_internal', help=argparse.SUPPRESS) - parser.add_argument('--verbose', action='store_true', help='Enable verbose mode') subparsers = parser.add_subparsers(title='subcommands', dest='subcommand') # 子命令query subparser_query = subparsers.add_parser('query', help='execute godel script') @@ -38,7 +66,7 @@ def parse_args(): # query的子命令run subparser_query_run = sub_subparser_query.add_parser('run', help='execute godel script') subparser_query_run.add_argument("--database", '-d', help='Directory to a godel database to query') - subparser_query_run.add_argument("--format", '-f', choices=['csv', 'json'], default='json', + subparser_query_run.add_argument("--format", '-f', choices=['csv', 'json', "sqlite"], default='json', help='Select output format. default output json') subparser_query_run.add_argument("--output", '-o', default=os.getcwd(), help='Select output dir, The output file will have the same name as the Godel ' @@ -48,7 +76,33 @@ def parse_args(): subparser_query_run.add_argument("--gdl", '-gdl', required=True, nargs="*", dest="godel_dir", help='The location of the godel script that needs to execute') subparser_query_run.add_argument('--verbose', action='store_true', help='Enable verbose mode') - + subparser_query_run.add_argument('--merge', '-m', action='store_true', help='Combined execution of multiple ' + 'scripts,Only one function and class ' + 'with the same name in the script ' + 'will be retained, and the output and ' + 'header files will be merged ' + 'together. and they must to be ' + 'godel-script query') + subparser_query_run.add_argument('--log-dir', help='Log storage dir') + # 子命令package + subparser_package = subparsers.add_parser('package', help='godel script package manager') + # 子命令package的子解析器 + sub_subparser_package = subparser_package.add_subparsers(title='package_subcommands', dest='package_subcommands') + # package的子命令create + subparser_package_create = sub_subparser_package.add_parser('create', help='create package or manifest.json') + subparser_package_create.add_argument("--manifest", '-m', help='Create a new manifest file in this directory', + action="store_true") + subparser_package_create.add_argument("--pack", '-p', help='Pack up project and generate zip file', + action="store_true") + subparser_package_create.add_argument("--install", '-i', help='Unpack zip file and install') + subparser_package_create.add_argument('--log-dir', help='Log storage dir') + # package的子命令run + subparser_package_run = sub_subparser_package.add_parser('run', help='execute godel script with packages') + subparser_package_run.add_argument("--database", '-d', help='Directory to a godel database to query') + subparser_package_run.add_argument("--gdl", '-gdl', required=True, nargs="*", dest="godel_dir", + help='The location of the godel script that needs to execute') + subparser_package_run.add_argument('--verbose', action='store_true', help='Enable verbose mode') + subparser_package_run.add_argument('--log-dir', help='Log storage dir') # 子命令database subparser_database = subparsers.add_parser('database', help='data extract') # 子命令database子解析器 @@ -67,6 +121,7 @@ def parse_args(): subparser_database_create.add_argument("--timeout", '-t', type=int, default=3600, help='Set extract timeout in seconds, default 3600') subparser_database_create.add_argument('--verbose', action='store_true', help='Enable verbose mode') + subparser_database_create.add_argument('--log-dir', help='Log storage dir') subparser_database_create.add_argument('--overwrite', action='store_true', help='whether to overwrite the database, Default not') subparser_database_create.add_argument('--extraction-config-file', @@ -89,12 +144,14 @@ def parse_args(): 'by whitespace. e.g. ' '-lang java xml') subparser_rebuild_lib.add_argument('--verbose', action='store_true', help='Enable verbose mode') + subparser_rebuild_lib.add_argument('--log-dir', help='Log storage dir') args = parser.parse_args() # 如果没有参数输入，直接回车，则显示帮助信息 if len(sys.argv) == 1: parser.print_help() - logging.warning("please give conf to start sparrow as help, it will not start") + logging.warning("please give conf to start sparrow as help, it will not start, You can use -h to view the " + "help list") return # 设置SPARROW_HOME @@ -107,34 +164,75 @@ def parse_args(): sparrow(SPARROW_HOME) Extractor() + # 暂不开启 + # if not sparrow_check(): + # logging.error("Please specify the sparrow-cli directory correctly") + # return # 查询版本号, 查找配置文件 if args.version: print(sparrow.version) return - if args.verbose: - log_level = logging.DEBUG - else: - log_level = logging.INFO - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setLevel(level=log_level) - formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') - - console_handler.setFormatter(formatter) - logging.basicConfig(level=log_level, handlers=[console_handler]) logging.info("sparrow %s will start", sparrow.version) + if args.subcommand: + if args.verbose: + log_level = logging.DEBUG + else: + log_level = logging.INFO + logger = logging.getLogger() + logger.setLevel(log_level) + for hdlr in logger.handlers: + hdlr.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) + if args.log_dir: + log_format = '%(asctime)s %(levelname)s: %(message)s' + # error warning info 分开存储 + file_handler_error = logging.FileHandler(os.path.join(args.log_dir, 'sparrow-cli-error.log')) + formatter_error = logging.Formatter(log_format) + file_handler_error.setFormatter(formatter_error) + file_handler_error.addFilter(ErrorFilter()) + + file_handler_warning = logging.FileHandler(os.path.join(args.log_dir, 'sparrow-cli-warn.log')) + formatter_warning = logging.Formatter(log_format) + file_handler_warning.setFormatter(formatter_warning) + file_handler_warning.addFilter(WarningFilter()) + + file_handler_info = logging.FileHandler(os.path.join(args.log_dir, 'sparrow-cli-info.log')) + formatter_info = logging.Formatter(log_format) + file_handler_info.setFormatter(formatter_info) + file_handler_info.addFilter(InfoFilter()) + + logger.addHandler(file_handler_warning) + logger.addHandler(file_handler_info) + logger.addHandler(file_handler_error) + else: + parser.print_help() + logging.warning("please give conf to start sparrow as help, it will not start, You can use -h to view the " + "help list") + return if args.subcommand == "query": if args.query_subcommands == "run": query_run(args) + return elif args.subcommand == "database": if args.database_subcommands == "create": database_create(args) + return elif args.subcommand == "rebuild": if args.rebuild_subcommands == "lib": rebuild_lib(args) + else: + sub_subparser_rebuild.print_help() + return + elif args.subcommand == "package": + if args.package_subcommands == "create": + package_create(args) + elif args.package_subcommands == "run": + package_run(args) + return else: - logging.warning("please give conf to start sparrow, it will not start") - return + parser.print_help() + logging.warning("please give conf to start sparrow as help, it will not start, You can use -h to view the " + "help list") if __name__ == '__main__': diff --git a/cli/sparrow-cli.spec b/cli/sparrow-cli.spec new file mode 100644 index 00000000..489e6ff3 --- /dev/null +++ b/cli/sparrow-cli.spec @@ -0,0 +1,43 @@ +# -*- mode: python ; coding: utf-8 -*- + + +a = Analysis( + ['sparrow-cli.py'], + pathex=[], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='sparrow-cli', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='cli', +) diff --git a/cli/sparrow_schema/schema.py b/cli/sparrow_schema/schema.py index b8d0a402..34dbd601 100644 --- a/cli/sparrow_schema/schema.py +++ b/cli/sparrow_schema/schema.py @@ -4,28 +4,26 @@ class sparrow: home = "" - godel_1_0 = "" + godel_0_3 = "" godel_script = "" language = "" - lib = "" - lib_1_0 = "" + lib_03 = "" lib_script = "" version = "" def __init__(self, sparrow_home): sparrow.home = Path(sparrow_home).expanduser().resolve() - sparrow.godel_1_0 = sparrow.home/"godel-1.0"/"usr"/"bin"/"godel" + sparrow.godel_0_3 = sparrow.home/"godel-0.3"/"usr"/"bin"/"godel" sparrow.godel_script = sparrow.home/"godel-script"/"usr"/"bin"/"godel" sparrow.language = sparrow.home/"language" - sparrow.lib = sparrow.home/"lib" - sparrow.lib_1_0 = sparrow.home/"lib-1.0" - sparrow.lib_script = sparrow.home/"lib-script" + sparrow.lib_03 = sparrow.home/"lib-03" + sparrow.lib_script = sparrow.home/"lib" version_path = sparrow.home/"version.txt" if version_path.exists(): try: with open(version_path, "r") as f: content = f.read() - sparrow.version = content + sparrow.version = content.strip() except PermissionError as pe: sparrow.version = "no version file found!" logging.warning(f"can not open version.txt: {str(pe)}")