diff --git a/reclaimer/__init__.py b/reclaimer/__init__.py index 23639820..837e5abb 100644 --- a/reclaimer/__init__.py +++ b/reclaimer/__init__.py @@ -12,8 +12,8 @@ # ############## __author__ = "Sigmmma" # YYYY.MM.DD -__date__ = "2021.03.23" -__version__ = (2, 11, 2) +__date__ = "2024.03.18" +__version__ = (2, 22, 0) __website__ = "https://github.com/Sigmmma/reclaimer" __all__ = ( "animation", "bitmaps", "h2", "h3", "halo_script", "hek", "meta", "misc", diff --git a/reclaimer/animation/animation_compilation.py b/reclaimer/animation/animation_compilation.py index 3f19f2bb..3d463b5e 100644 --- a/reclaimer/animation/animation_compilation.py +++ b/reclaimer/animation/animation_compilation.py @@ -19,7 +19,7 @@ "ANIMATION_COMPILE_MODE_NEW", "ANIMATION_COMPILE_MODE_PRESERVE", "ANIMATION_COMPILE_MODE_ADDITIVE") -ANIMATION_COMPILE_MODE_NEW = 0 +ANIMATION_COMPILE_MODE_NEW = 0 ANIMATION_COMPILE_MODE_PRESERVE = 1 ANIMATION_COMPILE_MODE_ADDITIVE = 2 @@ -39,9 +39,9 @@ def compile_animation(anim, jma_anim, ignore_size_limits=False, endian=">"): default_data_size = jma_anim.default_data_size frame_data_size = jma_anim.frame_data_frame_size * stored_frame_count - max_frame_info_size = anim.frame_info.get_desc('MAX', 'size') - max_default_data_size = anim.default_data.get_desc('MAX', 'size') - max_frame_data_size = anim.frame_data.get_desc('MAX', 'size') + max_frame_info_size = util.get_block_max(anim.frame_info) + max_default_data_size = util.get_block_max(anim.default_data) + max_frame_data_size = util.get_block_max(anim.frame_data) if not ignore_size_limits: if frame_info_size > max_frame_info_size: @@ -103,10 +103,10 @@ def compile_animation(anim, jma_anim, ignore_size_limits=False, endian=">"): return errors -def compile_model_animations(antr_tag, jma_anim_set, ignore_size_limits=False, - animation_count_limit=256, delta_tolerance=None, - update_mode=ANIMATION_COMPILE_MODE_PRESERVE, - mod2_nodes=None): +def compile_model_animations( + antr_tag, jma_anim_set, ignore_size_limits=False, delta_tolerance=None, + update_mode=ANIMATION_COMPILE_MODE_PRESERVE, mod2_nodes=None + ): errors = [] tagdata = antr_tag.data.tagdata @@ -220,7 +220,7 @@ def compile_model_animations(antr_tag, jma_anim_set, ignore_size_limits=False, anim_index = antr_indices_by_type_strings.get( name_pieces, len(antr_anims)) - if anim_index >= animation_count_limit: + if anim_index >= util.get_block_max(tagdata.animations): errors.append( "Too many animations. Cannot add '%s'" % jma_anim_name) continue diff --git a/reclaimer/animation/jma.py b/reclaimer/animation/jma.py index 1e4a87bd..ae2cc274 100644 --- a/reclaimer/animation/jma.py +++ b/reclaimer/animation/jma.py @@ -27,6 +27,18 @@ ".jma", ".jmm", ".jmo", ".jmr", ".jmt", ".jmw", ".jmz", ) +JMA_VER_HALO_1_OLDEST_KNOWN = 16390 +JMA_VER_HALO_1_NODE_NAMES = 16391 +JMA_VER_HALO_1_NODE_HIERARCHY = 16392 +JMA_VER_HALO_1_RETAIL = JMA_VER_HALO_1_NODE_HIERARCHY + +JMA_VER_ALL = frozenset(( + JMA_VER_HALO_1_OLDEST_KNOWN, + JMA_VER_HALO_1_NODE_NAMES, + JMA_VER_HALO_1_NODE_HIERARCHY, + JMA_VER_HALO_1_RETAIL, + )) + def get_anim_ext(anim_type, frame_info_type, world_relative=False): anim_type = anim_type.lower() @@ -150,6 +162,7 @@ class JmaAnimation: node_list_checksum = 0 nodes = () frames = () + version = 0 rot_keyframes = () trans_keyframes = () @@ -173,7 +186,9 @@ class JmaAnimation: def __init__(self, name="", node_list_checksum=0, anim_type="", frame_info_type="", world_relative=False, - nodes=None, frames=None, actors=None, frame_rate=30): + nodes=None, frames=None, actors=None, frame_rate=30, + version=JMA_VER_HALO_1_RETAIL, + ): self.name = name.strip(" ") self.node_list_checksum = node_list_checksum @@ -184,6 +199,7 @@ def __init__(self, name="", node_list_checksum=0, self.frame_info_type = frame_info_type self.frame_rate = frame_rate self.actors = actors if actors else ["unnamedActor"] + self.version = version self.root_node_info = [] self.setup_keyframes() @@ -203,6 +219,11 @@ def has_keyframe_data(self): return False return True + @property + def has_node_names(self): return self.version >= JMA_VER_HALO_1_NODE_NAMES + @property + def has_node_hierarchy(self): return self.version >= JMA_VER_HALO_1_NODE_HIERARCHY + @property def ext(self): return get_anim_ext(self.anim_type, self.frame_info_type, @@ -438,8 +459,27 @@ def calculate_root_node_info(self): z += dz yaw += dyaw - verify_nodes_valid = JmsModel.verify_nodes_valid - get_node_depths = JmsModel.get_node_depths + def get_node_depths(self): + if self.has_node_hierarchy: + return JmsModel.get_node_depths(self) + return [] + + def verify_nodes_valid(self): + errors = [] + if self.has_node_hierarchy or len(self.nodes) not in range(1, 64): + return JmsModel.verify_nodes_valid(self) + elif self.has_node_names: + seen_names = set() + for i in range(len(self.nodes)): + n = self.nodes[i] + if len(n.name) >= 32: + errors.append("Node name node '%s' is too long." % n.name) + elif n.name.lower() in seen_names: + errors.append("Multiple nodes named '%s'." % n.name) + + seen_names.add(n.name.lower()) + + return errors def calculate_animation_flags(self, tolerance=None): if tolerance is None: @@ -486,9 +526,15 @@ def verify_animations_match(self, other_jma): errors.append("Node counts do not match.") return errors - for i in range(len(self.nodes)): - if not self.nodes[i].is_node_hierarchy_equal(other_jma.nodes[i]): - errors.append("Nodes '%s' do not match." % i) + if self.has_node_names and other_jma.has_node_names: + for i in range(len(self.nodes)): + if not self.nodes[i].is_node_hierarchy_equal( + other_jma.nodes[i], not ( + self.has_node_hierarchy and + other_jma.has_node_hierarchy + ) + ): + errors.append("Nodes '%s' do not match." % i) return errors @@ -516,6 +562,7 @@ class JmaAnimationSet: node_list_checksum = 0 nodes = () animations = () + version = JMA_VER_HALO_1_OLDEST_KNOWN def __init__(self, *jma_animations): self.nodes = [] @@ -526,14 +573,27 @@ def __init__(self, *jma_animations): verify_animations_match = JmaAnimation.verify_animations_match + @property + def has_node_names(self): return self.version >= JMA_VER_HALO_1_NODE_NAMES + @property + def has_node_hierarchy(self): return self.version >= JMA_VER_HALO_1_NODE_HIERARCHY + def merge_jma_animation(self, other_jma): assert isinstance(other_jma, JmaAnimation) if not other_jma: return + if other_jma.version > self.version and self.nodes: + # if the other jma's version is newer, grab its nodes if they + # match well enough, since they'll contain more detailed data + if (self.node_list_checksum == other_jma.node_list_checksum and + not self.verify_animations_match(other_jma)): + self.nodes = [] + if not self.nodes: self.node_list_checksum = other_jma.node_list_checksum + self.version = other_jma.version self.nodes = [] for node in other_jma.nodes: self.nodes.append( @@ -559,15 +619,18 @@ def read_jma(jma_string, stop_at="", anim_name=""): anim_name, ext = os.path.splitext(anim_name) anim_type, frame_info_type, world_relative = get_anim_types(ext) - jma_anim = JmaAnimation(anim_name, 0, anim_type, - frame_info_type, world_relative) jma_string = jma_string.replace("\n", "\t") data = tuple(d for d in jma_string.split("\t") if d) dat_i = 0 + version = parse_jm_int(data[dat_i]) - if parse_jm_int(data[dat_i]) != 16392: - print("JMA identifier '16392' not found.") + jma_anim = JmaAnimation(anim_name, 0, anim_type, + frame_info_type, world_relative, + version=version) + + if version not in JMA_VER_ALL: + print("Unknown JMA version '%s' found." % version) return jma_anim dat_i += 1 @@ -640,6 +703,10 @@ def read_jma(jma_string, stop_at="", anim_name=""): print("Could not read node list checksum.") return jma_anim + if jma_anim.node_list_checksum >= 0x80000000: + # jma gave us an unsigned checksum.... sign it + jma_anim.node_list_checksum -= 0x100000000 + if stop_at == "nodes": continue # read the nodes @@ -647,10 +714,20 @@ def read_jma(jma_string, stop_at="", anim_name=""): i = 0 # make sure i is defined in case of exception nodes[:] = [None] * node_count for i in range(node_count): - nodes[i] = JmsNode( - data[dat_i], parse_jm_int(data[dat_i+1]), parse_jm_int(data[dat_i+2]) - ) - dat_i += 3 + if jma_anim.has_node_hierarchy: + node_data = ( + data[dat_i], + parse_jm_int(data[dat_i+1]), + parse_jm_int(data[dat_i+2]) + ) + dat_i += 3 + elif jma_anim.has_node_names: + node_data = (data[dat_i], ) + dat_i += 1 + else: + node_data = ("fake_node_%d" % i, ) + + nodes[i] = JmsNode(*node_data) JmsNode.setup_node_hierarchy(nodes) except Exception: print(traceback.format_exc()) @@ -705,7 +782,7 @@ def write_jma(filepath, jma_anim, use_blitzkrieg_rounding=False): jma_anim.apply_root_node_info_to_states() with filepath.open("w", encoding='latin1', newline="\r\n") as f: - f.write("16392\n") # version number + f.write("%d\n" % jma_anim.version) f.write("%s\n" % jma_anim.frame_count) f.write("%s\n" % jma_anim.frame_rate) @@ -720,10 +797,14 @@ def write_jma(filepath, jma_anim, use_blitzkrieg_rounding=False): f.write("%s\n" % len(nodes)) f.write("%s\n" % int(jma_anim.node_list_checksum)) - for node in nodes: - f.write("%s\n%s\n%s\n" % - (node.name[: 31], node.first_child, node.sibling_index) - ) + if jma_anim.has_node_hierarchy: + for node in nodes: + f.write("%s\n%s\n%s\n" % + (node.name[: 31], node.first_child, node.sibling_index) + ) + elif jma_anim.has_node_names: + for node in nodes: + f.write("%s\n" % node.name[: 31]) for frame in frames: for nf in frame: diff --git a/reclaimer/animation/util.py b/reclaimer/animation/util.py index f4a5b11a..ac284f3f 100644 --- a/reclaimer/animation/util.py +++ b/reclaimer/animation/util.py @@ -11,9 +11,10 @@ from math import pi +from reclaimer.util import get_block_max from reclaimer.enums import unit_animation_names, unit_weapon_animation_names,\ unit_weapon_type_animation_names, vehicle_animation_names,\ - weapon_animation_names, device_animation_names, fp_animation_names,\ + weapon_animation_names, device_animation_names, fp_animation_names_mcc,\ unit_damage_animation_names from reclaimer.hek.defs.mod2 import TagDef, Pad, marker as mod2_marker_desc,\ node as mod2_node_desc, reflexive, Struct @@ -89,7 +90,7 @@ def split_anim_name_into_type_strings(anim_name): if part3: remainder = " ".join((part3, remainder)) part2, part2_sani, perm_num = split_permutation_number(part2, remainder) - if ((part1_sani == "first-person" and part2_sani in fp_animation_names) or + if ((part1_sani == "first-person" and part2_sani in fp_animation_names_mcc) or (part1_sani == "vehicle" and part2_sani in vehicle_animation_names) or (part1_sani == "device" and part2_sani in device_animation_names)): type_strings = part1_sani, part2_sani, perm_num @@ -215,9 +216,10 @@ def set_animation_index(antr_tag, anim_name, anim_index, if part1 == "suspension": anim_name = anim_name.split(" ", 1)[1] - anim_enums = antr_vehicles[0].suspension_animations.STEPTREE - if len(anim_enums) >= 8: - # max of 8 suspension animations + susp_anims = antr_vehicles[0].suspension_animations + anim_enums = susp_anims.STEPTREE + if len(anim_enums) >= get_block_max(susp_anims): + # at the limit of suspension animations return False anim_enums.append() @@ -226,7 +228,8 @@ def set_animation_index(antr_tag, anim_name, anim_index, elif part1 in ("first-person", "device", "vehicle"): if part1 == "first-person": - options = fp_animation_names + # NOTE: using mcc because they're the same, with an extension + options = fp_animation_names_mcc block = antr_fp_animations elif part1 == "device": options = device_animation_names @@ -238,6 +241,10 @@ def set_animation_index(antr_tag, anim_name, anim_index, if not block: block.append() + # trim the options to how many are actually allowed + # NOTE: this is really just for fp_animation_names_mcc + max_anims = get_block_max(block[0].animations) + options = options[:max_anims] try: enum_index = options.index(part2) except ValueError: @@ -338,7 +345,7 @@ def set_animation_index(antr_tag, anim_name, anim_index, break if unit is None: - if len(antr_units) >= antr_units.MAX: + if len(antr_units) >= get_block_max(antr_units.parent): return False antr_units.append() @@ -369,7 +376,7 @@ def set_animation_index(antr_tag, anim_name, anim_index, break if unit_weap is None: - if len(unit_weaps) >= unit_weaps.MAX: + if len(unit_weaps) >= get_block_max(unit_weaps.parent): return False unit_weaps.append() @@ -397,7 +404,7 @@ def set_animation_index(antr_tag, anim_name, anim_index, break if unit_weap_type is None: - if len(unit_weap_types) >= unit_weap_types.MAX: + if len(unit_weap_types) >= get_block_max(unit_weap_types.parent): return False unit_weap_types.append() diff --git a/reclaimer/bitmaps/bitmap_compilation.py b/reclaimer/bitmaps/bitmap_compilation.py index d5f8a998..06969060 100644 --- a/reclaimer/bitmaps/bitmap_compilation.py +++ b/reclaimer/bitmaps/bitmap_compilation.py @@ -13,6 +13,7 @@ from arbytmap.bitmap_io import get_channel_order_by_masks,\ get_channel_swap_mapping, swap_array_items from supyr_struct.defs.bitmaps.dds import dds_def +from reclaimer.util import get_block_max __all__ = ("compile_bitmap_from_dds_files", "add_bitmap_to_bitmap_tag", "parse_dds_file", ) @@ -45,16 +46,20 @@ def add_bitmap_to_bitmap_tag(bitm_tag, width, height, depth, typ, fmt, mip_count, new_pixels, seq_name=""): bitm_data = bitm_tag.data.tagdata sequences = bitm_data.sequences.STEPTREE - bitmaps = bitm_data.bitmaps.STEPTREE - seq_name = seq_name[: 31] + bitmaps = bitm_data.bitmaps.STEPTREE + seq_name = seq_name[: 31] - if len(bitmaps) >= 2048: - raise ValueError("Cannot add more bitmaps(max of 2048 per tag).") + max_bitmaps = get_block_max(bitm_data.bitmaps) + max_sequences = get_block_max(bitm_data.sequences) + max_pixel_bytes = get_block_max(bitm_data.processed_pixel_data) + + if len(bitmaps) >= max_bitmaps: + raise ValueError("Cannot add more bitmaps(max of %s per tag)." % max_bitmaps) bitmaps.append() if not sequences or sequences[-1].sequence_name != seq_name: - if len(sequences) >= 256: - print("Cannot add more sequences(max of 256 per tag).") + if len(sequences) >= max_sequences: + print("Cannot add more sequences(max of %s per tag)." % max_sequences) else: sequences.append() sequences[-1].sequence_name = seq_name @@ -102,6 +107,9 @@ def add_bitmap_to_bitmap_tag(bitm_tag, width, height, depth, typ, fmt, bitm_block.pixels_offset = len(bitm_data.processed_pixel_data.data) + if len(bitm_data.processed_pixel_data.data) + len(new_pixels) >= max_pixel_bytes: + raise ValueError("Cannot add more pixel data(max of %s bytes per tag)." % max_pixel_bytes) + # place the pixels from the dds tag into the bitmap tag bitm_data.processed_pixel_data.data += new_pixels diff --git a/reclaimer/bitmaps/bitmap_decompilation.py b/reclaimer/bitmaps/bitmap_decompilation.py index 2daf996e..33e898d2 100644 --- a/reclaimer/bitmaps/bitmap_decompilation.py +++ b/reclaimer/bitmaps/bitmap_decompilation.py @@ -8,33 +8,25 @@ # try: - import arbytmap - if not hasattr(arbytmap, "FORMAT_P8"): - arbytmap.FORMAT_P8 = "P8" + import arbytmap as ab + if not hasattr(ab, "FORMAT_P8"): + ab.FORMAT_P8 = "P8" """ADD THE P8 FORMAT TO THE BITMAP CONVERTER""" - arbytmap.register_format(format_id=arbytmap.FORMAT_P8, - depths=(8,8,8,8)) + ab.register_format( + format_id=ab.FORMAT_P8, depths=(8,8,8,8) + ) - if not hasattr(arbytmap, "FORMAT_P8_BUMP"): - arbytmap.FORMAT_P8_BUMP = "P8-BUMP" + if not hasattr(ab, "FORMAT_P8_BUMP"): + ab.FORMAT_P8_BUMP = "P8-BUMP" """ADD THE P8 FORMAT TO THE BITMAP CONVERTER""" - arbytmap.register_format(format_id=arbytmap.FORMAT_P8_BUMP, - depths=(8,8,8,8)) - - from arbytmap import Arbytmap, bitmap_io, TYPE_2D, TYPE_3D, TYPE_CUBEMAP,\ - FORMAT_A8, FORMAT_L8, FORMAT_AL8, FORMAT_A8L8,\ - FORMAT_R5G6B5, FORMAT_A1R5G5B5, FORMAT_A4R4G4B4,\ - FORMAT_X8R8G8B8, FORMAT_A8R8G8B8,\ - FORMAT_DXT1, FORMAT_DXT3, FORMAT_DXT5, FORMAT_CTX1, FORMAT_DXN,\ - FORMAT_DXT3Y, FORMAT_DXT3A, FORMAT_DXT3AY,\ - FORMAT_DXT5Y, FORMAT_DXT5A, FORMAT_DXT5AY,\ - FORMAT_P8_BUMP, FORMAT_P8, FORMAT_V8U8, FORMAT_R8G8,\ - FORMAT_R16G16B16F, FORMAT_A16R16G16B16F,\ - FORMAT_R32G32B32F, FORMAT_A32R32G32B32F + ab.register_format( + format_id=ab.FORMAT_P8_BUMP, depths=(8,8,8,8) + ) + except ImportError: - arbytmap = Arbytmap = None + ab = None import zlib @@ -126,13 +118,12 @@ def extract_bitmaps(tagdata, tag_path, **kw): if not ext: ext = "dds" - is_xbox = get_is_xbox_map(engine) is_gen3 = hasattr(tagdata, "zone_assets_normal") - if Arbytmap is None: + if ab is None: # cant extract xbox bitmaps yet return " Arbytmap not loaded. Cannot extract bitmaps." - arby = Arbytmap() + arby = ab.Arbytmap() bitm_i = 0 multi_bitmap = len(tagdata.bitmaps.STEPTREE) > 1 size_calc = get_h3_pixel_bytes_size if is_gen3 else get_pixel_bytes_size @@ -162,8 +153,8 @@ def extract_bitmaps(tagdata, tag_path, **kw): filepath=str(filepath_base.joinpath(filename + "." + ext)) ) tex_info["texture_type"] = { - "texture_2d": TYPE_2D, "texture_3d": TYPE_3D, - "cubemap": TYPE_CUBEMAP}.get(typ, TYPE_2D) + "texture_2d": ab.TYPE_2D, "texture_3d": ab.TYPE_3D, + "cubemap": ab.TYPE_CUBEMAP}.get(typ, ab.TYPE_2D) tex_info["sub_bitmap_count"] = { "texture_2d": 1, "texture_3d": 1, "cubemap": 6, "multipage_2d": d}.get(typ, 1) @@ -175,65 +166,48 @@ def extract_bitmaps(tagdata, tag_path, **kw): if fmt == "p8_bump": tex_info.update( palette=[p8_palette.p8_palette_32bit_packed]*(bitmap.mipmaps + 1), - palette_packed=True, indexing_size=8, format=FORMAT_P8_BUMP) + palette_packed=True, indexing_size=8, format=ab.FORMAT_P8_BUMP) else: tex_info["format"] = { - "a8": FORMAT_A8, "y8": FORMAT_L8, "ay8": FORMAT_AL8, - "a8y8": FORMAT_A8L8, "p8": FORMAT_A8, - "v8u8": FORMAT_V8U8, "g8b8": FORMAT_R8G8, - "x8r8g8b8": FORMAT_A8R8G8B8, "a8r8g8b8": FORMAT_A8R8G8B8, - "r5g6b5": FORMAT_R5G6B5, "a1r5g5b5": FORMAT_A1R5G5B5, - "a4r4g4b4": FORMAT_A4R4G4B4, - "dxt1": FORMAT_DXT1, "dxt3": FORMAT_DXT3, "dxt5": FORMAT_DXT5, - "ctx1": FORMAT_CTX1, "dxn": FORMAT_DXN, "dxt5ay": FORMAT_DXT5AY, - "dxt3a": FORMAT_DXT3A, "dxt3y": FORMAT_DXT3Y, - "dxt5a": FORMAT_DXT5A, "dxt5y": FORMAT_DXT5Y, - "rgbfp16": FORMAT_R16G16B16F, "argbfp32": FORMAT_A32R32G32B32F, - "rgbfp32": FORMAT_R32G32B32F}.get(fmt, None) + "a8": ab.FORMAT_A8, "y8": ab.FORMAT_L8, "ay8": ab.FORMAT_AL8, + "a8y8": ab.FORMAT_A8L8, "p8": ab.FORMAT_A8, + "v8u8": ab.FORMAT_V8U8, "g8b8": ab.FORMAT_R8G8, + "x8r8g8b8": ab.FORMAT_A8R8G8B8, "a8r8g8b8": ab.FORMAT_A8R8G8B8, + "r5g6b5": ab.FORMAT_R5G6B5, "a1r5g5b5": ab.FORMAT_A1R5G5B5, + "a4r4g4b4": ab.FORMAT_A4R4G4B4, + "dxt1": ab.FORMAT_DXT1, "dxt3": ab.FORMAT_DXT3, "dxt5": ab.FORMAT_DXT5, + "ctx1": ab.FORMAT_CTX1, "dxn": ab.FORMAT_DXN, "dxt5ay": ab.FORMAT_DXT5AY, + "dxt3a": ab.FORMAT_DXT3A, "dxt3y": ab.FORMAT_DXT3Y, + "dxt5a": ab.FORMAT_DXT5A, "dxt5y": ab.FORMAT_DXT5Y, + "rgbfp16": ab.FORMAT_R16G16B16F, "argbfp32": ab.FORMAT_A32R32G32B32F, + "rgbfp32": ab.FORMAT_R32G32B32F}.get(fmt, None) arby_fmt = tex_info["format"] if arby_fmt is None: continue - i_max = tex_info["sub_bitmap_count"] if is_xbox else bitmap.mipmaps + 1 - j_max = bitmap.mipmaps + 1 if is_xbox else tex_info['sub_bitmap_count'] off = bitmap.pixels_offset - for i in range(i_max): - if not is_xbox: - mip_size = size_calc(arby_fmt, w, h, d, i, tiled) - - for j in range(j_max): - if is_xbox: - mip_size = size_calc(arby_fmt, w, h, d, j, tiled) - + for m in range(bitmap.mipmaps + 1): + mip_size = size_calc(arby_fmt, w, h, d, m, tiled) + for f in range(tex_info['sub_bitmap_count']): if fmt == "p8_bump": tex_block.append( array('B', pix_data[off: off + (mip_size // 4)])) off += len(tex_block[-1]) else: - off = bitmap_io.bitmap_bytes_to_array( + off = ab.bitmap_io.bitmap_bytes_to_array( pix_data, off, tex_block, arby_fmt, 1, 1, 1, mip_size) - # skip the xbox alignment padding to get to the next texture - if is_xbox and typ == "cubemap": + if typ == "cubemap": off += ((CUBEMAP_PADDING - (off % CUBEMAP_PADDING)) % CUBEMAP_PADDING) - - if is_xbox and typ == "cubemap": - template = tuple(tex_block) - i = 0 - for f in (0, 2, 1, 3, 4, 5): - for m in range(bitmap.mipmaps + 1): - tex_block[m*6 + f] = template[i] - i += 1 - if not tex_block: # nothing to extract continue arby.load_new_texture(texture_block=tex_block, texture_info=tex_info, tile_mode=False, swizzle_mode=False) - arby.save_to_file(keep_alpha=keep_alpha) + arby.save_to_file(keep_alpha=keep_alpha, overwrite=kw.get("overwrite", False)) diff --git a/reclaimer/common_descs.py b/reclaimer/common_descs.py index fdd3091f..a7e6b311 100644 --- a/reclaimer/common_descs.py +++ b/reclaimer/common_descs.py @@ -9,25 +9,32 @@ from copy import copy, deepcopy from math import pi -try: - from mozzarilla.widgets.field_widgets import ReflexiveFrame,\ - HaloRawdataFrame, HaloUInt32ColorPickerFrame, TextFrame,\ - ColorPickerFrame, EntryFrame, HaloScriptSourceFrame,\ - SoundSampleFrame, DynamicArrayFrame, DynamicEnumFrame,\ - HaloScriptTextFrame, HaloBitmapTagFrame, FontCharacterFrame,\ - MeterImageFrame, HaloHudMessageTextFrame -except Exception: - ReflexiveFrame = HaloRawdataFrame = HaloUInt32ColorPickerFrame =\ - TextFrame = ColorPickerFrame = EntryFrame =\ - HaloScriptSourceFrame = SoundSampleFrame =\ - DynamicArrayFrame = DynamicEnumFrame =\ - HaloScriptTextFrame = HaloBitmapTagFrame =\ - FontCharacterFrame = MeterImageFrame =\ - HaloHudMessageTextFrame = None +import os + +from reclaimer.constants import RECLAIMER_NO_GUI + +HaloRawdataFrame = ReflexiveFrame = SoundPlayerFrame = \ + HaloUInt32ColorPickerFrame = ColorPickerFrame = ContainerFrame =\ + EntryFrame = TextFrame = HaloScriptSourceFrame = SoundSampleFrame =\ + HaloScriptTextFrame = HaloBitmapTagFrame = MeterImageFrame =\ + FontCharacterFrame = HaloHudMessageTextFrame =\ + DynamicArrayFrame = DynamicEnumFrame = None + +if not os.environ.get(RECLAIMER_NO_GUI): + try: + from mozzarilla.widgets.field_widgets import \ + HaloRawdataFrame, ReflexiveFrame, SoundPlayerFrame,\ + HaloUInt32ColorPickerFrame, ColorPickerFrame, ContainerFrame,\ + EntryFrame, TextFrame, HaloScriptSourceFrame, SoundSampleFrame,\ + HaloScriptTextFrame, HaloBitmapTagFrame, MeterImageFrame,\ + FontCharacterFrame, HaloHudMessageTextFrame,\ + DynamicArrayFrame, DynamicEnumFrame + except Exception: + print("Unable to import mozzarilla widgets. UI features may not work.") from supyr_struct.defs.common_descs import * from supyr_struct.defs.block_def import BlockDef -from supyr_struct.util import desc_variant +from supyr_struct.util import desc_variant_with_verify as desc_variant from reclaimer.field_types import * from reclaimer.field_type_methods import tag_ref_str_size,\ @@ -35,6 +42,7 @@ from reclaimer.constants import * from reclaimer.enums import * + # before we do anything, we need to inject these constants so any definitions # that are built that use them will have them in their descriptor entries. inject_halo_constants() @@ -57,28 +65,36 @@ def tag_class(*args, **kwargs): ) -def reflexive(name, substruct, max_count=MAX_REFLEXIVE_COUNT, *names, **desc): +def reflexive(name, substruct, max_count=MAX_REFLEXIVE_COUNT, *names, + EXT_MAX=SANE_MAX_REFLEXIVE_COUNT, **kwargs): '''This function serves to macro the creation of a reflexive''' - desc.update( - INCLUDE=reflexive_struct, + EXT_MAX = max(EXT_MAX, max_count) + reflexive_fields = ( + SInt32("size", VISIBLE=VISIBILITY_METADATA, EDITABLE=False, MAX=max_count, EXT_MAX=EXT_MAX), + reflexive_struct[1], + reflexive_struct[2], + ) + kwargs.update( STEPTREE=ReflexiveArray(name + "_array", - SIZE=".size", MAX=max_count, - SUB_STRUCT=substruct, WIDGET=ReflexiveFrame + SIZE=".size", SUB_STRUCT=substruct, WIDGET=ReflexiveFrame, + # NOTE: also adding max here since various things rely on it + # (i.e. compilation/mozz tag block size limit/etc) + MAX=max_count, EXT_MAX=EXT_MAX ), SIZE=12 ) - if DYN_NAME_PATH in desc: - desc[STEPTREE][DYN_NAME_PATH] = desc.pop(DYN_NAME_PATH) + if DYN_NAME_PATH in kwargs: + kwargs[STEPTREE][DYN_NAME_PATH] = kwargs.pop(DYN_NAME_PATH) if names: name_map = {} for i in range(len(names)): e_name = BlockDef.str_to_name(None, names[i]) name_map[e_name] = i - desc[STEPTREE][NAME_MAP] = name_map + kwargs[STEPTREE][NAME_MAP] = name_map - return Reflexive(name, **desc) + return Reflexive(name, *reflexive_fields, **kwargs) def get_raw_reflexive_offsets(desc, two_byte_offs, four_byte_offs, off=0): if INCLUDE in desc: @@ -148,18 +164,17 @@ def rawdata_ref(name, f_type=BytearrayRaw, max_size=None, if max_size is not None: ref_struct_kwargs[MAX] = max_size ref_struct = desc_variant(ref_struct, - ("size", SInt32("size", - GUI_NAME="", SIDETIP="bytes", EDITABLE=False, MAX=max_size, ) - ) + SInt32("size", GUI_NAME="", SIDETIP="bytes", + EDITABLE=False, MAX=max_size + ) ) if widget is not None: kwargs[WIDGET] = widget - return RawdataRef(name, - INCLUDE=ref_struct, + return desc_variant(ref_struct, STEPTREE=f_type("data", GUI_NAME="", SIZE=".size", **kwargs), - **ref_struct_kwargs + NAME=name, **ref_struct_kwargs ) @@ -176,10 +191,10 @@ def rawtext_ref(name, f_type=StrRawLatin1, max_size=None, if max_size is not None: ref_struct_kwargs[MAX] = max_size ref_struct = desc_variant(ref_struct, - ("size", SInt32("size", + SInt32("size", GUI_NAME="", SIDETIP="bytes", EDITABLE=False, - MAX=max_size, VISIBLE=VISIBILITY_METADATA, ) - ) + MAX=max_size, VISIBLE=VISIBILITY_METADATA + ) ) return RawdataRef(name, @@ -221,12 +236,12 @@ def dependency(name='tag_ref', valid_ids=None, **kwargs): elif valid_ids is None: valid_ids = valid_tags - return TagRef(name, + return desc_variant(tag_ref_struct, valid_ids, - INCLUDE=tag_ref_struct, STEPTREE=StrTagRef( - "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=254), - **kwargs + "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=254 + ), + NAME=name, **kwargs ) @@ -239,6 +254,9 @@ def object_type(default=-1): VISIBLE=False, DEFAULT=default ) +def obje_attrs_variant(obje_attrs, typ="", **desc): + obj_index = object_types.index(typ) if typ else 0 + return desc_variant(obje_attrs, object_type(obj_index - 1)) def zone_asset(name, **kwargs): return ZoneAsset(name, INCLUDE=zone_asset_struct, **kwargs) @@ -268,12 +286,11 @@ def string_id(name, index_bit_ct, set_bit_ct, len_bit_ct=None, **kwargs): def blam_header(tagid, version=1): '''This function serves to macro the creation of a tag header''' return desc_variant(tag_header, - ("tag_class", UEnum32("tag_class", + UEnum32("tag_class", GUI_NAME="tag_class", INCLUDE=valid_tags, EDITABLE=False, DEFAULT=tagid - ) - ), - ("version", UInt16("version", DEFAULT=version, EDITABLE=False)), + ), + UInt16("version", DEFAULT=version, EDITABLE=False), ) @@ -590,6 +607,32 @@ def anim_src_func_per_pha_sca_rot_macro(name, **desc): valid_widgets = tag_class('ant!', 'flag', 'glw!', 'mgs2', 'elec') +# Map related descriptors +map_version = UEnum32("version", + ("halo1xbox", 5), + ("halo1pcdemo", 6), + ("halo1pc", 7), + ("halo2", 8), + ("halo3beta", 9), + ("halo3", 11), + ("haloreach", 12), + ("halo1mcc", 13), + ("halo1ce", 609), + ("halo1vap", 134), + ) +gen1_map_type = UEnum16("map_type", + "sp", + "mp", + "ui", + ) +gen2_map_type = UEnum16("map_type", + "sp", + "mp", + "ui", + "shared", + "sharedsp", + ) + # Descriptors tag_header = Struct("blam_header", @@ -655,6 +698,7 @@ def anim_src_func_per_pha_sca_rot_macro(name, **desc): UInt32("raw_pointer", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), # doesnt use magic UInt32("pointer", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), UInt32("id", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), + SIZE=20, ORIENT='h' ) @@ -663,6 +707,7 @@ def anim_src_func_per_pha_sca_rot_macro(name, **desc): SInt32("size", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), UInt32("pointer", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), UInt32("id", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), # 0 in meta it seems + SIZE=12, ) # This is the descriptor used wherever a tag references another tag @@ -671,6 +716,7 @@ def anim_src_func_per_pha_sca_rot_macro(name, **desc): SInt32("path_pointer", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), SInt32("path_length", MAX=MAX_TAG_PATH_LEN, VISIBLE=VISIBILITY_METADATA, EDITABLE=False), UInt32("id", VISIBLE=VISIBILITY_METADATA, EDITABLE=False), + SIZE=16, ORIENT='h' ) @@ -689,10 +735,14 @@ def anim_src_func_per_pha_sca_rot_macro(name, **desc): UInt32("unused", VISIBLE=VISIBILITY_METADATA), ) + extra_layers_block = dependency("extra_layer", valid_shaders) damage_modifiers = QStruct("damage_modifiers", - *(float_zero_to_inf(material_name) for material_name in materials_list) + *(float_zero_to_inf(material_name) for material_name in materials_list), + # NOTE: there's enough allocated for 40 materials. We're assuming + # the rest of the space is all for these damage modifiers + SIZE=4*40 ) # Miscellaneous shared descriptors @@ -804,24 +854,23 @@ def dependency_os(name='tag_ref', valid_ids=None, **kwargs): elif valid_ids is None: valid_ids = valid_tags_os - return TagRef(name, + return desc_variant(tag_ref_struct, valid_ids, - INCLUDE=tag_ref_struct, STEPTREE=StrTagRef( - "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=254), - **kwargs + "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=254 + ), + NAME=name, **kwargs ) def blam_header_os(tagid, version=1): '''This function serves to macro the creation of a tag header''' return desc_variant(tag_header_os, - ("tag_class", UEnum32("tag_class", + UEnum32("tag_class", GUI_NAME="tag_class", INCLUDE=valid_tags_os, EDITABLE=False, DEFAULT=tagid - ) - ), - ("version", UInt16("version", DEFAULT=version, EDITABLE=False)), + ), + UInt16("version", DEFAULT=version, EDITABLE=False), ) diff --git a/reclaimer/constants.py b/reclaimer/constants.py index 2cceb439..c47cc7fd 100644 --- a/reclaimer/constants.py +++ b/reclaimer/constants.py @@ -7,11 +7,46 @@ # See LICENSE for more information. # +import os from struct import unpack from supyr_struct.defs.constants import * from supyr_struct.util import fourcc_to_int -from binilla.constants import * + +# environment variable controls +RECLAIMER_NO_GUI = "RECLAIMER_NO_GUI" # if non-empty, tells reclaimer to +# not load binilla or mozzarilla +# modules wherever possible. +RECLAIMER_NO_ARBY = "RECLAIMER_NO_ARBY" # if non-empty, tells reclaimer to not +# load arbytmap wherever possible. + +# copied from binilla.constants for in case they're not available +EDITABLE = "EDITABLE" +VISIBLE = "VISIBLE" +GUI_NAME = "GUI_NAME" +HIDE_TITLE = "HIDE_TITLE" +ORIENT = "ORIENT" +WIDGET_WIDTH = "WIDGET_WIDTH" +TOOLTIP = "TOOLTIP" +COMMENT = "COMMENT" +SIDETIP = "SIDETIP" +ALLOW_MAX = "ALLOW_MAX" +ALLOW_MIN = "ALLOW_MIN" +UNIT_SCALE = "UNIT_SCALE" +EXT = "EXT" +PORTABLE = "PORTABLE" +WIDGET = "WIDGET" +DYN_NAME_PATH = "DYN_NAME_PATH" +DYN_I = "[DYN_I]" +VISIBILITY_SHOWN = 1 +VISIBILITY_HIDDEN = 0 +VISIBILITY_METADATA = -1 + +if not os.environ.get(RECLAIMER_NO_GUI): + try: + from binilla.constants import * + except ImportError: + pass # some reflexives are so massive that it's significantly faster to treat them # as raw data and just byteswap them using precalculated offsets and sizes @@ -30,6 +65,7 @@ def inject_halo_constants(): PCDEMO_INDEX_MAGIC = 0x4BF10000 +MCC_INDEX_MAGIC = 0x50000000 PC_INDEX_MAGIC = 0x40440000 CE_INDEX_MAGIC = 0x40440000 ANNIVERSARY_INDEX_MAGIC = 0x004B8000 @@ -42,6 +78,7 @@ def inject_halo_constants(): map_build_dates = { "stubbs": "400", "stubbspc": "", + "stubbspc64bit": "", "shadowrun_proto": "01.12.07.0132", "halo1xboxdemo": "", "halo1xbox": "01.10.12.2276", @@ -51,6 +88,7 @@ def inject_halo_constants(): "halo1yelo": "01.00.00.0609", "halo1vap": "01.00.00.0609", "halo1pc": "01.00.00.0564", + "halo1mcc": "01.03.43.0000", "halo2alpha": "02.01.07.4998", "halo2beta": "02.06.28.07902", "halo2epsilon": "02.08.28.09214", @@ -69,12 +107,14 @@ def inject_halo_constants(): map_versions = { "stubbs": 5, "stubbspc": 5, + "stubbspc64bit": 5, "shadowrun_proto": 5, "halo1xboxdemo": 5, "halo1xbox": 5, "halo1pcdemo": 6, "halo1pc": 7, "halo1anni": 7, + "halo1mcc": 13, "halo1ce": 609, "halo1yelo": 609, "halo1vap": 134, @@ -93,19 +133,48 @@ def inject_halo_constants(): #"halo5": ????, } -GEN_1_HALO_ENGINES = ("halo1xboxdemo", "halo1xbox", - "halo1ce", "halo1vap", "halo1yelo", - "halo1pcdemo", "halo1pc", "halo1anni", ) - -GEN_1_ENGINES = GEN_1_HALO_ENGINES + ( - "stubbs", "stubbspc", "shadowrun_proto", ) - -GEN_2_ENGINES = ("halo2alpha", "halo2beta", "halo2epsilon", - "halo2xbox", "halo2vista", ) - -GEN_3_ENGINES = ("halo3", "halo3odst", "halo3beta", - "haloreachbeta", "haloreach", - "halo4", "halo4nettest", "halo5", ) +# NOTE: do NOT change these. they are used in various places +# to determine how to read tags/data from different maps +GEN_1_HALO_XBOX_ENGINES = frozenset(( + "halo1xboxdemo", "halo1xbox", + )) +GEN_1_STUBBS_ENGINES = frozenset(( + "stubbs", "stubbspc", "stubbspc64bit", + )) +GEN_1_SHADOWRUN_ENGINES = frozenset(( + "shadowrun_proto", + )) +GEN_1_HALO_CUSTOM_ENGINES = frozenset(( + "halo1ce", "halo1vap", "halo1yelo", "halo1mcc", + )) +GEN_1_HALO_PC_ENGINES = frozenset(( + "halo1pcdemo", "halo1pc", + )) + +GEN_1_HALO_GBX_ENGINES = GEN_1_HALO_PC_ENGINES.union( + GEN_1_HALO_CUSTOM_ENGINES + ) + +GEN_1_HALO_ENGINES = frozenset(( + "halo1anni", + )).union(GEN_1_HALO_GBX_ENGINES)\ + .union(GEN_1_HALO_XBOX_ENGINES) + +GEN_1_ENGINES = GEN_1_HALO_ENGINES\ + .union(GEN_1_STUBBS_ENGINES)\ + .union(GEN_1_SHADOWRUN_ENGINES) + +GEN_2_ENGINES = frozenset(( + "halo2alpha", "halo2beta", "halo2epsilon", + "halo2xbox", "halo2vista", + )) + +# NOTE: these aren't all gen3, but this basically means "anything halo 3 and newer". +GEN_3_ENGINES = frozenset(( + "halo3", "halo3odst", "halo3beta", + "haloreachbeta", "haloreach", + "halo4", "halo4nettest", "halo5", + )) # magic is actually the virtual address the map is loaded at. Halo 3 and # beyond instead partition the map into sections with a virtual address for @@ -113,12 +182,14 @@ def inject_halo_constants(): map_magics = { "stubbs": STUBBS_INDEX_MAGIC, "stubbspc": PC_INDEX_MAGIC, + "stubbspc64bit": PC_INDEX_MAGIC, "shadowrun_proto": SHADOWRUN_PROTO_INDEX_MAGIC, "halo1xboxdemo": XBOX_INDEX_MAGIC, "halo1xbox": XBOX_INDEX_MAGIC, "halo1pcdemo": PCDEMO_INDEX_MAGIC, "halo1pc": PC_INDEX_MAGIC, "halo1anni": ANNIVERSARY_INDEX_MAGIC, + "halo1mcc": MCC_INDEX_MAGIC, "halo1ce": CE_INDEX_MAGIC, "halo1yelo": CE_INDEX_MAGIC, "halo1vap": CE_INDEX_MAGIC, @@ -182,14 +253,20 @@ def inject_halo_constants(): "UNUSED4", "UNUSED5", "DXT1", "DXT3", "DXT5", "P8-BUMP", "P8", "A32R32G32B32F", "R32G32B32F", "R16G16B16F", - "V8U8", "G8B8", "UNUSED6", "UNUSED7", "UNUSED8", + "V8U8", "G8B8", "UNUSED6", "A16R16G16B16F", "UNUSED8", "UNUSED9", "UNUSED10", "UNUSED11", "UNUSED12", "UNUSED13", "UNUSED14", "DXN", "CTX1", "DXT3A", "DXT3Y", "DXT5A", "DXT5Y", "DXT5AY") +MCC_FORMAT_NAME_MAP = FORMAT_NAME_MAP[:FORMAT_NAME_MAP.index("P8")] + ("BC7", ) +SWIZZLEABLE_FORMATS = ( + "A8", "L8", "AL8", "A8L8", + "R5G6B5", "UNUSED3", "A1R5G5B5", "A4R4G4B4", + "X8R8G8B8", "A8R8G8B8", "P8-BUMP", "P8", + "A32R32G32B32F", "R32G32B32F", "R16G16B16F", + "V8U8", "G8B8" + ) -I_FORMAT_NAME_MAP = {} -for i in range(len(FORMAT_NAME_MAP)): - if i not in I_FORMAT_NAME_MAP: - I_FORMAT_NAME_MAP[FORMAT_NAME_MAP[i]] = i +I_FORMAT_NAME_MAP = {fmt: i for i, fmt in enumerate(FORMAT_NAME_MAP)} +I_MCC_FORMAT_NAME_MAP = {fmt: i for i, fmt in enumerate(MCC_FORMAT_NAME_MAP)} #each bitmap's number of bytes must be a multiple of 512 BITMAP_PADDING = 512 @@ -199,12 +276,38 @@ def inject_halo_constants(): # max value a reflexive count is theoretically allowed to be MAX_REFLEXIVE_COUNT = 2**31-1 -# this number was taken by seeing what the highest indexable reflexive number -# is. +EXT_MAX = "EXT_MAX" # absolute max number of elements a reflexive can +# contain. enforced even when not using safe mode +# this of this as an "extreme" max value. + +# this number was taken by seeing what the highest indexable +# reflexive number is. SANE_MAX_REFLEXIVE_COUNT = 0xFFFE +SINT24_MAX = 0x7FffFF +UINT16_MAX = 0xFFff +SINT16_MAX = 0x7Fff +UINT8_MAX = 0xFF +SINT24_INDEX_MAX = SINT24_MAX + 1 +UINT16_INDEX_MAX = UINT16_MAX + 1 +SINT16_INDEX_MAX = SINT16_MAX + 1 +UINT8_INDEX_MAX = UINT8_MAX + 1 MAX_TAG_PATH_LEN = 254 +# NOTE: do not change these names. they are used with Block.set_to +H1_TRIANGLE_BUFFER_TYPES = ( + "triangle_list", + "triangle_strip" + ) +H1_VERTEX_BUFFER_TYPES = ( + "sbsp_uncomp_material_verts", + "sbsp_comp_material_verts", + "sbsp_uncomp_lightmap_verts", + "sbsp_comp_lightmap_verts", + "model_uncomp_verts", + "model_comp_verts", + ) + # maps tag class four character codes(fccs) in # their string encoding to their int encoding. tag_class_fcc_to_be_int = {} @@ -214,6 +317,10 @@ def inject_halo_constants(): tag_class_be_int_to_fcc = {} tag_class_le_int_to_fcc = {} +LOD_NAMES = ( + "superhigh", "high", "medium", "low", "superlow" + ) + # maps tag class four character codes to the tags file extension tag_class_fcc_to_ext = { 'actr': "actor", diff --git a/reclaimer/enums.py b/reclaimer/enums.py index eda27995..ca7a4be4 100644 --- a/reclaimer/enums.py +++ b/reclaimer/enums.py @@ -400,6 +400,17 @@ "device_name", "scenery_name", # 48 ) +# used in determining which script object types are tag refs +script_object_tag_ref_types = ( + "sound", + "effect", + "damage", + "looping_sound", + "animation_graph", + "actor_variant", + "damage_effect", + "object_definition", + ) # DO NOT MODIFY ANY OF THESE ENUMS! # The exact lettering is important! # INCOMPLETE: Entries with None haven't been determined yet. @@ -461,9 +472,9 @@ "objects_can_see_object", "objects_can_see_flag", "objects_delete_by_definition", - None, + None, # NOTE: might be related to sound_get/set_gain # these next 5 might be shifted. a little - "sound_set_gain", # * + "sound_get_gain", # ** "sound_set_gain", # * same as previous, but extra arg "script_recompile", # * "help", # * @@ -562,100 +573,103 @@ "cheat_spawn_warthog", # * "cheat_all_vehicles", # * "cheat_teleport_to_camera", # * - "cheat_active_camouflage", # * - # no space for this command. Might not actually exist - #"cheat_active_camouflage_local_player", # * + # seems cheat_active_camouflage is a variant of + # cheat_active_camouflage_local_player with the + # local player index set to 0. + #"cheat_active_camouflage", + "cheat_active_camouflage_local_player", # * "cheats_load", # * "ai_free", "ai_free_units", "ai_attach", - "ai_attach", + "ai_attach_free", # ** "ai_detach", # 160 - "ai_place", - "ai_place", + "ai_place", # ??? + "ai_place", # ??? "ai_kill", "ai_kill_silent", "ai_erase", "ai_erase_all", "ai_select", # * - # no space for this command. Might not actually exist + # seems ai_deselect is a variant of ai_select #"ai_deselect", # * "ai_spawn_actor", - "ai_spawn_actor", + "ai_set_respawn", # ** "ai_set_deaf", # 170 "ai_set_blind", - "ai_magically_see_encounter", - "ai_magically_see_encounter", + "ai_magically_see_encounter", # ??? + "ai_magically_see_encounter", # ??? "ai_magically_see_players", "ai_magically_see_unit", # * "ai_timer_expire", "ai_attack", - "ai_attack", + "ai_defend", # ** "ai_retreat", "ai_maneuver", # 180 - "ai_maneuver", - "ai_migrate", - "ai_migrate", + "ai_maneuver_enable", # ** + "ai_migrate", # ?? + "ai_migrate", # ?? "ai_migrate_and_speak", "ai_migrate_by_unit", "ai_allegiance", + # seems ai_allegiance_remove is a variant of ai_allegiance "ai_living_count", - "ai_living_count", + "ai_living_fraction", # ** "ai_strength", - "ai_strength", # 190 - "ai_nonswarm_count", + "ai_strength", # 190 ??? + "ai_swarm_count", # ** "ai_nonswarm_count", "ai_actors", "ai_go_to_vehicle", "ai_going_to_vehicle", "ai_exit_vehicle", "ai_braindead", - "ai_braindead", + "ai_braindead", # ??? "ai_braindead_by_unit", "ai_prefer_target", # 200 "ai_teleport_to_starting_location", "ai_teleport_to_starting_location_if_unsupported", - "ai_teleport_to_starting_location_if_unsupported", + "ai_teleport_to_starting_location_if_unsupported", # ??? "ai_renew", - "ai_try_to_fight", + "ai_try_to_fight_nothing", # ** "ai_try_to_fight", "ai_try_to_fight_player", "ai_command_list", "ai_command_list_by_unit", "ai_command_list_advance", # 210 "ai_command_list_status", - "ai_command_list_status", - "ai_force_active", + "ai_is_attacking", # ** "ai_force_active", + "ai_force_active_by_unit", # ** "ai_set_return_state", # * "ai_set_current_state", # * "ai_playfight", - "ai_playfight", + "ai_playfight", # ??? "ai_status", "ai_vehicle_encounter", # 220 "ai_vehicle_enterable_distance", - "ai_vehicle_enterable_distance", + "ai_vehicle_enterable_team", # ** "ai_vehicle_enterable_actor_type", "ai_vehicle_enterable_actors", # * "ai_vehicle_enterable_disable", "ai_look_at_object", # * "ai_stop_looking", # * "ai_automatic_migration_target", - "ai_automatic_migration_target", + "ai_automatic_migration_target", # ??? "ai_follow_target_disable", # 230 "ai_follow_target_players", "ai_follow_target_ai", "ai_follow_distance", "ai_conversation", - "ai_conversation", + "ai_conversation", # ??? "ai_conversation_stop", "ai_conversation_advance", - "ai_conversation_status", + "ai_conversation_line", # ** "ai_conversation_status", "ai_link_activation", # 240 "ai_berserk", "ai_set_team", - "ai_allow_dormant", + "ai_allow_charge", # ** "ai_allow_dormant", "ai_allegiance_broken", "camera_control", @@ -663,21 +677,21 @@ "camera_set_relative", "camera_set_first_person", "camera_set_dead", # 250 - "camera_set_dead", + "camera_set_dead", # ??? "camera_time", "debug_camera_load", "debug_camera_save", "game_speed", "game_variant", # * "game_difficulty_get", - "game_difficulty_get", + "game_difficulty_get", # ??? "game_difficulty_get_real", "profile_service_clear_timers", # * 260 "profile_service_dump_timers", # * "map_reset", "map_name", - "switch_bsp", - "structure_bsp_index", + "multiplayer_map_name", # ** + "game_difficulty_set", # ** "crash", # * "switch_bsp", "structure_bsp_index", @@ -693,23 +707,26 @@ "debug_tags", # * "profile_reset", # * "profile_dump", # *280 - # no space for these commands. Might not actually exist + # seems profile_activate, profile_deactivate, and + # profile_graph_toggle are variants of profile_dump #"profile_activate", # * #"profile_deactivate", # * #"profile_graph_toggle", # * "debug_pvs", # * "radiosity_start", # * - # no space for these commands. Might not actually exist + # seems radiosity_save and radiosity_debug_point + # are variants of radiosity_start #"radiosity_save", # * #"radiosity_debug_point", # * "ai", "ai_dialogue_triggers", "ai_grenades", - None, None, - "ai", - "ai_dialogue_triggers", - "ai_grenades", # 290 - None, + "ai_lines", # * + "ai_debug_sound_point_set", # * + "ai_debug_vocalize", # * + "ai_debug_teleport_to", # * + "ai_debug_speak", # 290 * + "ai_debug_speak_list", # ** "fade_in", "fade_out", "cinematic_start", @@ -1227,52 +1244,6 @@ def TEST_PRINT_HSC_BUILT_IN_FUNCTIONS(): 'throw-overheated', 'overheating', 'overheating-again', 'enter', 'exit-empty', 'exit-full', 'o-h-exit', 'o-h-s-enter' ) -mcc_actor_types = ( # Used to determine score for killing different actor types. - "brute", - "grunt", - "jackal", - "skirmisher", - "marine", - "spartan", - "drone", - "hunter", - "flood infection", - "flood carrier", - "flood combat", - "flood pure", - "sentinel", - "elite", - "huragok", - "mule", - "turret", - "mongoose", - "warthog", - "scorpion", - "hornet", - "pelican", - "revenant", - "seraph", - "shade", - "watchtower", - "ghost", - "chopper", - "prowler", - "wraith", - "banshee", - "phantom", - "scarab", - "guntower", - "spirit", - "broadsword", - "mammoth", - "lich", - "mantis", - "wasp", - "phaeton", - "watcher", - "knight", - "crawler" - ) unit_damage_animation_names = [] for typ in ("s-ping", "h-ping", "s-kill", "h-kill"): @@ -1292,8 +1263,7 @@ def TEST_PRINT_HSC_BUILT_IN_FUNCTIONS(): #Shared Enumerator options grenade_types_os = ( - 'human_frag', - 'covenant_plasma', + *grenade_types, 'custom_2', 'custom_3', ) @@ -1312,3 +1282,82 @@ def TEST_PRINT_HSC_BUILT_IN_FUNCTIONS(): 'searching', 'fleeing' ) + +unit_animation_names_os = unit_animation_names + ( + "boarding", "mounted" + ) + +# MCC Shared Enumerator options +grenade_types_mcc = grenade_types_os # they're the same +hud_anchors_mcc = ( + "from_parent", + *hud_anchors, + "top_center", + "bottom_center", + "left_center", + "right_center", + ) +actor_types_mcc = ( # Used to determine score for killing different actor types. + "brute", + "grunt", + "jackal", + "skirmisher", + "marine", + "spartan", + "bugger", + "hunter", + "flood_infection", + "flood_carrier", + "flood_combat", + "flood_pure", + "sentinel", + "elite", + "engineer", + "mule", + "turret", + "mongoose", + "warthog", + "scorpion", + "hornet", + "pelican", + "revenant", + "seraph", + "shade", + "watchtower", + "ghost", + "chopper", + "mauler", + "wraith", + "banshee", + "phantom", + "scarab", + "guntower", + "tuning_fork", + "broadsword", + "mammoth", + "lich", + "mantis", + "wasp", + "phaeton", + "bishop", + "knight", + "pawn" + ) +actor_classes_mcc = ( + "infantry", + "leader", + "hero", + "specialist", + "light_vehicle", + "heavy_vehicle", + "giant_vehicle", + "standard_vehicle" + ) +fp_animation_names_mcc = ( + *fp_animation_names, + 'reload-empty-2', 'reload-full-2', + ) +weapon_types_mcc = ( + *weapon_types, + "rocket_launcher" + ) \ No newline at end of file diff --git a/reclaimer/field_type_methods.py b/reclaimer/field_type_methods.py index 498a0b16..30a672fb 100644 --- a/reclaimer/field_type_methods.py +++ b/reclaimer/field_type_methods.py @@ -229,13 +229,21 @@ def reflexive_parser(self, desc, node=None, parent=None, attr_index=None, s_desc = desc.get(STEPTREE) if s_desc: pointer_converter = kwargs.get('map_pointer_converter') - safe_mode = kwargs.get("safe_mode", True) and not desc.get(IGNORE_SAFE_MODE) + safe_mode = kwargs.get("safe_mode", True) and not desc.get(IGNORE_SAFE_MODE) + # get the max value from the size field + arr_len_max = desc[0].get(MAX, 0) + arr_ext_len_max = desc[0].get(EXT_MAX, SANE_MAX_REFLEXIVE_COUNT) + arr_abs_len_max = arr_len_max if (safe_mode and arr_len_max) else arr_ext_len_max if pointer_converter is not None: file_ptr = pointer_converter.v_ptr_to_f_ptr(node[1]) if safe_mode: # make sure the reflexive sizes are within sane bounds. - node[0] = min(node[0], max(SANE_MAX_REFLEXIVE_COUNT, s_desc.get(MAX, 0))) + if node[0] > arr_abs_len_max: + print("Warning: Clipped %s reflexive size from %s to %s" % ( + desc[NAME], node[0], arr_abs_len_max + )) + node[0] = arr_abs_len_max if (file_ptr < 0 or file_ptr + node[0]*s_desc[SUB_STRUCT].get(SIZE, 0) > len(rawdata)): @@ -243,7 +251,7 @@ def reflexive_parser(self, desc, node=None, parent=None, attr_index=None, # (ex: bad hek+ extraction) node[0] = node[1] = 0 - elif node[0] > max(SANE_MAX_REFLEXIVE_COUNT, s_desc.get(MAX, 0)): + elif node[0] > arr_abs_len_max: raise ValueError("Reflexive size is above highest allowed value.") if not node[0]: diff --git a/reclaimer/h2/common_descs.py b/reclaimer/h2/common_descs.py index cb8225d2..ec5ebcbd 100644 --- a/reclaimer/h2/common_descs.py +++ b/reclaimer/h2/common_descs.py @@ -8,19 +8,18 @@ # from copy import copy, deepcopy +import os + +from reclaimer.h2.constants import RECLAIMER_NO_GUI, h2_tag_class_fcc_to_ext + +Halo2BitmapTagFrame = None +if not os.environ.get(RECLAIMER_NO_GUI): + try: + from mozzarilla.widgets.field_widgets import Halo2BitmapTagFrame + except ImportError: + print("Unable to import mozzarilla widgets. UI features may not work.") -try: - from mozzarilla.widgets.field_widgets import ReflexiveFrame, HaloRawdataFrame,\ - TextFrame, ColorPickerFrame, EntryFrame, SoundSampleFrame,\ - DynamicArrayFrame, Halo2BitmapTagFrame -except Exception: - ReflexiveFrame = HaloRawdataFrame = TextFrame = ColorPickerFrame =\ - EntryFrame = SoundSampleFrame = DynamicArrayFrame =\ - Halo2BitmapTagFrame = None from reclaimer.common_descs import * -from reclaimer.h2.constants import STEPTREE, DYN_NAME_PATH, NAME_MAP,\ - COMMENT, TOOLTIP, WIDGET, MAX, MAX_REFLEXIVE_COUNT, VISIBLE, ORIENT,\ - MAX_TAG_PATH_LEN, DEFAULT, h2_tag_class_fcc_to_ext from reclaimer.h2.field_types import * from reclaimer.h2.enums import * @@ -213,6 +212,7 @@ def h2_blam_header(tagid, version=1): "object_definition", "shader", + # should there be a space in these 3? "render model", "structure definition", "lightmap definition", @@ -229,6 +229,21 @@ def h2_blam_header(tagid, version=1): "device_name", "scenery_name", ) +# used in determining which script object types are tag refs +# NOTE: these are a bit of a guess based on the enums above and +# the pattern displayed in the h1 script object types +script_object_tag_ref_types = ( + "effect", + "damage", + "looping_sound", + "animation_graph", + "damage_effect", + "object_definition", + "shader", + "render model", + "structure definition", + "lightmap definition", + ) #Shared Enumerator options old_materials_list = materials_list diff --git a/reclaimer/h3/common_descs.py b/reclaimer/h3/common_descs.py index 8864ba77..88a77c8c 100644 --- a/reclaimer/h3/common_descs.py +++ b/reclaimer/h3/common_descs.py @@ -8,20 +8,19 @@ # from copy import copy, deepcopy +import os + +from reclaimer.h3.constants import RECLAIMER_NO_GUI, h3_tag_class_fcc_to_ext + +Halo3BitmapTagFrame = None +if not os.environ.get(RECLAIMER_NO_GUI): + try: + from mozzarilla.widgets.field_widgets import Halo3BitmapTagFrame + except ImportError: + print("Unable to import mozzarilla widgets. UI features may not work.") -try: - from mozzarilla.widgets.field_widgets import ReflexiveFrame,\ - HaloRawdataFrame, TextFrame, ColorPickerFrame, EntryFrame,\ - SoundSampleFrame, DynamicArrayFrame, Halo3BitmapTagFrame -except Exception: - ReflexiveFrame = HaloRawdataFrame = TextFrame = ColorPickerFrame =\ - EntryFrame = SoundSampleFrame = DynamicArrayFrame = \ - Halo3BitmapTagFrame = None from reclaimer.common_descs import * from reclaimer.h3.field_types import H3TagRef, H3Reflexive, H3RawdataRef -from reclaimer.h3.constants import DYN_NAME_PATH, STEPTREE, NAME_MAP, WIDGET,\ - TOOLTIP, COMMENT, MAX, DEFAULT, VISIBLE, SIZE, MAX_REFLEXIVE_COUNT,\ - h3_tag_class_fcc_to_ext from reclaimer.h3.enums import * from supyr_struct.defs.block_def import BlockDef diff --git a/reclaimer/h3/defs/bitm.py b/reclaimer/h3/defs/bitm.py index 7fe27cfd..b1a70066 100644 --- a/reclaimer/h3/defs/bitm.py +++ b/reclaimer/h3/defs/bitm.py @@ -63,10 +63,9 @@ ("rgbfp32", 20), ("rgbfp16", 21), ("v8u8", 22), - "unused23", + "g8b8", "unused24", - "unused25", # ui\halox\main_menu.bkd.bitmap is set to this - # and is palettized with dimensions 231 x 1 x 1 + "a16r16g16b16f", "unused26", "unused27", "unused28", diff --git a/reclaimer/h3/defs/zone.py b/reclaimer/h3/defs/zone.py index ae4d4e48..95e9a18f 100644 --- a/reclaimer/h3/defs/zone.py +++ b/reclaimer/h3/defs/zone.py @@ -234,6 +234,22 @@ ENDIAN=">", SIZE=532 ) +# this defines JUST enough of the tag for bitmaps to be readable +zone_body_odst_partial = Struct("tagdata", + SEnum16("map_type", *zone_map_type), + Pad(2), + h3_reflexive("resource_types", zone_resource_type, + DYN_NAME_PATH='.name.string'), + h3_reflexive("resource_structure_types", zone_resource_structure_type, + DYN_NAME_PATH='.name.string'), + Pad(60), + h3_reflexive("tag_resources", zone_tag_resource), + Pad(12*15), + Pad(36), + h3_rawdata_ref("fixup_info"), + ENDIAN=">" + ) + def get(): return zone_def @@ -244,3 +260,11 @@ def get(): ext=".%s" % h3_tag_class_fcc_to_ext["zone"], endian=">", tag_cls=H3Tag ) + + +zone_def_odst_partial = TagDef("zone", + h3_blam_header('zone'), + zone_body_odst_partial, + + ext=".%s" % h3_tag_class_fcc_to_ext["zone"], endian=">", tag_cls=H3Tag + ) \ No newline at end of file diff --git a/reclaimer/h3/util.py b/reclaimer/h3/util.py index dfbe01aa..ba5d6612 100644 --- a/reclaimer/h3/util.py +++ b/reclaimer/h3/util.py @@ -9,11 +9,23 @@ from math import ceil, log -from arbytmap.bitmap_io import get_pixel_bytes_size -from reclaimer.h3.constants import HALO3_SHARED_MAP_TYPES +from reclaimer.h3.constants import HALO3_SHARED_MAP_TYPES, RECLAIMER_NO_ARBY from reclaimer.util import * +def get_pixel_bytes_size(*args, **kwargs): + raise NameError( + "Arbytmap is not installed. " + "Cannot determine pixel bytes size" + ) + +if not os.environ.get(RECLAIMER_NO_ARBY): + try: + from arbytmap.bitmap_io import get_pixel_bytes_size + except ImportError: + pass + + def get_virtual_dimension(bitm_fmt, dim, mip_level=0, tiled=False): dim = max(1, dim >> mip_level) if bitm_fmt in ("A8R8G8B8", "X8R8G8B8"): diff --git a/reclaimer/halo_script/defs/hsc.py b/reclaimer/halo_script/defs/hsc.py new file mode 100644 index 00000000..995b45dd --- /dev/null +++ b/reclaimer/halo_script/defs/hsc.py @@ -0,0 +1,345 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.field_types import * +from reclaimer.constants import * +from reclaimer.common_descs import DynamicArrayFrame,\ + script_object_types as h1_script_object_types,\ + desc_variant +# NOTE: using this desc_variant override to ensure verify is defaulted on +from reclaimer.util import float_to_str +from reclaimer.h2.common_descs import \ + script_object_types as h2_script_object_types +from reclaimer.stubbs.common_descs import \ + script_object_types as stubbs_script_object_types +from reclaimer.shadowrun_prototype.common_descs import\ + script_object_types as sr_script_object_types + + +from supyr_struct.defs.block_def import BlockDef +from supyr_struct.defs.tag_def import TagDef + + +HSC_HEADER_LEN = 56 +HSC_STUBBS_64BIT_HEADER_LEN = 64 +HSC_NODE_SIZE = 20 +HSC_SIG_OFFSET = 40 +HSC_IS_PRIMITIVE = 1 << 0 +HSC_IS_SCRIPT_CALL = 1 << 1 +HSC_IS_GLOBAL = 1 << 2 +HSC_IS_GARBAGE_COLLECTABLE = 1 << 3 +HSC_IS_PARAMETER = 1 << 4 +HSC_IS_STRIPPED = 1 << 5 +# yelo +# TODO: do something with supporting these if necessary +HSC_YELO_PARAM_IDX = 1 << 4 +HSC_YELO_LOCAL_IDX = 1 << 5 +HSC_YELO_CONST_IDX = 1 << 6 +HSC_IS_SCRIPT_OR_GLOBAL = HSC_IS_SCRIPT_CALL | HSC_IS_GLOBAL + +script_node_ref = BitStruct("node", + SBitInt("idx", SIZE=16), + SBitInt("salt", SIZE=16), + SIZE=4 + ) +script_node_data_union = Union("data", + CASES={ + "boolean": Struct("bool", UInt8("data")), + "short": Struct("int16", SInt16("data")), + "long": Struct("int32", SInt32("data")), + "real": Struct("real", Float("data")), + "node": script_node_ref, + }, + CASE=(lambda parent=None, **kw: ( + "node" if not parent else { + s: s for s in ("boolean", "short", "long", "real") + }.get(parent.type.enum_name, "node") + )), + SIZE=4 + ) + +# this is a more complete version of the fast_script_node def below +script_node = Struct("script_node", + UInt16("salt"), + Union("index_union", + CASES={ + "constant_type": Struct("value", SInt16("constant_type")), + "function_name": Struct("value", SInt16("function_index")), + "script": Struct("value", SInt16("script_index")), + }, + COMMENT=""" +For most intents and purposes, this value mirrors the 'type' field""" + ), + SEnum16("type"), + Bool16("flags", + "is_primitive", + "is_script_call", + "is_global", + "is_garbage_collectable", + "is_parameter", # MCC only + "is_stripped", # MCC only + ), + desc_variant(script_node_ref, NAME="next_node"), + UInt32("string_offset"), + script_node_data_union, + SIZE=20 + ) + +h1_script_node = desc_variant(script_node, + SEnum16("type", *h1_script_object_types) + ) +h2_script_node = desc_variant(script_node, + SEnum16("type", *h2_script_object_types) + ) +stubbs_script_node = desc_variant(script_node, + SEnum16("type", *stubbs_script_object_types) + ) +stubbs_64bit_script_node = desc_variant(script_node, + SEnum16("type", *stubbs_script_object_types), + desc_variant(script_node_data_union, + SIZE=8, verify=False + ), + SIZE=24, verify=False, + ) +sr_script_node = desc_variant(script_node, + SEnum16("type", *sr_script_object_types) + ) + +script_syntax_data_header = Struct("header", + StrLatin1('name', DEFAULT="script node", SIZE=30), + FlUInt16("total_nodes", DEFAULT=0), # always little endian + SInt16("max_nodes", DEFAULT=19001), # this is 1 more than expected + UInt16("node_size", DEFAULT=20), + UInt8("is_valid", DEFAULT=1), + UInt8("identifier_zero_invalid"), # zero? + UInt16("unused"), + UInt32("sig", DEFAULT="d@t@"), + UInt16("next_node"), # always zero? + UInt16("last_node"), + BytesRaw("next", SIZE=4), # seems to be garbage? + Pointer32("first"), + SIZE=HSC_HEADER_LEN, + ) +stubbs_64bit_script_syntax_data_header = desc_variant(script_syntax_data_header, + UInt16("node_size", DEFAULT=24), + BytesRaw("next", SIZE=8), # seems to be garbage? + Pointer64("first"), # seems to be unset + SIZE=HSC_STUBBS_64BIT_HEADER_LEN, verify=False, + ) + +fast_script_node = QStruct("script_node", + UInt16("salt"), + UInt16("index_union"), + UInt16("type"), + UInt16("flags"), + UInt32("next_node"), + UInt32("string_offset"), + UInt32("data"), + SIZE=HSC_NODE_SIZE + ) + +fast_stubbs_64bit_script_node = desc_variant(fast_script_node, + UInt64("data"), + SIZE=24, verify=False, + ) + +h1_script_syntax_data = desc_variant(script_syntax_data_header, + STEPTREE=WhileArray("nodes", SUB_STRUCT=fast_script_node) + ) +h1_script_syntax_data_os = desc_variant(h1_script_syntax_data, + SInt16("max_nodes", DEFAULT=28501) + ) +h1_script_syntax_data_mcc = desc_variant(h1_script_syntax_data, + FlSInt16("total_nodes", DEFAULT=32766), # only set in mcc? + SInt16("max_nodes", DEFAULT=32767) + ) +stubbs_64bit_script_syntax_data = desc_variant( + stubbs_64bit_script_syntax_data_header, + STEPTREE=WhileArray("nodes", SUB_STRUCT=fast_stubbs_64bit_script_node) + ) + +h1_script_syntax_data_def = BlockDef(h1_script_syntax_data) +h1_script_syntax_data_os_def = BlockDef(h1_script_syntax_data_os) +h1_script_syntax_data_mcc_def = BlockDef(h1_script_syntax_data_mcc) +stubbs_64bit_script_syntax_data_def = BlockDef(stubbs_64bit_script_syntax_data) +# NOTE: update the stubbs and shadowrun script syntax defs if +# differences are ever discovered between them and halo. +# for now, these work perfectly since the max_nodes is +# the same as halo 1, and these defs are fast(no enums) +stubbs_script_syntax_data_def = h1_script_syntax_data_def +sr_script_syntax_data_def = h1_script_syntax_data_def + + +def _generate_script_syntax_tagdef( + prefix, script_node_desc, header_desc=script_syntax_data_header + ): + def get_script_data_endianness(parent=None, rawdata=None, **kw): + try: + rawdata.seek(parent.header_pointer + HSC_SIG_OFFSET) + sig = bytes(rawdata.read(4)) + return ">" if sig == b'd@t@' else "<" + except Exception: + asdf + pass + + def header_pointer(parent=None, rawdata=None, offset=0, root_offset=0, **kw): + if parent is None: + return + + base = offset + root_offset + ptr = -1 + try: + while ptr < base: + ptr = rawdata.find(b"script node\x00", base) + if ptr < 0: + break + + rawdata.seek(ptr + HSC_SIG_OFFSET) + if bytes(rawdata.read(4)) in (b'd@t@', b'@t@d'): + break + + base = ptr + 12 # 12 is size of "script node" string + ptr = -1 + + except Exception: + ptr = parent.base_pointer + + parent.header_pointer = ptr + + def strings_pointer(parent=None, header_size=header_desc["SIZE"], **kw): + if parent is None: + return + header = parent.header + parent.strings_pointer = ( + parent.parent.header_pointer + header_size + + header.max_nodes * header.node_size + ) + + def get_script_string_data_size(parent=None, **kw): + try: + return parent.header.max_nodes * parent.header.node_size + except Exception: + return 0 + + def get_node_string(parent=None, rawdata=None, **kw): + if parent is None: + return + + node = parent.parent + node_type = node.type.enum_name + flags = node.flags.data + data = node.data + is_global = flags & HSC_IS_GLOBAL + is_script = flags & HSC_IS_SCRIPT_CALL + is_primitive = flags & HSC_IS_PRIMITIVE + is_script_or_global = is_global or is_script + is_list = not is_primitive + is_unused = node.salt == 0 or node_type in("special_form", "unparsed") + prefixes = [] + string = None + + if node.next_node.salt != -1 and node.next_node.idx != -1: + prefixes.append(f"NEXT={node.next_node.idx}") + + if is_unused: + prefixes = ["UNUSED", *prefixes] + elif is_list: + prefixes = [f"FIRST={data.node.idx}", *prefixes] + else: + if not is_global and node_type in ("boolean", "real", "short", "long"): + string = ( + bool(data.boolean.data&1) if node_type == "boolean" else + data.real.data if node_type == "real" else + data.short.data if node_type == "short" else + data.long.data if node_type == "long" else + None + ) + else: + start = node.parent.parent.strings_pointer + node.string_offset + end = rawdata.find(b"\x00", start) + try: + rawdata.seek(start-1) + if rawdata.read(1) == b"\x00" or node.string_offset == 0: + string = rawdata.read(max(0, end-start)).decode(encoding="latin-1") + except Exception: + pass + + if string is None: + node_type, string = "undef", "" + + if is_global or is_script: + prefixes = ["GLOBAL" if is_global else "SCRIPT", *prefixes] + + if not is_unused and not is_list: + prefixes.append(f"TYPE={node_type}") + + parent.string = string + parent.description = '<%s> %s' % ( + " ".join(prefixes), + "" if string is None else f'"{string}"' + ) + + script_node_desc = desc_variant(script_node_desc, + STEPTREE=Container("computed", + Computed("description", COMPUTE_READ=get_node_string, WIDGET_WIDTH=120), + Computed("string", WIDGET_WIDTH=120), + ), + ) + script_data = Container("script_data", + desc_variant(header_desc, POINTER="..header_pointer"), + Computed("strings_pointer", COMPUTE_READ=strings_pointer, WIDGET_WIDTH=20), + Array("nodes", + SUB_STRUCT=script_node_desc, SIZE=".header.last_node", + DYN_NAME_PATH=".computed.description", WIDGET=DynamicArrayFrame + ), + ) + + return ( + TagDef('%s_%s_scripts' % (prefix, extension), + Computed("header_pointer", COMPUTE_READ=header_pointer, WIDGET_WIDTH=20), + Switch("script_data", + CASES={ + ">": desc_variant(script_data, ENDIAN=">"), + "<": desc_variant(script_data, ENDIAN="<"), + }, + CASE=get_script_data_endianness, DEFAULT=script_data + ), + ext="." + extension + ) + for extension in ("scenario", "map") + ) + +# for loading in binilla for debugging script data issues +def get(): + return ( + *h1_script_syntax_data_tagdefs, + *h2_script_syntax_data_tagdefs, + *stubbs_script_syntax_data_tagdefs, + *stubbs_64bit_script_syntax_data_tagdefs, + *sr_script_syntax_data_tagdefs + ) + +h1_script_syntax_data_tagdefs = _generate_script_syntax_tagdef( + "h1", h1_script_node + ) +h2_script_syntax_data_tagdefs = _generate_script_syntax_tagdef( + "h2", h2_script_node + ) +stubbs_script_syntax_data_tagdefs = _generate_script_syntax_tagdef( + "stubbs", stubbs_script_node + ) +stubbs_64bit_script_syntax_data_tagdefs = _generate_script_syntax_tagdef( + "stubbs_64bit", stubbs_64bit_script_node, + stubbs_64bit_script_syntax_data_header + ) +sr_script_syntax_data_tagdefs = _generate_script_syntax_tagdef( + "sr", sr_script_node + ) + +del _generate_script_syntax_tagdef # just for quick generation \ No newline at end of file diff --git a/reclaimer/halo_script/hsc.py b/reclaimer/halo_script/hsc.py index c2872e7b..0183051d 100644 --- a/reclaimer/halo_script/hsc.py +++ b/reclaimer/halo_script/hsc.py @@ -10,16 +10,25 @@ from struct import pack, unpack from types import MethodType -from reclaimer.field_types import * -from reclaimer.constants import * -from reclaimer.common_descs import ascii_str32,\ +from reclaimer.halo_script.defs.hsc import * + +from reclaimer.common_descs import \ script_types as h1_script_types,\ - script_object_types as h1_script_object_types + script_object_types as h1_script_object_types,\ + script_object_tag_ref_types as h1_script_object_tag_ref_types from reclaimer.util import float_to_str from reclaimer.h2.common_descs import script_types as h2_script_types,\ - script_object_types as h2_script_object_types + script_object_types as h2_script_object_types,\ + script_object_tag_ref_types as h2_script_object_tag_ref_types +from reclaimer.stubbs.common_descs import \ + script_types as stubbs_script_types,\ + script_object_types as stubbs_script_object_types,\ + script_object_tag_ref_types as stubbs_script_object_tag_ref_types +from reclaimer.shadowrun_prototype.common_descs import \ + script_types as sr_script_types,\ + script_object_types as sr_script_object_types,\ + script_object_tag_ref_types as sr_script_object_tag_ref_types -from supyr_struct.defs.block_def import BlockDef from supyr_struct.field_types import FieldType try: @@ -27,6 +36,7 @@ except Exception: _script_built_in_functions_test = None + # in a list that begins with the keyed expression, this # is the number of nodes required before we will indent them. INDENT_LIST_MIN = { @@ -47,90 +57,23 @@ "ai_debug_communication_focus": 1, } -HSC_IS_PRIMITIVE = 1 << 0 -HSC_IS_SCRIPT_CALL = 1 << 1 -HSC_IS_GLOBAL = 1 << 2 -HSC_IS_GARBAGE_COLLECTABLE = 1 << 3 -HSC_IS_SCRIPT_OR_GLOBAL = HSC_IS_SCRIPT_CALL | HSC_IS_GLOBAL - +_script_objects = ("object", "unit", "vehicle", "weapon", "device", "scenery") SCRIPT_OBJECT_TYPES_TO_SCENARIO_REFLEXIVES = dict(( - (10, "scripts"), (11, "trigger_volumes"), (12, "cutscene_flags"), - (13, "cutscene_camera_points"), (14, "cutscene_titles"), - (15, "recorded_animations"), (16, "device_groups"), - (17, "encounters"), (18, "command_lists"), - (19, "player_starting_profiles"), (20, "ai_conversations"), - ) + tuple((i, "object_names") for i in range(37, 49))) - - -h1_script_type = SEnum16("type", *h1_script_types) -h1_return_type = SEnum16("return_type", *h1_script_object_types) - - -# this is a more complete version of the fast_script_node def below -script_node = Struct("script_node", - UInt16("salt"), - Union("index_union", - CASES={ - "constant_type": Struct("constant_type", SInt16("value")), - "function_index": Struct("function_index", SInt16("value")), - "script_index": Struct("script_index", SInt16("value")), - }, - COMMENT=""" -For most intents and purposes, this value mirrors the 'type' field""" - ), - SEnum16("type", *h1_script_object_types), - Bool16("flags", - "is_primitive", - "is_script_call", - "is_global", - "is_garbage_collectable", - ), - UInt32("next_node"), - UInt32("string_offset"), - Union("data", - CASES={ - "bool": Struct("bool", UInt8("data")), - "int16": Struct("int16", SInt16("data")), - "int32": Struct("int32", SInt32("data")), - "real": Struct("real", Float("data")), - "node": Struct("node", UInt32("data")), - } - ), - SIZE=20 - ) - -fast_script_node = QStruct("script_node", - UInt16("salt"), - UInt16("index_union"), - UInt16("type"), - UInt16("flags"), - UInt32("next_node"), - UInt32("string_offset"), - UInt32("data"), - SIZE=20 - ) - -h1_script_syntax_data = Struct("script syntax data header", - ascii_str32('name', DEFAULT="script node"), - UInt16("max_nodes", DEFAULT=19001), # this is 1 more than expected - UInt16("node_size", DEFAULT=20), - UInt8("is_valid", DEFAULT=1), - UInt8("identifier_zero_invalid"), # zero? - UInt16("unused"), - UInt32("sig", DEFAULT="d@t@"), - UInt16("next_node"), # zero? - UInt16("last_node"), - BytesRaw("next", SIZE=4), - Pointer32("first"), - SIZE=56, - STEPTREE=WhileArray("nodes", SUB_STRUCT=fast_script_node) - ) - -h1_script_syntax_data_os = dict(h1_script_syntax_data) -h1_script_syntax_data_os[1] = UInt16("max_nodes", DEFAULT=28501) - -h1_script_syntax_data_def = BlockDef(h1_script_syntax_data) -h1_script_syntax_data_os_def = BlockDef(h1_script_syntax_data_os) + ("script", "scripts"), + ("trigger_volume", "trigger_volumes"), + ("cutscene_flag", "cutscene_flags"), + ("cutscene_camera_point", "cutscene_camera_points"), + ("cutscene_title", "cutscene_titles"), + ("cutscene_recording", "recorded_animations"), + ("device_group", "device_groups"), + ("ai", "encounters"), + ("ai_command_list", "command_lists"), + ("starting_profile", "player_starting_profiles"), + ("conversation", "ai_conversations"), + *((typ, "object_names") for typ in _script_objects), + *(("%s_name" % typ, "object_names") for typ in _script_objects), + )) +del _script_objects def cast_uint32_to_float(uint32, packer=MethodType(pack, "= len(nodes): return "", False, 0 @@ -292,56 +269,64 @@ def decompile_node_bytecode(node_index, nodes, script_blocks, string_data, newl = False node = nodes[node_index] node_type = node.type - union_i = node.index_union node_str = "" + if node.flags == HSC_IS_GARBAGE_COLLECTABLE: start_node = get_first_significant_node(node, nodes, string_data) child_node, salt = start_node.data & 0xFFff, start_node.data >> 16 if salt != 0: node_str, newl, ct = decompile_node_bytecode( - child_node, nodes, script_blocks, string_data, - object_types, indent + 1, indent_size, **kwargs) + child_node, nodes, string_data, + object_types, script_types, indent + 1, indent_size, + indent_char, return_char, bool_as_int, **kwargs) if ct > 1 or newl or (node_str[:1] != "(" and node_str[-1:] != ")"): # only add a start parenthese so the end can be added # on later when we decide how to pad the list elements - node_str = "(%s" % node_str[1:] + node_str = "(" + node_str else: # reparse the node at a lesser indent node_str, newl, ct = decompile_node_bytecode( - child_node, nodes, script_blocks, string_data, - object_types, indent, indent_size, **kwargs) + child_node, nodes, string_data, + object_types, script_types, indent, indent_size, + indent_char, return_char, bool_as_int, **kwargs) + node_str = (" " + node_str) if node_str else "" elif node.flags & HSC_IS_GLOBAL: node_str = get_hsc_node_string(string_data, node, hsc_node_strings_by_type) - if "global_uses" in kwargs: + if "global_uses" in kwargs and node_str: kwargs["global_uses"].add(node_str) - elif (node.flags & HSC_IS_SCRIPT_CALL and - union_i >= 0 and union_i < len(script_blocks)): + elif node.flags & HSC_IS_SCRIPT_CALL: # is_script_call is set - block = script_blocks[union_i] - node_str = "(%s" % block.name - if "static_calls" in kwargs: - kwargs["static_calls"].add(block.name) + args_node, salt = node.data & 0xFFff, node.data >> 16 + node_str = "(" + script_call_node_str, newl, ct = decompile_node_bytecode( + args_node, nodes, string_data, + object_types, script_types, indent + 1, indent_size, + indent_char, return_char, bool_as_int, **kwargs) + + node_str += script_call_node_str elif node.flags & HSC_IS_PRIMITIVE: + # TODO: remove value hardcoding in these node_type checks # is_primitive is set if node_type in (2, 10): # function/script name node_str = get_hsc_node_string(string_data, node, hsc_node_strings_by_type) - if node_type == 10 and "static_calls" in kwargs: + if node_type == 10 and "static_calls" in kwargs and node_str: kwargs["static_calls"].add(node_str) - elif node_type in (3, 4): - # passthrough/void type + elif node_type in range(5): + # special form/unparsed/passthrough/void type pass elif node_type == 5: # bool - node_str = "true" if node.data&1 else "false" + val = node.data&1 + node_str = str(val) if bool_as_int else ["false", "true"][val] elif node_type == 6: # float node_str = float_to_str(cast_uint32_to_float(node.data)) @@ -358,7 +343,7 @@ def decompile_node_bytecode(node_index, nodes, script_blocks, string_data, node_strs.append((node_str, newl)) node_index, salt = node.next_node & 0xFFff, node.next_node >> 16 - if salt == 0: + if salt == -1 and node_index == -1: break i += 1 @@ -368,7 +353,7 @@ def decompile_node_bytecode(node_index, nodes, script_blocks, string_data, return "", False, 0 string = "" - indent_str = " " * indent_size * indent + indent_str = indent_char * indent_size * indent returned = False i = 0 first_node = node_strs[0] @@ -390,67 +375,167 @@ def decompile_node_bytecode(node_index, nodes, script_blocks, string_data, if returned: node_str = indent_str + node_str else: - node_str = "\n" + indent_str + node_str + node_str = return_char + indent_str + node_str has_newlines = True elif i > 1: + # ensure there's a space to separate parameters node_str = " " + node_str if cap_end: if local_newlines: - node_str += "\n" + indent_str + node_str += return_char + indent_str node_str += ")" - returned = node_str[-1] == "\n" + returned = node_str[-1] == return_char string += node_str - if string: - # always add a space to the beginning. - # it can be stripped off by whatever we return to - string = " " + string - return string, has_newlines, i +def get_script_types(engine="halo1"): + '''Returns the script types and script object types for this engine.''' + return ( + (h1_script_types, h1_script_object_types) if "yelo" in engine else + (h1_script_types, h1_script_object_types) if "halo1" in engine else + (h2_script_types, h2_script_object_types) if "halo2" in engine else + (sr_script_types, sr_script_object_types) if "shadowrun" in engine else + (stubbs_script_types, stubbs_script_object_types) if "stubbs" in engine else + ((), ()) + ) + + +def get_script_tag_ref_type_names(engine="halo1"): + ''' + Returns a list containing the enum name each script node tag + ref type for this engine. + ''' + # these are the names of the script object types that are tag references + return ( + h1_script_object_tag_ref_types if "yelo" in engine else + h1_script_object_tag_ref_types if "halo1" in engine else + h2_script_object_tag_ref_types if "halo2" in engine else + sr_script_object_tag_ref_types if "shadowrun" in engine else + stubbs_script_object_tag_ref_types if "stubbs" in engine else + () + ) + + +def get_script_tag_ref_types(engine="halo1"): + ''' + Returns a list containing the enum value of each script node tag + ref type for this engine. + ''' + # these are the names of the script object types that are tag references + tag_ref_script_types = get_script_tag_ref_type_names(engine) + + _, script_object_types = get_script_types(engine) + return [ + script_object_types.index(typ) + for typ in tag_ref_script_types + ] + + +def get_script_syntax_node_tag_refs(syntax_data, engine="halo1"): + '''Returns a list of all script nodes that are tag references.''' + tag_ref_type_enums = set(get_script_tag_ref_types(engine)) + tag_ref_nodes = [] + + # null all references to tags + for node in syntax_data.nodes: + if (node.flags & HSC_IS_SCRIPT_OR_GLOBAL or + node.type not in tag_ref_type_enums): + # not a tag index ref + continue + + tag_ref_nodes.append(node) + + return tag_ref_nodes + + +def clean_script_syntax_nodes(syntax_data, engine="halo1"): + ''' + Scans through script nodes and nulls tag references. + This is necessary for script syntax data in tag form. + ''' + # null all references to tags + for node in get_script_syntax_node_tag_refs(syntax_data, engine): + # null the reference + node.data = 0xFFffFFff + + def hsc_bytecode_to_string(syntax_data, string_data, block_index, script_blocks, global_blocks, block_type, - engine="halo1", indent_size=4, **kwargs): - if block_type not in ("script", "global"): + engine="halo1", indent_size=4, minify=False, + indent_char=" ", return_char="\n", **kwargs): + is_global = (block_type == "global") + is_script = (block_type == "script") + script_types, object_types = get_script_types(engine) + if not((is_global or is_script) and script_types and object_types): return "" - if ("halo1" in engine or "yelo" in engine or - "stubbs" in engine or "shadowrun" in engine): - script_types = h1_script_types - object_types = h1_script_object_types - elif "halo2" in engine: - script_types = h2_script_types - object_types = h2_script_object_types - else: - return "" + # figure out which reflexive and type enums to use + blocks = script_blocks if is_script else global_blocks + typ_names = script_types if is_script else object_types - if block_type == "global": - block = global_blocks[block_index] - script_type = "" - node_index = block.initialization_expression_index & 0xFFFF - main_type = object_types[block.type.data] - else: - block = script_blocks[block_index] - script_type = script_types[block.type.data] - node_index = block.root_expression_index & 0xFFFF - main_type = object_types[block.return_type.data] - if script_type in ("dormant", "startup", "continuous"): - # these types wont compile if a return type is specified - main_type = "" - else: - script_type += " " - - if "INVALID" in main_type: + block = blocks[block_index] + typ = block.type.data + + # invalid script/global type + if typ not in range(len(typ_names)): return "" - indent_str = " " * indent_size - head = "%s %s%s %s" % (block_type, script_type, main_type, block.name) + if minify: + indent_size = 0 + indent_char = "" + return_char = "" + + # get the index of the node in the nodes array + node_index = ( + block.root_expression_index if is_script else + block.initialization_expression_index + ) & 0xFFff + + # figure out the type of the node + node_type = typ_names[typ] + + # scripts also have a return type(except the 3 specified below) + if is_script and node_type not in ("dormant", "startup", "continuous"): + return_typ = block.return_type.data + if return_typ not in range(len(object_types)): + # invalid return type + return "" + + node_type += " " + object_types[return_typ] + + # generate the suffix of the header, which includes the function/global + # name, and any parameters the function accepts(params are MCC only) + suffix = block.name + if hasattr(block, "parameters") and block.parameters.STEPTREE: + for param in block.parameters.STEPTREE: + return_typ = param.return_type.data + if return_typ not in range(len(object_types)): + # invalid return type + print("Invalid return type '%s' in script '%s'" % + (param.name, block.name) + ) + return "" + suffix += "%s(%s %s)" % ( + indent_char, object_types[return_typ], param.name + ) + suffix = "(%s)" % suffix + + head = "%s %s %s" % (block_type, node_type, suffix) body, _, __ = decompile_node_bytecode( - node_index, syntax_data.nodes, script_blocks, string_data, - object_types, 1, indent_size, **kwargs) - if block_type == "global": - return "(%s%s)" % (head, body) - return "(%s\n%s%s\n)" % (head, indent_str, body[1:]) + node_index, syntax_data.nodes, string_data, + object_types=object_types, script_types=script_types, + indent_size=indent_size, indent_char=indent_char, + return_char=return_char, **kwargs + ) + + if is_script: + body = "".join((return_char, " " * indent_size, body, return_char)) + else: + # add a space to separate global definition from its value + body = " " + body + + return "(%s%s)" % (head, body) diff --git a/reclaimer/halo_script/hsc_decompilation.py b/reclaimer/halo_script/hsc_decompilation.py index cf3e8bd5..cd06bd98 100644 --- a/reclaimer/halo_script/hsc_decompilation.py +++ b/reclaimer/halo_script/hsc_decompilation.py @@ -15,149 +15,180 @@ __all__ = ("extract_h1_scripts", ) - -MAX_SCRIPT_SOURCE_SIZE = 1 << 18 - - -def extract_h1_scripts(tagdata, tag_path, **kw): - dirpath = Path(kw.get("out_dir", "")).joinpath( - Path(tag_path).parent, "scripts") - - overwrite = kw.get('overwrite', True) - hsc_node_strings_by_type = kw.get("hsc_node_strings_by_type", ()) - - dirpath.mkdir(exist_ok=True, parents=True) - - engine = kw.get('engine') - if not engine and 'halo_map' in kw: - engine = kw['halo_map'].engine +def _lf_to_crlf(string): # laziness + return string.replace("\n", "\r\n") + + +SCRIPT_HEADER = _lf_to_crlf("; Extracted with Reclaimer\n\n") +MAX_SCRIPT_SOURCE_SIZE = 1 << 18 +MCC_MAX_SCRIPT_SOURCE_SIZE = 1 << 20 + + +def generate_scenario_references_comment(tagdata): + ''' + Generate a string which lists out all scenario references + that may be useful to have when editing extracted scripts. + ''' + comments = "\n; scenario names(each sorted alphabetically)\n" + comments += "\n; object names:\n" + for name in sorted(set(obj.name for obj in + tagdata.object_names.STEPTREE)): + comments += "; %s\n" % name + + comments += "\n; trigger volumes:\n" + for name in sorted(set(tv.name for tv in + tagdata.trigger_volumes.STEPTREE)): + comments += "; %s\n" % name + + comments += "\n; device groups:\n" + for name in sorted(set(dg.name for dg in + tagdata.device_groups.STEPTREE)): + comments += "; %s\n" % name + + comments += "\n; globals:\n" + return _lf_to_crlf(comments) + + +def extract_scripts( + tagdata, engine=None, halo_map=None, add_comments=True, + minify=False, max_script_size=None, default_engine=None, **kwargs + # NOTE: accepting arbitrary kwargs cause caller wont know what args we use + ): + # this will hold the decompiled script source file strings + script_sources, global_sources = [], [] + + engine = engine or getattr(halo_map, "engine", default_engine) + if max_script_size is None: + max_script_size = ( + MCC_MAX_SCRIPT_SOURCE_SIZE if engine == "halo1mcc" else + MAX_SCRIPT_SOURCE_SIZE + ) + + # unknown currently if original halo allows this, but mcc does + kwargs.setdefault("bool_as_int", engine == "halo1mcc") syntax_data = get_hsc_data_block(tagdata.script_syntax_data.data) string_data = tagdata.script_string_data.data.decode("latin-1") if not syntax_data or not string_data: - return "No script data to extract." + return script_sources, global_sources + + # generate comments string(unless we want sources as small as possible) + comments = "" if minify or not add_comments else ( + generate_scenario_references_comment(tagdata) + ) already_sorted = set() for typ, arr in (("global", tagdata.globals.STEPTREE), ("script", tagdata.scripts.STEPTREE)): - filename_base = "%ss" % typ - - header = "; Extracted with Reclaimer\n\n" - comments = "" - src_file_i = 0 global_uses = {} static_calls = {} - if typ == "global": - sort_by = global_uses - try: - comments += "; scenario names(each sorted alphabetically)\n" - comments += "\n; object names:\n" - for name in sorted(set(obj.name for obj in - tagdata.object_names.STEPTREE)): - comments += "; %s\n" % name - - comments += "\n; trigger volumes:\n" - for name in sorted(set(tv.name for tv in - tagdata.trigger_volumes.STEPTREE)): - comments += "; %s\n" % name - - comments += "\n; device groups:\n" - for name in sorted(set(dg.name for dg in - tagdata.device_groups.STEPTREE)): - comments += "; %s\n" % name - - except Exception: - pass - else: - sort_by = static_calls - - try: - sources = {} - for i in range(len(arr)): - # assemble source code for each function/global - name = arr[i].name - global_uses[name] = set() - static_calls[name] = set() - sources[name] = hsc_bytecode_to_string( - syntax_data, string_data, i, tagdata.scripts.STEPTREE, - tagdata.globals.STEPTREE, typ, engine, - global_uses=global_uses[name], - hsc_node_strings_by_type=hsc_node_strings_by_type, - static_calls=static_calls[name]) - - sorted_sources = [] - need_to_sort = sources - while need_to_sort: - # sort the functions/globals so dependencies come first - next_need_to_sort = {} - for name in sorted(need_to_sort.keys()): - source = need_to_sort[name] - if sort_by[name].issubset(already_sorted): - sorted_sources.append(source) - already_sorted.add(name) - else: - next_need_to_sort[name] = source - - if need_to_sort.keys() == next_need_to_sort.keys(): - print("Could not sort these %ss so dependencies come first:" % typ) - for name in need_to_sort.keys(): - print("\t%s" % name) - print("\t Requires: ", ", ".join(sorted(sort_by[name]))) - print() - sorted_sources.append(need_to_sort[name]) - break - need_to_sort = next_need_to_sort - - merged_sources = [] - merged_src = "" - merged_src_len = 0 - # figure out how much data we can fit in the source file - max_size = MAX_SCRIPT_SOURCE_SIZE - len(header) - len(comments) - - for src in sorted_sources: - if not src: - continue - - src += "\n\n\n" - # \n will be translated to \r\n, so the actual serialized string - # length will be incremented by the number of newline characters - src_len = len(src) + src.count("\n") - - # concatenate sources until they are too large to be compiled - if merged_src_len + src_len >= max_size: - merged_sources.append(merged_src) - merged_src = "" - merged_src_len = 0 - - merged_src += src - merged_src_len += src_len - - if merged_src: - merged_sources.append(merged_src) - - i = 0 - for out_data in merged_sources: - # write sources to hsc files - if len(merged_sources) > 1: - filename = "%s_%s.hsc" % (filename_base, i) + sort_by = global_uses if (typ == "global") else static_calls + sources = global_sources if (typ == "global") else script_sources + + decompiled_scripts = {} + for i in range(len(arr)): + # assemble source code for each function/global + name = arr[i].name + global_uses[name] = set() + static_calls[name] = set() + + decompiled_scripts[name] = hsc_bytecode_to_string( + syntax_data, string_data, i, + tagdata.scripts.STEPTREE, + tagdata.globals.STEPTREE, typ, engine, + global_uses=global_uses[name], + static_calls=static_calls[name], minify=minify, **kwargs + ) + + sorted_sources = [] + need_to_sort = decompiled_scripts + while need_to_sort: + # sort the functions/globals so dependencies come first + next_need_to_sort = {} + for name in sorted(need_to_sort.keys()): + source = need_to_sort[name] + sort_remainder = sort_by[name].difference(already_sorted) + if sort_remainder: + next_need_to_sort[name] = source + sort_by[name] = sort_remainder else: - filename = "%s.hsc" % filename_base - - filepath = dirpath.joinpath(filename) - if not overwrite and filepath.is_file(): - continue - - # apparently the scripts use latin1 encoding, who knew.... - with filepath.open("w", encoding='latin1', newline="\r\n") as f: - f.write(header) - f.write(out_data) - f.write(comments) - - i += 1 - except Exception: - return format_exc() + sorted_sources.append(source) + already_sorted.add(name) + + if need_to_sort.keys() == next_need_to_sort.keys(): + print("Could not sort these %ss so dependencies come first:" % typ) + for name in need_to_sort.keys(): + print("\t%s" % name) + print("\t Requires: ", ", ".join(sorted(sort_by[name]))) + print() + sorted_sources.append(need_to_sort[name]) + break + need_to_sort = next_need_to_sort + + # header to put before each extracted source file + header = "" if minify else SCRIPT_HEADER + ( + comments if (typ == "global") else "" + ) + + # concatenate script sources until they are too large to be compiled + concat_src = "" + for i, src in enumerate(sorted_sources): + if src: + # translate \n to \r\n since that's what haloscripts uses. + src = _lf_to_crlf(src + ("\n" if minify else "\n\n")) + + # we're gonna pass the limit on script size if we concatenate + # this script source, so we need to append it and start anew. + if len(concat_src) + len(src) >= max_script_size: + sources.append(header + concat_src) + concat_src = src + else: + concat_src += src + + # ensure the last script source is appended if it's not empty + if i+1 == len(sorted_sources): + sources.append(header + concat_src) # TEMPORARY CODE #from reclaimer.enums import TEST_PRINT_HSC_BUILT_IN_FUNCTIONS #TEST_PRINT_HSC_BUILT_IN_FUNCTIONS() # TEMPORARY CODE + + return script_sources, global_sources + + +def extract_scripts_to_file(tagdata, tag_path, **kwargs): + out_dir = kwargs.pop("out_dir", "") + overwrite = kwargs.pop('overwrite', True) + script_sources, global_sources = extract_scripts(tagdata, **kwargs) + + if not script_sources or not global_sources: + return "No scripts to extract." + + dirpath = Path(out_dir).joinpath(Path(tag_path).parent, "scripts") + dirpath.mkdir(exist_ok=True, parents=True) + for typ, sources in ( + ("scripts", script_sources), + ("globals", global_sources), + ): + + for i in range(len(sources)): + # write sources to hsc files + filename = "%s_%s.hsc" % (typ, i) + filepath = dirpath.joinpath(filename) + if not overwrite and filepath.is_file(): + continue + + # apparently the scripts use latin1 encoding, who knew.... + with filepath.open("w", encoding='latin1', newline="") as f: + f.write(sources[i]) + + +def extract_h1_scripts(tagdata, tag_path, **kwargs): + kwargs.setdefault("default_engine", "halo1yelo") + return extract_scripts_to_file(tagdata, tag_path, **kwargs) + + +def extract_h1_mcc_scripts(tagdata, tag_path, **kwargs): + kwargs.setdefault("default_engine", "halo1mcc") + return extract_scripts_to_file(tagdata, tag_path, **kwargs) \ No newline at end of file diff --git a/reclaimer/hek/defs/actr.py b/reclaimer/hek/defs/actr.py index fd7b047f..4e1c76d6 100644 --- a/reclaimer/hek/defs/actr.py +++ b/reclaimer/hek/defs/actr.py @@ -24,6 +24,19 @@ "unused5", ) +# split out to be reused in stubbs +panic = Struct("panic", + from_to_sec("cowering_time"), # seconds + float_zero_to_one("friend_killed_panic_chance"), + SEnum16("leader_type", *actor_types), + + Pad(2), + float_zero_to_one("leader_killed_panic_chance"), + float_zero_to_one("panic_damage_threshold"), + float_wu("surprise_distance"), # world units + SIZE=28 + ) + actr_body = Struct("tagdata", Bool32('flags', "can_see_in_darkness", @@ -115,9 +128,10 @@ float_wu("stationary_movement_dist"), # world units float_wu("free_flying_sidestep"), # world units float_rad("begin_moving_angle"), # radians + float_neg_one_to_one("cosine_begin_moving_angle", VISIBLE=False), ), - Pad(4), + Pad(0), Struct("looking", yp_float_rad("maximum_aiming_deviation"), # radians yp_float_rad("maximum_looking_deviation"), # radians @@ -162,16 +176,7 @@ ), Pad(8), - Struct("panic", - from_to_sec("cowering_time"), # seconds - float_zero_to_one("friend_killed_panic_chance"), - SEnum16("leader_type", *actor_types), - - Pad(2), - float_zero_to_one("leader_killed_panic_chance"), - float_zero_to_one("panic_damage_threshold"), - float_wu("surprise_distance"), # world units - ), + panic, Pad(28), Struct("defensive", diff --git a/reclaimer/hek/defs/actv.py b/reclaimer/hek/defs/actv.py index 571f988e..e687b86b 100644 --- a/reclaimer/hek/defs/actv.py +++ b/reclaimer/hek/defs/actv.py @@ -39,7 +39,8 @@ float_wu("collateral_damage_radius"), float_zero_to_one("grenade_chance"), float_sec("grenade_check_time", UNIT_SCALE=sec_unit_scale), - float_sec("encounter_grenade_timeout", UNIT_SCALE=sec_unit_scale) + float_sec("encounter_grenade_timeout", UNIT_SCALE=sec_unit_scale), + SIZE=52, ) actv_body = Struct("tagdata", @@ -56,12 +57,12 @@ dependency("actor_definition", "actr"), dependency("unit", valid_units), dependency("major_variant", "actv"), - SEnum16("mcc_scoring_type", TOOLTIP="Used to determine score in MCC", *mcc_actor_types), + Pad(4), # replaced with metagame_scoring in mcc_hek #Movement switching Struct("movement_switching", - Pad(22), + Pad(20), SEnum16("movement_type", "always_run", "always_crouch", diff --git a/reclaimer/hek/defs/antr.py b/reclaimer/hek/defs/antr.py index 9399a550..bb6a17df 100644 --- a/reclaimer/hek/defs/antr.py +++ b/reclaimer/hek/defs/antr.py @@ -103,7 +103,7 @@ *unit_weapon_animation_names ), reflexive("ik_points", ik_point_desc, 4, DYN_NAME_PATH=".marker"), - reflexive("weapon_types", weapon_types_desc, 10, DYN_NAME_PATH=".label"), + reflexive("weapon_types", weapon_types_desc, 16, DYN_NAME_PATH=".label"), SIZE=188, ) @@ -123,11 +123,12 @@ SInt16("up_frame_count"), Pad(8), - reflexive("animations", anim_enum_desc, 30, + reflexive("animations", anim_enum_desc, len(unit_animation_names), *unit_animation_names ), reflexive("ik_points", ik_point_desc, 4, DYN_NAME_PATH=".marker"), reflexive("weapons", unit_weapon_desc, 16, DYN_NAME_PATH=".name"), + Pad(0), # replaced with unknown reflexive in stubbs SIZE=100, ) @@ -161,7 +162,8 @@ SInt16("down_frame_count"), SInt16("up_frame_count"), - Pad(68), + Pad(56), + Pad(12), # replaced with seats reflexive in stubbs reflexive("animations", anim_enum_desc, 8, *vehicle_animation_names ), @@ -220,6 +222,7 @@ Float("weight"), SInt16("key_frame_index"), SInt16("second_key_frame_index"), + Pad(0), # replaced with Pad(8) in stubbs dyn_senum16("next_animation", DYN_NAME_PATH="..[DYN_I].name"), @@ -288,7 +291,8 @@ ), Pad(2), reflexive("nodes", nodes_desc, 64, DYN_NAME_PATH=".name"), - reflexive("animations", animation_desc, 256, DYN_NAME_PATH=".name"), + reflexive("animations", animation_desc, 256, DYN_NAME_PATH=".name", EXT_MAX=2048), + Pad(0), # replaced with stock_animation in magy tag class in os_hek SIZE=128, ) diff --git a/reclaimer/hek/defs/bipd.py b/reclaimer/hek/defs/bipd.py index 18d22720..0b76bbf6 100644 --- a/reclaimer/hek/defs/bipd.py +++ b/reclaimer/hek/defs/bipd.py @@ -19,14 +19,8 @@ from .objs.bipd import BipdTag from .obje import * from .unit import * -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(0)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "bipd") contact_point = Struct("contact_point", Pad(32), ascii_str32('marker_name'), diff --git a/reclaimer/hek/defs/bitm.py b/reclaimer/hek/defs/bitm.py index 7cfec1c6..cde55087 100644 --- a/reclaimer/hek/defs/bitm.py +++ b/reclaimer/hek/defs/bitm.py @@ -144,7 +144,7 @@ def pixel_block_size(node, *a, **kwa): SInt16("first_bitmap_index"), SInt16("bitmap_count"), Pad(16), - reflexive("sprites", sprite, 64), + reflexive("sprites", sprite, 64, EXT_MAX=SINT16_MAX), SIZE=64, ) @@ -256,7 +256,7 @@ def pixel_block_size(node, *a, **kwa): {NAME: "x512", VALUE: 4, GUI_NAME: "512x512"}, ), UInt16("sprite_budget_count"), - COMMENT=sprite_processing_comment + SIZE=4, COMMENT=sprite_processing_comment ), UInt16("color_plate_width", SIDETIP="pixels", EDITABLE=False), UInt16("color_plate_height", SIDETIP="pixels", EDITABLE=False), @@ -275,7 +275,7 @@ def pixel_block_size(node, *a, **kwa): UInt16("sprite_spacing", SIDETIP="pixels"), Pad(2), reflexive("sequences", sequence, 256, - DYN_NAME_PATH='.sequence_name', IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.sequence_name', IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("bitmaps", bitmap, 2048, IGNORE_SAFE_MODE=True), SIZE=108, WIDGET=HaloBitmapTagFrame ) diff --git a/reclaimer/hek/defs/cdmg.py b/reclaimer/hek/defs/cdmg.py index dd6f27b3..7820e493 100644 --- a/reclaimer/hek/defs/cdmg.py +++ b/reclaimer/hek/defs/cdmg.py @@ -8,9 +8,20 @@ # from ...common_descs import * +from .jpt_ import damage, camera_shaking from .objs.tag import HekTag from supyr_struct.defs.tag_def import TagDef +damage = desc_variant(damage, + ("aoe_core_radius", Pad(4)), + ("active_camouflage_damage", Pad(4)), + ) + +camera_shaking = desc_variant(camera_shaking, + ("duration", Pad(4)), + ("fade_function", Pad(2)), + ) + cdmg_body = Struct("tagdata", from_to_wu("radius"), float_zero_to_one("cutoff_scale"), @@ -19,59 +30,13 @@ QStruct("vibrate_parameters", float_zero_to_one("low_frequency"), float_zero_to_one("high_frequency"), - Pad(24), + Pad(16), ), - Struct("camera_shaking", - float_wu("random_translation"), - float_rad("random_rotation"), # radians - Pad(12), - - SEnum16("wobble_function", *animation_functions), - Pad(2), - float_sec("wobble_function_period"), - Float("wobble_weight"), - Pad(192), - ), - - Struct("damage", - SEnum16("priority", - "none", - "harmless", - {NAME: "backstab", GUI_NAME: "lethal to the unsuspecting"}, - "emp", - ), - SEnum16("category", *damage_category), - Bool32("flags", - "does_not_hurt_owner", - {NAME: "headshot", GUI_NAME: "can cause headshots"}, - "pings_resistant_units", - "does_not_hurt_friends", - "does_not_ping_shields", - "detonates_explosives", - "only_hurts_shields", - "causes_flaming_death", - {NAME: "indicator_points_down", - GUI_NAME: "damage indicator always points down"}, - "skips_shields", - "only_hurts_one_infection_form", - {NAME: "multiplayer_headshot", - GUI_NAME: "can cause multiplayer headshots"}, - "infection_form_pop", - ), - Pad(4), - Float("damage_lower_bound"), - QStruct("damage_upper_bound", INCLUDE=from_to), - float_zero_to_one("vehicle_passthrough_penalty"), - Pad(4), - float_zero_to_one("stun"), - float_zero_to_one("maximum_stun"), - float_sec("stun_time"), - Pad(4), - float_zero_to_inf("instantaneous_acceleration"), - Pad(8), - ), + camera_shaking, + Pad(160), + damage, damage_modifiers, SIZE=512, ) diff --git a/reclaimer/hek/defs/coll.py b/reclaimer/hek/defs/coll.py index 8728ae38..2c996a48 100644 --- a/reclaimer/hek/defs/coll.py +++ b/reclaimer/hek/defs/coll.py @@ -65,7 +65,8 @@ dependency("shield_depleted_effect", "effe"), dependency("shield_recharging_effect", "effe"), Pad(8), - Float("shield_recharge_rate", VISIBLE=False), + FlFloat("shield_recharge_rate", VISIBLE=False), + SIZE=248 ) bsp3d_node = QStruct("bsp3d_node", @@ -140,10 +141,10 @@ permutation_bsp = Struct("permutation_bsp", reflexive("bsp3d_nodes", bsp3d_node, 131072), - reflexive("planes", plane, 65535), - reflexive("leaves", leaf, 65535), + reflexive("planes", plane, UINT16_INDEX_MAX), + reflexive("leaves", leaf, UINT16_INDEX_MAX), reflexive("bsp2d_references", bsp2d_reference, 131072), - reflexive("bsp2d_nodes", bsp2d_node, 65535), + reflexive("bsp2d_nodes", bsp2d_node, UINT16_INDEX_MAX), reflexive("surfaces", surface, 131072), reflexive("edges", edge, 262144), reflexive("vertices", vertex, 131072), @@ -162,8 +163,22 @@ DYN_NAME_PATH="..[DYN_I].name"), Pad(8), - FlSInt16("unknown0", VISIBLE=False), - FlSInt16("unknown1", VISIBLE=False), + FlSInt16("unknown", VISIBLE=False), + FlSEnum16("damage_region", + "waist", + "torso", + "head", + "l_arm", + "l_hand", + "unused0", + "l_leg", + "r_arm", + "r_hand", + "unused1", + "r_leg", + ("none", -1), + VISIBLE=False + ), reflexive("bsps", permutation_bsp, 32), SIZE=64 ) @@ -259,10 +274,10 @@ fast_permutation_bsp = Struct("permutation_bsp", raw_reflexive("bsp3d_nodes", bsp3d_node, 131072), - raw_reflexive("planes", plane, 65535), - raw_reflexive("leaves", leaf, 65535), + raw_reflexive("planes", plane, UINT16_INDEX_MAX), + raw_reflexive("leaves", leaf, UINT16_INDEX_MAX), raw_reflexive("bsp2d_references", bsp2d_reference, 131072), - raw_reflexive("bsp2d_nodes", bsp2d_node, 65535), + raw_reflexive("bsp2d_nodes", bsp2d_node, UINT16_INDEX_MAX), raw_reflexive("surfaces", surface, 131072), raw_reflexive("edges", edge, 262144), raw_reflexive("vertices", vertex, 131072), @@ -281,8 +296,8 @@ DYN_NAME_PATH="..[DYN_I].name"), Pad(8), - FlSInt16("unknown0", VISIBLE=False), - FlSInt16("unknown1", VISIBLE=False), + FlSInt16("unknown", VISIBLE=False), + FlSInt16("damage_region", VISIBLE=False), reflexive("bsps", fast_permutation_bsp, 32), SIZE=64 ) diff --git a/reclaimer/hek/defs/cont.py b/reclaimer/hek/defs/cont.py index df29fa6e..fe01049b 100644 --- a/reclaimer/hek/defs/cont.py +++ b/reclaimer/hek/defs/cont.py @@ -83,7 +83,7 @@ def get(): return cont_def SInt16("sequence_count"), Pad(100), - FlUInt32("unknown0", VISIBLE=False), + FlUInt32("unknown", VISIBLE=False), Bool16("shader_flags", *shader_flags), SEnum16("framebuffer_blend_function", *framebuffer_blend_functions), SEnum16("framebuffer_fade_mode", *render_fade_mode), diff --git a/reclaimer/hek/defs/ctrl.py b/reclaimer/hek/defs/ctrl.py index 6ce72336..619d24e2 100644 --- a/reclaimer/hek/defs/ctrl.py +++ b/reclaimer/hek/defs/ctrl.py @@ -10,15 +10,8 @@ from .obje import * from .devi import * from .objs.ctrl import CtrlTag -from supyr_struct.defs.tag_def import TagDef -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(8)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "ctrl") ctrl_attrs = Struct("ctrl_attrs", SEnum16('type', 'toggle_switch', diff --git a/reclaimer/hek/defs/deca.py b/reclaimer/hek/defs/deca.py index c8a779cd..50d7b367 100644 --- a/reclaimer/hek/defs/deca.py +++ b/reclaimer/hek/defs/deca.py @@ -8,7 +8,7 @@ # from ...common_descs import * -from .objs.tag import HekTag +from .objs.deca import DecaTag from supyr_struct.defs.tag_def import TagDef decal_comment = """COMPOUND DECALS: @@ -91,7 +91,7 @@ #Sprite info Pad(20), - Float("maximum_sprite_extent", SIDETIP="pixels"), + Float("maximum_sprite_extent", SIDETIP="pixels", VISIBLE=False), SIZE=268, ) @@ -105,5 +105,5 @@ def get(): blam_header('deca'), deca_body, - ext=".decal", endian=">", tag_cls=HekTag + ext=".decal", endian=">", tag_cls=DecaTag ) diff --git a/reclaimer/hek/defs/dobc.py b/reclaimer/hek/defs/dobc.py index 6086a765..0b98f02b 100644 --- a/reclaimer/hek/defs/dobc.py +++ b/reclaimer/hek/defs/dobc.py @@ -20,8 +20,7 @@ def get(): return dobc_def ("interpolate_color_in_hsv", 4), ("more_colors", 8), ), - #UInt8("unknown0", VISIBLE=False), - Pad(1), + UInt8("first_sprite_index", VISIBLE=False), UInt8("sequence_sprite_count", VISIBLE=False), float_zero_to_one("color_override_factor"), Pad(8), diff --git a/reclaimer/hek/defs/effe.py b/reclaimer/hek/defs/effe.py index 32340e93..8dcc5f8a 100644 --- a/reclaimer/hek/defs/effe.py +++ b/reclaimer/hek/defs/effe.py @@ -154,15 +154,18 @@ effe_body = Struct("tagdata", Bool32("flags", {NAME: "deleted_when_inactive", GUI_NAME: "deleted when attachment deactivates"}, + # NOTE: on xbox the must_be_deterministic flag is in place + # of required, as the required flag didn't exist. {NAME: "required", GUI_NAME: "required for gameplay (cannot optimize out)"}, - {NAME: "never_cull", VISIBLE: VISIBILITY_HIDDEN} + {NAME: "must_be_deterministic", VISIBLE: VISIBILITY_HIDDEN} ), dyn_senum16("loop_start_event", DYN_NAME_PATH=".events.events_array[DYN_I].NAME"), dyn_senum16("loop_stop_event", DYN_NAME_PATH=".events.events_array[DYN_I].NAME"), + FlFloat("max_damage_radius", VISIBLE=False), - Pad(32), + Pad(28), reflexive("locations", location, 32, DYN_NAME_PATH='.marker_name'), reflexive("events", event, 32), diff --git a/reclaimer/hek/defs/eqip.py b/reclaimer/hek/defs/eqip.py index cd8b091b..e737fa06 100644 --- a/reclaimer/hek/defs/eqip.py +++ b/reclaimer/hek/defs/eqip.py @@ -10,15 +10,8 @@ from .obje import * from .item import * from .objs.obje import ObjeTag -from supyr_struct.defs.tag_def import TagDef -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(3)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "eqip") eqip_attrs = Struct("eqip_attrs", SEnum16('powerup_type', 'none', @@ -32,6 +25,8 @@ SEnum16('grenade_type', *grenade_types), float_sec('powerup_time'), dependency('pickup_sound', "snd!"), + Pad(144), # looks like open sauce HAD plans for this at one point. + # keeping the padding defined here cause, well, who knows? SIZE=168 ) diff --git a/reclaimer/hek/defs/font.py b/reclaimer/hek/defs/font.py index cc4e50a7..28cc3605 100644 --- a/reclaimer/hek/defs/font.py +++ b/reclaimer/hek/defs/font.py @@ -40,7 +40,7 @@ def get(): return font_def font_body = Struct("tagdata", SInt32("flags"), SInt16("ascending_height"), - SInt16("decending_height"), + SInt16("descending_height"), SInt16("leading_height"), SInt16("leading_width"), Pad(36), diff --git a/reclaimer/hek/defs/garb.py b/reclaimer/hek/defs/garb.py index 039e4a46..3828202e 100644 --- a/reclaimer/hek/defs/garb.py +++ b/reclaimer/hek/defs/garb.py @@ -10,15 +10,8 @@ from .obje import * from .item import * from .objs.obje import ObjeTag -from supyr_struct.defs.tag_def import TagDef -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(4)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "garb") garb_body = Struct("tagdata", obje_attrs, item_attrs, diff --git a/reclaimer/hek/defs/hudg.py b/reclaimer/hek/defs/hudg.py index 4990b261..25ee11f8 100644 --- a/reclaimer/hek/defs/hudg.py +++ b/reclaimer/hek/defs/hudg.py @@ -182,7 +182,8 @@ SInt16("checkpoint_begin_text"), SInt16("checkpoint_end_text"), dependency("checkpoint", "snd!"), - BytearrayRaw("unknown", SIZE=96, VISIBLE=False), + BytearrayRaw("unknown0", SIZE=12, VISIBLE=False), # replaced with remaps in mcc + BytearrayRaw("unknown1", SIZE=84, VISIBLE=False), SIZE=120 ) diff --git a/reclaimer/hek/defs/jpt_.py b/reclaimer/hek/defs/jpt_.py index bb37426f..2cfb879c 100644 --- a/reclaimer/hek/defs/jpt_.py +++ b/reclaimer/hek/defs/jpt_.py @@ -11,6 +11,59 @@ from .objs.tag import HekTag from supyr_struct.defs.tag_def import TagDef +# split out to be reused in mcc_hek +damage = Struct("damage", + SEnum16("priority", + "none", + "harmless", + {NAME: "backstab", GUI_NAME: "lethal to the unsuspecting"}, + "emp", + ), + SEnum16("category", *damage_category), + Bool32("flags", + "does_not_hurt_owner", + {NAME: "headshot", GUI_NAME: "can cause headshots"}, + "pings_resistant_units", + "does_not_hurt_friends", + "does_not_ping_units", + "detonates_explosives", + "only_hurts_shields", + "causes_flaming_death", + {NAME: "indicator_points_down", GUI_NAME: "damage indicator always points down"}, + "skips_shields", + "only_hurts_one_infection_form", + {NAME: "multiplayer_headshot", GUI_NAME: "can cause multiplayer headshots"}, + "infection_form_pop", + ), + float_wu("aoe_core_radius"), + Float("damage_lower_bound"), + QStruct("damage_upper_bound", INCLUDE=from_to), + float_zero_to_one("vehicle_passthrough_penalty"), + float_zero_to_one("active_camouflage_damage"), + float_zero_to_one("stun"), + float_zero_to_one("maximum_stun"), + float_sec("stun_time"), + Pad(4), + float_zero_to_inf("instantaneous_acceleration"), + Pad(8), + SIZE=60 + ) + +camera_shaking = Struct("camera_shaking", + float_sec("duration"), + SEnum16("fade_function", *fade_functions), + Pad(2), + + float_wu("random_translation"), + float_rad("random_rotation"), # radians + Pad(12), + + SEnum16("wobble_function", *animation_functions), + Pad(2), + float_sec("wobble_function_period"), + Float("wobble_weight"), + Pad(32), + ) frequency_vibration = Struct("", float_zero_to_one("frequency"), @@ -72,22 +125,7 @@ float_rad("permanent_camera_impulse_angle"), Pad(16), - - Struct("camera_shaking", - float_sec("duration"), - SEnum16("fade_function", *fade_functions), - Pad(2), - - float_wu("random_translation"), - float_rad("random_rotation"), # radians - Pad(12), - - SEnum16("wobble_function", *animation_functions), - Pad(2), - float_sec("wobble_function_period"), - Float("wobble_weight"), - Pad(32), - ), + camera_shaking, dependency("sound", "snd!"), Pad(112), @@ -104,44 +142,7 @@ Pad(12), ), - Struct("damage", - SEnum16("priority", - "none", - "harmless", - {NAME: "backstab", GUI_NAME: "lethal to the unsuspecting"}, - "emp", - ), - SEnum16("category", *damage_category), - Bool32("flags", - "does_not_hurt_owner", - {NAME: "headshot", GUI_NAME: "causes headshots"}, - "pings_resistant_units", - "does_not_hurt_friends", - "does_not_ping_units", - "detonates_explosives", - "only_hurts_shields", - "causes_flaming_death", - {NAME: "indicator_points_down", - GUI_NAME: "damage indicator always points down"}, - "skips_shields", - "only_hurts_one_infection_form", - {NAME: "multiplayer_headshot", - GUI_NAME: "causes multiplayer headshots"}, - "infection_form_pop", - ), - float_wu("aoe_core_radius"), - Float("damage_lower_bound"), - QStruct("damage_upper_bound", INCLUDE=from_to), - float_zero_to_one("vehicle_passthrough_penalty"), - float_zero_to_one("active_camouflage_damage"), - float_zero_to_one("stun"), - float_zero_to_one("maximum_stun"), - float_sec("stun_time"), - Pad(4), - float_zero_to_inf("instantaneous_acceleration"), - Pad(8), - ), - + damage, damage_modifiers, SIZE=672, ) diff --git a/reclaimer/hek/defs/lens.py b/reclaimer/hek/defs/lens.py index 15cc1bf6..794b910d 100644 --- a/reclaimer/hek/defs/lens.py +++ b/reclaimer/hek/defs/lens.py @@ -62,6 +62,15 @@ SIZE=128 ) +# for reuse in mcc +bitmaps = Struct("bitmaps", + dependency("bitmap", "bitm"), + Bool16("flags", + "sun", + ), + Pad(78), + SIZE=96 + ) lens_body = Struct("tagdata", float_rad("falloff_angle"), # radians @@ -81,14 +90,7 @@ COMMENT=occlusion_comment ), - Struct("bitmaps", - dependency("bitmap", "bitm"), - Bool16("flags", - "sun", - ), - Pad(78), - ), - + bitmaps, Struct("corona_rotation", SEnum16("function", "none", diff --git a/reclaimer/hek/defs/lsnd.py b/reclaimer/hek/defs/lsnd.py index 0914ebbf..794c7591 100644 --- a/reclaimer/hek/defs/lsnd.py +++ b/reclaimer/hek/defs/lsnd.py @@ -75,7 +75,7 @@ FlFloat("unknown3", DEFAULT=1.0, VISIBLE=False), FlSInt16("unknown4", DEFAULT=-1, VISIBLE=False), FlSInt16("unknown5", DEFAULT=-1, VISIBLE=False), - FlFloat("unknown6", DEFAULT=1.0, VISIBLE=False), + FlFloat("max_distance", DEFAULT=1.0, VISIBLE=False), Pad(8), dependency("continuous_damage_effect", "cdmg"), diff --git a/reclaimer/hek/defs/mach.py b/reclaimer/hek/defs/mach.py index c532f018..184fc548 100644 --- a/reclaimer/hek/defs/mach.py +++ b/reclaimer/hek/defs/mach.py @@ -10,15 +10,8 @@ from .obje import * from .devi import * from .objs.mach import MachTag -from supyr_struct.defs.tag_def import TagDef -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(7)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "mach") mach_attrs = Struct("mach_attrs", SEnum16('type', 'door', diff --git a/reclaimer/hek/defs/matg.py b/reclaimer/hek/defs/matg.py index 15319bea..c291c903 100644 --- a/reclaimer/hek/defs/matg.py +++ b/reclaimer/hek/defs/matg.py @@ -291,6 +291,10 @@ def get(): dependency('vehicle_killed_unit_damage', "jpt!"), dependency('vehicle_collision_damage', "jpt!"), dependency('flaming_death_damage', "jpt!"), + Pad(16), + FlFloat("max_falling_velocity", VISIBLE=False), + FlFloat("harmful_falling_velocity", VISIBLE=False), + Pad(4), SIZE=152 ) diff --git a/reclaimer/hek/defs/mgs2.py b/reclaimer/hek/defs/mgs2.py index 076a3290..63771c19 100644 --- a/reclaimer/hek/defs/mgs2.py +++ b/reclaimer/hek/defs/mgs2.py @@ -8,7 +8,7 @@ # from ...common_descs import * -from .objs.tag import HekTag +from .objs.mgs2 import Mgs2Tag from supyr_struct.defs.tag_def import TagDef light_volume_comment = """LIGHT VOLUME @@ -84,5 +84,5 @@ def get(): blam_header("mgs2"), mgs2_body, - ext=".light_volume", endian=">", tag_cls=HekTag, + ext=".light_volume", endian=">", tag_cls=Mgs2Tag, ) diff --git a/reclaimer/hek/defs/mod2.py b/reclaimer/hek/defs/mod2.py index 42264d95..30df8d0c 100644 --- a/reclaimer/hek/defs/mod2.py +++ b/reclaimer/hek/defs/mod2.py @@ -45,12 +45,12 @@ def get(): UInt32('binormal'), UInt32('tangent'), - SInt16('u', UNIT_SCALE=1/32767, MIN=-32767, WIDGET_WIDTH=10), - SInt16('v', UNIT_SCALE=1/32767, MIN=-32767, WIDGET_WIDTH=10), + SInt16('u', UNIT_SCALE=1/SINT16_MAX, MIN=-SINT16_MAX, WIDGET_WIDTH=10), + SInt16('v', UNIT_SCALE=1/SINT16_MAX, MIN=-SINT16_MAX, WIDGET_WIDTH=10), SInt8('node_0_index', UNIT_SCALE=1/3, MIN=0, WIDGET_WIDTH=10), SInt8('node_1_index', UNIT_SCALE=1/3, MIN=0, WIDGET_WIDTH=10), - SInt16('node_0_weight', UNIT_SCALE=1/32767, MIN=0, WIDGET_WIDTH=10), + SInt16('node_0_weight', UNIT_SCALE=1/SINT16_MAX, MIN=0, WIDGET_WIDTH=10), SIZE=32 ) @@ -79,12 +79,12 @@ def get(): BBitStruct('binormal', INCLUDE=compressed_normal_32, SIZE=4), BBitStruct('tangent', INCLUDE=compressed_normal_32, SIZE=4), - SInt16('u', UNIT_SCALE=1/32767, MIN=-32767, WIDGET_WIDTH=10), - SInt16('v', UNIT_SCALE=1/32767, MIN=-32767, WIDGET_WIDTH=10), + SInt16('u', UNIT_SCALE=1/SINT16_MAX, MIN=-SINT16_MAX, WIDGET_WIDTH=10), + SInt16('v', UNIT_SCALE=1/SINT16_MAX, MIN=-SINT16_MAX, WIDGET_WIDTH=10), SInt8('node_0_index', UNIT_SCALE=1/3, MIN=0, WIDGET_WIDTH=10), SInt8('node_1_index', UNIT_SCALE=1/3, MIN=0, WIDGET_WIDTH=10), - SInt16('node_0_weight', UNIT_SCALE=1/32767, MIN=0, WIDGET_WIDTH=10), + SInt16('node_0_weight', UNIT_SCALE=1/SINT16_MAX, MIN=0, WIDGET_WIDTH=10), SIZE=32 ) @@ -147,9 +147,27 @@ def get(): DYN_NAME_PATH="tagdata.geometries.geometries_array[DYN_I].NAME"), Pad(2), - reflexive("local_markers", local_marker, 32, DYN_NAME_PATH=".name"), + reflexive("local_markers", local_marker, 32, DYN_NAME_PATH=".name", EXT_MAX=SINT16_MAX), SIZE=88 ) + +model_meta_info = Struct("model_meta_info", + UEnum16("index_type", *H1_TRIANGLE_BUFFER_TYPES), + Pad(2), + UInt32("index_count"), + # THESE TWO VALUES ARE DIFFERENT THAN ON XBOX IT SEEMS + UInt32("indices_magic_offset"), + UInt32("indices_offset"), + + UEnum16("vertex_type", *H1_VERTEX_BUFFER_TYPES), + Pad(2), + UInt32("vertex_count"), + Pad(4), # always 0? + # THESE TWO VALUES ARE DIFFERENT THAN ON XBOX IT SEEMS + UInt32("vertices_magic_offset"), + UInt32("vertices_offset"), + VISIBLE=False, SIZE=36 + ) part = Struct('part', Bool32('flags', @@ -168,35 +186,14 @@ def get(): QStruct('centroid_translation', INCLUDE=xyz_float), - #reflexive("uncompressed_vertices", uncompressed_vertex_union, 32767), - #reflexive("compressed_vertices", compressed_vertex_union, 32767), - #reflexive("triangles", triangle_union, 32767), - reflexive("uncompressed_vertices", fast_uncompressed_vertex, 32767), - reflexive("compressed_vertices", fast_compressed_vertex, 32767), - reflexive("triangles", triangle, 32767), + #reflexive("uncompressed_vertices", uncompressed_vertex_union, SINT16_MAX), + #reflexive("compressed_vertices", compressed_vertex_union, SINT16_MAX), + #reflexive("triangles", triangle_union, SINT16_MAX), + reflexive("uncompressed_vertices", fast_uncompressed_vertex, SINT16_MAX), + reflexive("compressed_vertices", fast_compressed_vertex, SINT16_MAX), + reflexive("triangles", triangle, SINT16_MAX), #Pad(36), - Struct("model_meta_info", - UEnum16("index_type", # name is a guess. always 1? - ("uncompressed", 1), - ), - Pad(2), - UInt32("index_count"), - # THESE TWO VALUES ARE DIFFERENT THAN ON XBOX IT SEEMS - UInt32("indices_magic_offset"), - UInt32("indices_offset"), - - UEnum16("vertex_type", # name is a guess - ("uncompressed", 4), - ("compressed", 5), - ), - Pad(2), - UInt32("vertex_count"), - Pad(4), # always 0? - # THESE TWO VALUES ARE DIFFERENT THAN ON XBOX IT SEEMS - UInt32("vertices_magic_offset"), - UInt32("vertices_offset"), - VISIBLE=False, SIZE=36 - ), + model_meta_info, Pad(3), SInt8('local_node_count', MIN=0, MAX=22), @@ -210,9 +207,9 @@ def get(): ) fast_part = desc_variant(part, - ("uncompressed_vertices", raw_reflexive("uncompressed_vertices", fast_uncompressed_vertex, 65535)), - ("compressed_vertices", raw_reflexive("compressed_vertices", fast_compressed_vertex, 65535)), - ("triangles", raw_reflexive("triangles", triangle, 65535)), + raw_reflexive("uncompressed_vertices", fast_uncompressed_vertex, SINT16_MAX), + raw_reflexive("compressed_vertices", fast_compressed_vertex, SINT16_MAX), + raw_reflexive("triangles", triangle, SINT16_MAX), ) marker = Struct('marker', @@ -220,7 +217,7 @@ def get(): UInt16('magic_identifier'), Pad(18), - reflexive("marker_instances", marker_instance, 32), + reflexive("marker_instances", marker_instance, 32, EXT_MAX=SINT16_MAX), SIZE=64 ) @@ -252,19 +249,19 @@ def get(): region = Struct('region', ascii_str32("name"), Pad(32), - reflexive("permutations", permutation, 32, DYN_NAME_PATH=".name"), + reflexive("permutations", permutation, 32, DYN_NAME_PATH=".name", EXT_MAX=UINT8_MAX), SIZE=76 ) geometry = Struct('geometry', Pad(36), - reflexive("parts", part, 32), + reflexive("parts", part, 32, EXT_MAX=SINT16_MAX), SIZE=48 ) fast_geometry = Struct('geometry', Pad(36), - reflexive("parts", fast_part, 32), + reflexive("parts", fast_part, 32, EXT_MAX=SINT16_MAX), SIZE=48 ) @@ -289,11 +286,11 @@ def get(): Float('low_lod_cutoff', SIDETIP="pixels"), Float('superlow_lod_cutoff', SIDETIP="pixels"), - SInt16('superlow_lod_nodes', SIDETIP="nodes"), - SInt16('low_lod_nodes', SIDETIP="nodes"), - SInt16('medium_lod_nodes', SIDETIP="nodes"), - SInt16('high_lod_nodes', SIDETIP="nodes"), - SInt16('superhigh_lod_nodes', SIDETIP="nodes"), + SInt16('superlow_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('low_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('medium_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('high_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('superhigh_lod_nodes', SIDETIP="nodes", VISIBLE=False), Pad(10), @@ -302,17 +299,23 @@ def get(): Pad(116), - reflexive("markers", marker, 256, DYN_NAME_PATH=".name"), + reflexive("markers", marker, 256, DYN_NAME_PATH=".name", VISIBLE=False), + # NOTE: the extended max for nodes and regions can't be set higher due + # to data structure limits. there is an object struct used at + # runtime that only has enough room to reference 8 regions. for + # nodes, there is only enough room in each animation to flag 64 + # nodes as animated. theoretically you could add another 32 if + # if used neighboring padding, but this is untested, and a strech. reflexive("nodes", node, 64, DYN_NAME_PATH=".name"), reflexive("regions", region, 32, DYN_NAME_PATH=".name"), - reflexive("geometries", geometry, 256), - reflexive("shaders", shader, 256, DYN_NAME_PATH=".shader.filepath"), + reflexive("geometries", geometry, 256, EXT_MAX=SINT16_MAX), + reflexive("shaders", shader, 256, DYN_NAME_PATH=".shader.filepath", EXT_MAX=SINT16_MAX), SIZE=232 ) fast_mod2_body = desc_variant(mod2_body, - ("geometries", reflexive("geometries", fast_geometry, 256)), + reflexive("geometries", fast_geometry, 256), ) mod2_def = TagDef("mod2", diff --git a/reclaimer/hek/defs/mode.py b/reclaimer/hek/defs/mode.py index 2b67917c..d7b841ea 100644 --- a/reclaimer/hek/defs/mode.py +++ b/reclaimer/hek/defs/mode.py @@ -42,10 +42,27 @@ def get(): DYN_NAME_PATH="tagdata.geometries.geometries_array[DYN_I].NAME"), Pad(2), - reflexive("local_markers", local_marker, 32, DYN_NAME_PATH=".name"), + reflexive("local_markers", local_marker, 32, DYN_NAME_PATH=".name", EXT_MAX=SINT16_MAX), SIZE=88 ) +# dip == double-indirect pointer +dip_model_meta_info = Struct("model_meta_info", + UEnum16("index_type", *H1_TRIANGLE_BUFFER_TYPES), + Pad(2), + UInt32("index_count"), + UInt32("indices_offset"), + UInt32("indices_reflexive_offset"), + + UEnum16("vertex_type", *H1_VERTEX_BUFFER_TYPES), + Pad(2), + UInt32("vertex_count"), + Pad(4), # always 0? + UInt32("vertices_offset"), + UInt32("vertices_reflexive_offset"), + VISIBLE=False, SIZE=36 + ) + part = Struct('part', Bool32('flags', 'stripped', @@ -62,60 +79,41 @@ def get(): QStruct('centroid_translation', INCLUDE=xyz_float), - #reflexive("uncompressed_vertices", uncompressed_vertex_union, 65535), - #reflexive("compressed_vertices", compressed_vertex_union, 65535), - #reflexive("triangles", triangle_union, 65535), - reflexive("uncompressed_vertices", fast_uncompressed_vertex, 65535), - reflexive("compressed_vertices", fast_compressed_vertex, 65535), - reflexive("triangles", triangle, 65535), + #reflexive("uncompressed_vertices", uncompressed_vertex_union, SINT16_MAX), + #reflexive("compressed_vertices", compressed_vertex_union, SINT16_MAX), + #reflexive("triangles", triangle_union, SINT16_MAX), + reflexive("uncompressed_vertices", fast_uncompressed_vertex, SINT16_MAX), + reflexive("compressed_vertices", fast_compressed_vertex, SINT16_MAX), + reflexive("triangles", triangle, SINT16_MAX), #Pad(36), - Struct("model_meta_info", - UEnum16("index_type", # name is a guess. always 1? - ("uncompressed", 1), - ), - Pad(2), - UInt32("index_count"), - UInt32("indices_offset"), - UInt32("indices_reflexive_offset"), - - UEnum16("vertex_type", # name is a guess - ("uncompressed", 4), - ("compressed", 5), - ), - Pad(2), - UInt32("vertex_count"), - Pad(4), # always 0? - UInt32("vertices_offset"), - UInt32("vertices_reflexive_offset"), - VISIBLE=False, SIZE=36 - ), + dip_model_meta_info, SIZE=104 ) fast_part = desc_variant(part, - ("uncompressed_vertices", raw_reflexive("uncompressed_vertices", fast_uncompressed_vertex, 65535)), - ("compressed_vertices", raw_reflexive("compressed_vertices", fast_compressed_vertex, 65535)), - ("triangles", raw_reflexive("triangles", triangle, 65535)), + raw_reflexive("uncompressed_vertices", fast_uncompressed_vertex, SINT16_MAX), + raw_reflexive("compressed_vertices", fast_compressed_vertex, SINT16_MAX), + raw_reflexive("triangles", triangle, SINT16_MAX), ) region = Struct('region', ascii_str32("name"), Pad(32), - reflexive("permutations", permutation, 32, DYN_NAME_PATH=".name"), + reflexive("permutations", permutation, 32, DYN_NAME_PATH=".name", EXT_MAX=UINT8_MAX), SIZE=76 ) geometry = Struct('geometry', Pad(36), - reflexive("parts", part, 32), + reflexive("parts", part, 32, EXT_MAX=SINT16_MAX), SIZE=48 ) fast_geometry = Struct('geometry', Pad(36), - reflexive("parts", fast_part, 32), + reflexive("parts", fast_part, 32, EXT_MAX=SINT16_MAX), SIZE=48 ) @@ -125,37 +123,43 @@ def get(): ), SInt32('node_list_checksum'), - # xbox has these values swapped around in order - Float('superlow_lod_cutoff', SIDETIP="pixels"), - Float('low_lod_cutoff', SIDETIP="pixels"), - Float('medium_lod_cutoff', SIDETIP="pixels"), - Float('high_lod_cutoff', SIDETIP="pixels"), Float('superhigh_lod_cutoff', SIDETIP="pixels"), + Float('high_lod_cutoff', SIDETIP="pixels"), + Float('medium_lod_cutoff', SIDETIP="pixels"), + Float('low_lod_cutoff', SIDETIP="pixels"), + Float('superlow_lod_cutoff', SIDETIP="pixels"), - SInt16('superlow_lod_nodes', SIDETIP="nodes"), - SInt16('low_lod_nodes', SIDETIP="nodes"), - SInt16('medium_lod_nodes', SIDETIP="nodes"), - SInt16('high_lod_nodes', SIDETIP="nodes"), - SInt16('superhigh_lod_nodes', SIDETIP="nodes"), + SInt16('superlow_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('low_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('medium_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('high_lod_nodes', SIDETIP="nodes", VISIBLE=False), + SInt16('superhigh_lod_nodes', SIDETIP="nodes", VISIBLE=False), Pad(10), Float('base_map_u_scale'), Float('base_map_v_scale'), - Pad(116), + Pad(104), + Pad(12), # replaced with unknown reflexive in stubbs - reflexive("markers", marker, 256, DYN_NAME_PATH=".name"), + # NOTE: the extended max for nodes and regions can't be set higher due + # to data structure limits. there is an object struct used at + # runtime that only has enough room to reference 8 regions. for + # nodes, there is only enough room in each animation to flag 64 + # nodes as animated. theoretically you could add another 32 if + # if used neighboring padding, but this is untested, and a strech. + reflexive("markers", marker, 256, DYN_NAME_PATH=".name", VISIBLE=False), reflexive("nodes", node, 64, DYN_NAME_PATH=".name"), reflexive("regions", region, 32, DYN_NAME_PATH=".name"), - reflexive("geometries", geometry, 256), - reflexive("shaders", shader, 256, DYN_NAME_PATH=".shader.filepath"), + reflexive("geometries", geometry, 256, EXT_MAX=SINT16_MAX), + reflexive("shaders", shader, 256, DYN_NAME_PATH=".shader.filepath", EXT_MAX=SINT16_MAX), SIZE=232 ) fast_mode_body = desc_variant(mode_body, - ("geometries", reflexive("geometries", fast_geometry, 256)), + reflexive("geometries", fast_geometry, 256, EXT_MAX=SINT16_MAX), ) mode_def = TagDef("mode", diff --git a/reclaimer/hek/defs/obje.py b/reclaimer/hek/defs/obje.py index 7cbf0ef6..82d8fe5d 100644 --- a/reclaimer/hek/defs/obje.py +++ b/reclaimer/hek/defs/obje.py @@ -10,6 +10,8 @@ from ...common_descs import * from .objs.obje import ObjeTag from supyr_struct.defs.tag_def import TagDef +# import here so it can be reused in all variants of object +from supyr_struct.util import desc_variant def get(): return obje_def @@ -60,7 +62,11 @@ def get(): dyn_senum16('turn_off_with', DYN_NAME_PATH="..[DYN_I].usage"), Float('scale_by'), - Pad(268), + FlFloat('bounds_range_inverse', VISIBLE=False), + FlFloat('sawtooth_count_inverse', VISIBLE=False), + FlFloat('step_count_inverse', VISIBLE=False), + FlFloat('period_inverse', VISIBLE=False), + Pad(252), ascii_str32('usage'), SIZE=360 @@ -107,7 +113,10 @@ def get(): QStruct('origin_offset', INCLUDE=xyz_float), float_zero_to_inf('acceleration_scale', UNIT_SCALE=per_sec_unit_scale), - Pad(4), + FlBool32("runtime_flags", + "functions_control_color_scale", + VISIBLE=False + ), dependency('model', valid_models), dependency('animation_graph', "antr"), diff --git a/reclaimer/hek/defs/objs/actr.py b/reclaimer/hek/defs/objs/actr.py index 762f03b1..5d3ffa8e 100644 --- a/reclaimer/hek/defs/objs/actr.py +++ b/reclaimer/hek/defs/objs/actr.py @@ -16,6 +16,7 @@ class ActrTag(HekTag): def calc_internal_data(self): HekTag.calc_internal_data(self) perception = self.data.tagdata.perception + movement = self.data.tagdata.movement looking = self.data.tagdata.looking perception.inv_combat_perception_time = 0 @@ -35,6 +36,8 @@ def calc_internal_data(self): perception.inv_guard_perception_time /= 30 perception.inv_non_combat_perception_time /= 30 + movement.cosine_begin_moving_angle = cos(movement.begin_moving_angle) + for i in range(2): looking.cosine_maximum_aiming_deviation[i] = cos(looking.maximum_aiming_deviation[i]) looking.cosine_maximum_looking_deviation[i] = cos(looking.maximum_looking_deviation[i]) diff --git a/reclaimer/hek/defs/objs/bitm.py b/reclaimer/hek/defs/objs/bitm.py index cc1b5b0f..03beb196 100644 --- a/reclaimer/hek/defs/objs/bitm.py +++ b/reclaimer/hek/defs/objs/bitm.py @@ -9,18 +9,20 @@ from array import array from reclaimer.constants import TYPE_CUBEMAP, CUBEMAP_PADDING, BITMAP_PADDING,\ - FORMAT_NAME_MAP, TYPE_NAME_MAP, FORMAT_P8_BUMP + FORMAT_NAME_MAP, TYPE_NAME_MAP, FORMAT_P8_BUMP, SWIZZLEABLE_FORMATS from reclaimer.bitmaps.p8_palette import HALO_P8_PALETTE from reclaimer.hek.defs.objs.tag import HekTag try: import arbytmap as ab - if not hasattr(ab, "FORMAT_P8_BUMP"): ab.FORMAT_P8_BUMP = "P8-BUMP" """ADD THE P8 FORMAT TO THE BITMAP CONVERTER""" - ab.register_format(format_id=ab.FORMAT_P8_BUMP, depths=(8,8,8,8)) + ab.register_format( + format_id=ab.FORMAT_P8_BUMP, depths=(8,8,8,8) + ) + except (ImportError, AttributeError): ab = None @@ -29,10 +31,30 @@ class BitmTag(HekTag): tex_infos = () p8_palette = None + @property + def swizzleable_formats(self): + return SWIZZLEABLE_FORMATS + + @property + def format_name_map(self): + return FORMAT_NAME_MAP + + @property + def pixel_root_definition(self): + return self.definition.subdefs['pixel_root'] + def __init__(self, *args, **kwargs): HekTag.__init__(self, *args, **kwargs) self.p8_palette = HALO_P8_PALETTE + def calc_internal_data(self, **kwargs): + HekTag.calc_internal_data(self) + + tagdata = self.data.tagdata + if tagdata.compressed_color_plate_data.size == 0: + tagdata.color_plate_width = 0 + tagdata.color_plate_height = 0 + def bitmap_count(self, new_value=None): if new_value is None: return self.data.tagdata.bitmaps.size @@ -69,26 +91,21 @@ def bitmap_format(self, b_index=0, new_value=None): self.data.tagdata.bitmaps.bitmaps_array[b_index].format.data = new_value def fix_top_format(self): - if len(self.data.tagdata.bitmaps.bitmaps_array) <= 0: - self.data.tagdata.format.data = "color_key_transparency" - - # Why can't get_name get the name of the current option? - pixel_format = self.data.tagdata.bitmaps.bitmaps_array[0].format.get_name( - self.data.tagdata.bitmaps.bitmaps_array[0].format.data) - top_format = "color_key_transparency" - if pixel_format in ("a8", "y8", "ay8", "a8y8"): - top_format = "monochrome" - elif pixel_format in ("r5g6b5", "a1r5g5b5", "a4r4g4b4"): - top_format = "color_16bit" - elif pixel_format in ("x8r8g8b8", "a8r8g8b8", "p8_bump"): - top_format = "color_32bit" - elif pixel_format == "dxt1": - top_format = "color_key_transparency" - elif pixel_format == "dxt3": - top_format = "explicit_alpha" - elif pixel_format == "dxt5": - top_format = "interpolated_alpha" + if self.bitmap_count() > 0: + pixel_format = self.data.tagdata.bitmaps.bitmaps_array[0].format.enum_name + if pixel_format in ("a8", "y8", "ay8", "a8y8"): + top_format = "monochrome" + elif pixel_format in ("r5g6b5", "a1r5g5b5", "a4r4g4b4"): + top_format = "color_16bit" + elif pixel_format in ("x8r8g8b8", "a8r8g8b8", "p8_bump"): + top_format = "color_32bit" + elif pixel_format == "dxt1": + top_format = "color_key_transparency" + elif pixel_format == "dxt3": + top_format = "explicit_alpha" + elif pixel_format == "dxt5": + top_format = "interpolated_alpha" self.data.tagdata.format.set_to(top_format) @@ -309,7 +326,7 @@ def get_bitmap_size(self, b_index): "Arbytmap is not loaded. Cannot get bitmap size.") w, h, d, = self.bitmap_width_height_depth(b_index) - fmt = FORMAT_NAME_MAP[self.bitmap_format(b_index)] + fmt = self.format_name_map[self.bitmap_format(b_index)] bytes_count = 0 for mipmap in range(self.bitmap_mipmaps_count(b_index) + 1): @@ -379,7 +396,7 @@ def sanitize_bitmaps(self): tex_infos = self.tex_infos for i in range(self.bitmap_count()): - format = FORMAT_NAME_MAP[self.bitmap_format(i)] + format = self.format_name_map[self.bitmap_format(i)] flags = self.bitmap_flags(i) old_w, old_h, _ = self.bitmap_width_height_depth(i) @@ -409,7 +426,7 @@ def parse_bitmap_blocks(self): tex_infos = self.tex_infos = [] # this is the block that will hold all of the bitmap blocks - root_tex_block = self.definition.subdefs['pixel_root'].build() + root_tex_block = self.pixel_root_definition.build() is_xbox = self.is_xbox_bitmap get_mip_dims = ab.get_mipmap_dimensions @@ -420,7 +437,7 @@ def parse_bitmap_blocks(self): # since we need this information to read the bitmap we extract it mw, mh, md, = self.bitmap_width_height_depth(i) type = self.bitmap_type(i) - format = FORMAT_NAME_MAP[self.bitmap_format(i)] + format = self.format_name_map[self.bitmap_format(i)] mipmap_count = self.bitmap_mipmaps_count(i) + 1 sub_bitmap_count = ab.SUB_BITMAP_COUNTS[TYPE_NAME_MAP[type]] @@ -481,3 +498,35 @@ def parse_bitmap_blocks(self): # it's easier to work with bitmaps in one format so # we'll switch the mipmaps from XBOX to PC ordering self.change_sub_bitmap_ordering(False) + + def set_swizzled(self, swizzle): + '''Swizzles or unswizzles all applicable bitmap formats.''' + if ab is None: + raise NotImplementedError("Arbytmap is not loaded. Cannot swizzle.") + + pixel_data = self.data.tagdata.processed_pixel_data.data + bitmaps = self.data.tagdata.bitmaps.bitmaps_array + swizzler = ab.swizzler.Swizzler(mask_type="MORTON") + for i in range(len(bitmaps)): + bitmap = bitmaps[i] + format = self.format_name_map[bitmap.format.data] + if format not in self.swizzleable_formats: + bitmap.flags.swizzled = False + continue + elif bitmap.flags.swizzled == bool(swizzle): + continue + + w, h, d = bitmap.width, bitmap.height, bitmap.depth + faces = 6 if bitmap.type.data == TYPE_CUBEMAP else 1 + mipmaps = bitmap.mipmaps + 1 + + for f in range(faces): + for m in range(mipmaps): + j = m*faces + f + pixel_data[i][j] = swizzler.swizzle_single_array( + pixel_data[i][j], swizzle, 1, + *ab.get_mipmap_dimensions(w, h, d, m), + False + ) + + bitmap.flags.swizzled = swizzle \ No newline at end of file diff --git a/reclaimer/hek/defs/objs/deca.py b/reclaimer/hek/defs/objs/deca.py new file mode 100644 index 00000000..4112a546 --- /dev/null +++ b/reclaimer/hek/defs/objs/deca.py @@ -0,0 +1,17 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.hek.defs.objs.tag import HekTag + +class DecaTag(HekTag): + + def calc_internal_data(self): + HekTag.calc_internal_data(self) + # why is this hardcoded, but exposed to the user? + self.data.tagdata.maximum_sprite_extend = 16.0 diff --git a/reclaimer/hek/defs/objs/effe.py b/reclaimer/hek/defs/objs/effe.py index 84cee398..c2aa7aa5 100644 --- a/reclaimer/hek/defs/objs/effe.py +++ b/reclaimer/hek/defs/objs/effe.py @@ -9,28 +9,31 @@ from reclaimer.hek.defs.objs.tag import HekTag from reclaimer.util.matrices import euler_2d_to_vector_3d -#from reclaimer.common_descs import valid_objects +from reclaimer.enums import object_types +from reclaimer.util import fourcc_to_int + +object_tag_class_ids = tuple( + fourcc_to_int(fourcc, byteorder='big') for fourcc in object_types + ) class EffeTag(HekTag): def calc_internal_data(self): HekTag.calc_internal_data(self) - never_cull = False + dont_cull = False for event in self.data.tagdata.events.STEPTREE: for part in event.parts.STEPTREE: - if part.type.tag_class.enum_name == 'light': - never_cull = True - - part.effect_class = part.type.tag_class - - #TODO: There is no good way to do this right now - #object_types = valid_objects('b').desc[0]['NAME_MAP'].keys() - #if part.effect_class.enum_name in object_types: - # part.effect_class.enum_name = 'object' + tag_cls = part.type.tag_class + if tag_cls.data in object_tag_class_ids: + dont_cull = True + part.effect_class.set_to('object') + elif tag_cls.enum_name in ("damage_effect", "light"): + dont_cull = True + part.effect_class.set_to(tag_cls.enum_name) for particle in event.particles.STEPTREE: particle.relative_direction_vector[:] = euler_2d_to_vector_3d( *particle.relative_direction ) - self.data.tagdata.flags.never_cull = never_cull + self.data.tagdata.flags.must_be_deterministic = dont_cull diff --git a/reclaimer/hek/defs/objs/mgs2.py b/reclaimer/hek/defs/objs/mgs2.py new file mode 100644 index 00000000..ef19732d --- /dev/null +++ b/reclaimer/hek/defs/objs/mgs2.py @@ -0,0 +1,21 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.hek.defs.objs.tag import HekTag + +class Mgs2Tag(HekTag): + + def calc_internal_data(self): + HekTag.calc_internal_data(self) + for frame in self.data.tagdata.frames.STEPTREE: + if frame.offset_exponent <= 0: + frame.offset_exponent = 1.0 + + if frame.radius_exponent <= 0: + frame.radius_exponent = 1.0 \ No newline at end of file diff --git a/reclaimer/hek/defs/objs/mod2.py b/reclaimer/hek/defs/objs/mod2.py index c264ecdb..ea0a6341 100644 --- a/reclaimer/hek/defs/objs/mod2.py +++ b/reclaimer/hek/defs/objs/mod2.py @@ -11,45 +11,6 @@ class Mod2Tag(ModeTag): - def globalize_local_markers(self): - tagdata = self.data.tagdata - all_global_markers = tagdata.markers.STEPTREE - all_global_markers_by_name = {b.name: b.marker_instances.STEPTREE - for b in all_global_markers} - - for i in range(len(tagdata.regions.STEPTREE)): - region = tagdata.regions.STEPTREE[i] - for j in range(len(region.permutations.STEPTREE)): - perm = region.permutations.STEPTREE[j] - - for k in range(len(perm.local_markers.STEPTREE)): - local_marker = perm.local_markers.STEPTREE[k] - global_markers = all_global_markers_by_name.get( - local_marker.name, None) - - if global_markers is None or len(global_markers) >= 32: - all_global_markers.append() - all_global_markers[-1].name = local_marker.name - global_markers = all_global_markers[-1].marker_instances.STEPTREE - all_global_markers_by_name[local_marker.name] = global_markers - - global_markers.append() - global_marker = global_markers[-1] - - global_marker.region_index = i - global_marker.permutation_index = j - global_marker.node_index = local_marker.node_index - global_marker.rotation[:] = local_marker.rotation[:] - global_marker.translation[:] = local_marker.translation[:] - - del perm.local_markers.STEPTREE[:] - - # sort the markers how Halo's picky ass wants them - name_map = {all_global_markers[i].name: i - for i in range(len(all_global_markers))} - all_global_markers[:] = list(all_global_markers[name_map[name]] - for name in sorted(name_map)) - def delocalize_part_nodes(self, geometry_index, part_index): part = self.data.tagdata.geometries.STEPTREE\ [geometry_index].parts.STEPTREE[part_index] diff --git a/reclaimer/hek/defs/objs/mode.py b/reclaimer/hek/defs/objs/mode.py index a7d98cf6..83fa2091 100644 --- a/reclaimer/hek/defs/objs/mode.py +++ b/reclaimer/hek/defs/objs/mode.py @@ -11,20 +11,19 @@ from struct import unpack, pack_into from types import MethodType +from reclaimer.constants import LOD_NAMES from reclaimer.hek.defs.objs.tag import HekTag from reclaimer.util.compression import compress_normal32, decompress_normal32 from reclaimer.util.matrices import quaternion_to_matrix, Matrix -# TODO: Make calc_internal_data recalculate the lod nodes, and remove that -# same function from model.model_compilation.compile_gbxmodel and replace -# it with a call to calc_internal_data. lod nodes are recalculated when -# tags are compiled into maps, but the functionality should still be here. + class ModeTag(HekTag): def calc_internal_data(self): ''' For each node, this method recalculates the rotation matrix - from the quaternion, and the translation to the root bone. + from the quaternion, the translation to the root bone, and + the lod nodes. ''' HekTag.calc_internal_data(self) @@ -58,6 +57,125 @@ def calc_internal_data(self): node.rot_kk_ii[:] = rotation[1] node.rot_ii_jj[:] = rotation[2] + # calculate the highest node used by each geometry + geom_max_node_indices = [] + node_count = len(nodes) + for geometry in self.data.tagdata.geometries.STEPTREE: + geom_node_count = 0 + for part in geometry.parts.STEPTREE: + if geom_node_count == node_count: + break + elif getattr(part.flags, "ZONER", False): + # don't need to check every vert when they're all right here + geom_node_count = max(( + geom_node_count, + *(v for v in part.local_nodes[:part.local_node_count]) + )) + continue + + is_comp = not part.uncompressed_vertices.STEPTREE + verts = ( + part.compressed_vertices.STEPTREE if is_comp else + part.uncompressed_vertices.STEPTREE + ) + curr_highest = 0 + max_highest = (node_count - 1) * (3 if is_comp else 1) + if isinstance(verts, (bytes, bytearray)): + # verts are packed, so unpack what we need from it + vert_size = 32 if is_comp else 68 # vert size in bytes + unpack_vert = ( + MethodType(unpack, ">28x bbh") if is_comp else + MethodType(unpack, ">56x hhf 4x") + ) + # lazy unpack JUST the indices and weight + verts = [ + unpack_vert(verts[i: i+vert_size]) + for i in range(0, len(verts), vert_size) + ] + node_0_key, node_1_key, weight_key = 0, 1, 2 + else: + # verts aren't packed, so use as-is + weight_key = "node_0_weight" + node_0_key, node_1_key = "node_0_index", "node_1_index" + + for vert in verts: + node_0_weight = vert[weight_key] + if node_0_weight > 0 and vert[node_0_key] > curr_highest: + curr_highest = vert[node_0_key] + if curr_highest == max_highest: break + + if node_0_weight < 1 and vert[node_1_key] > curr_highest: + curr_highest = vert[node_1_key] + if curr_highest == max_highest: break + + if is_comp: + # compressed nodes use indices multiplied by 3 for some reason + curr_highest //= 3 + + geom_node_count = max(geom_node_count, curr_highest) + + geom_max_node_indices.append(max(0, geom_node_count)) + + # calculate the highest node for each lod + max_lod_nodes = {lod: 0 for lod in LOD_NAMES} + for region in self.data.tagdata.regions.STEPTREE: + for perm in region.permutations.STEPTREE: + for lod_name in LOD_NAMES: + try: + highest_node_count = geom_max_node_indices[ + perm["%s_geometry_block" % lod_name] + ] + except IndexError: + continue + + max_lod_nodes[lod_name] = max( + max_lod_nodes[lod_name], + highest_node_count + ) + + # set the node counts per lod + for lod, highest_node in max_lod_nodes.items(): + self.data.tagdata["%s_lod_nodes" % lod] = max(0, highest_node) + + def globalize_local_markers(self): + tagdata = self.data.tagdata + all_global_markers = tagdata.markers.STEPTREE + all_global_markers_by_name = {b.name: b.marker_instances.STEPTREE + for b in all_global_markers} + + for i in range(len(tagdata.regions.STEPTREE)): + region = tagdata.regions.STEPTREE[i] + for j in range(len(region.permutations.STEPTREE)): + perm = region.permutations.STEPTREE[j] + + for k in range(len(perm.local_markers.STEPTREE)): + local_marker = perm.local_markers.STEPTREE[k] + global_markers = all_global_markers_by_name.get( + local_marker.name, None) + + if global_markers is None or len(global_markers) >= 32: + all_global_markers.append() + all_global_markers[-1].name = local_marker.name + global_markers = all_global_markers[-1].marker_instances.STEPTREE + all_global_markers_by_name[local_marker.name] = global_markers + + global_markers.append() + global_marker = global_markers[-1] + + global_marker.region_index = i + global_marker.permutation_index = j + global_marker.node_index = local_marker.node_index + global_marker.rotation[:] = local_marker.rotation[:] + global_marker.translation[:] = local_marker.translation[:] + + del perm.local_markers.STEPTREE[:] + + # sort the markers how Halo's picky ass wants them + name_map = {all_global_markers[i].name: i + for i in range(len(all_global_markers))} + all_global_markers[:] = list(all_global_markers[name_map[name]] + for name in sorted(name_map)) + def compress_part_verts(self, geometry_index, part_index): part = self.data.tagdata.geometries.STEPTREE\ [geometry_index].parts.STEPTREE[part_index] @@ -70,6 +188,8 @@ def compress_part_verts(self, geometry_index, part_index): comp_verts = bytearray(b'\x00' * 32 * uncomp_verts_reflexive.size) uncomp_verts = uncomp_verts_reflexive.STEPTREE + if not isinstance(uncomp_verts, (bytes, bytearray)): + raise ValueError("Error: Uncompressed vertices must be in raw, unpacked form.") in_off = out_off = 0 # compress each of the verts and write them to the buffer @@ -105,6 +225,8 @@ def decompress_part_verts(self, geometry_index, part_index): uncomp_verts = bytearray(b'\x00' * 68 * comp_verts_reflexive.size) comp_verts = comp_verts_reflexive.STEPTREE + if not isinstance(comp_verts, (bytes, bytearray)): + raise ValueError("Error: Compressed vertices must be in raw, unpacked form.") in_off = out_off = 0 # uncompress each of the verts and write them to the buffer diff --git a/reclaimer/hek/defs/objs/obje.py b/reclaimer/hek/defs/objs/obje.py index 0adcac71..db9db624 100644 --- a/reclaimer/hek/defs/objs/obje.py +++ b/reclaimer/hek/defs/objs/obje.py @@ -21,7 +21,8 @@ def calc_internal_data(self): self.ext = '.' + full_class_name self.filepath = os.path.splitext(str(self.filepath))[0] + self.ext - object_type = self.data.tagdata.obje_attrs.object_type + obje_attrs = self.data.tagdata.obje_attrs + object_type = obje_attrs.object_type if full_class_name == "object": object_type.data = -1 elif full_class_name == "biped": @@ -50,3 +51,11 @@ def calc_internal_data(self): object_type.data = 11 else: raise ValueError("Unknown object type '%s'" % full_class_name) + + # normalize color change weights + for cc in obje_attrs.change_colors.STEPTREE: + perms = cc.permutations.STEPTREE + total_weight = sum(max(0, perm.weight) for perm in perms) + total_weight = total_weight or len(perms) + for perm in perms: + perm.weight = (max(0, perm.weight) or 1) / total_weight \ No newline at end of file diff --git a/reclaimer/hek/defs/objs/part.py b/reclaimer/hek/defs/objs/part.py new file mode 100644 index 00000000..4f720b2a --- /dev/null +++ b/reclaimer/hek/defs/objs/part.py @@ -0,0 +1,18 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.hek.defs.objs.tag import HekTag + +class PartTag(HekTag): + + def calc_internal_data(self): + HekTag.calc_internal_data(self) + + # crash if this is nonzero? + self.data.tagdata.rendering.contact_deterioration = 0.0 \ No newline at end of file diff --git a/reclaimer/hek/defs/objs/scnr.py b/reclaimer/hek/defs/objs/scnr.py new file mode 100644 index 00000000..e7a1cd65 --- /dev/null +++ b/reclaimer/hek/defs/objs/scnr.py @@ -0,0 +1,37 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +import os +import traceback + +from reclaimer.hek.defs.objs.tag import HekTag +from reclaimer.halo_script.hsc import HSC_IS_SCRIPT_OR_GLOBAL,\ + get_hsc_data_block, clean_script_syntax_nodes + +class ScnrTag(HekTag): + # used to determine what syntax node block to use when parsing script data + engine = "halo1ce" + + def calc_internal_data(self): + HekTag.calc_internal_data(self) + self.clean_script_syntax_data() + + def clean_script_syntax_data(self): + try: + script_syntax_data = self.data.tagdata.script_syntax_data + script_nodes = get_hsc_data_block( + script_syntax_data.data, self.engine + ) + clean_script_syntax_nodes(script_nodes) + + # replace the sanitized data + script_syntax_data.data = script_nodes.serialize() + except Exception: + print(traceback.format_exc()) + print("Failed to sanitize script syntax data nodes.") \ No newline at end of file diff --git a/reclaimer/hek/defs/objs/snd_.py b/reclaimer/hek/defs/objs/snd_.py index 1dd5a730..f42b35d2 100644 --- a/reclaimer/hek/defs/objs/snd_.py +++ b/reclaimer/hek/defs/objs/snd_.py @@ -8,13 +8,91 @@ # from reclaimer.hek.defs.objs.tag import HekTag +from reclaimer.sounds import constants as sound_const, ogg class Snd_Tag(HekTag): def calc_internal_data(self): HekTag.calc_internal_data(self) + tagdata = self.data.tagdata - for pitch_range in self.data.tagdata.pitch_ranges.STEPTREE: + bytes_per_sample = sound_const.channel_counts.get( + tagdata.encoding.data, 1 + ) * 2 + for pitch_range in tagdata.pitch_ranges.STEPTREE: pitch_range.playback_rate = 1 if pitch_range.natural_pitch: pitch_range.playback_rate = 1 / pitch_range.natural_pitch + + # ensure buffer sizes are correct + for perm in pitch_range.permutations.STEPTREE: + if perm.compression.enum_name == "none": + perm.buffer_size = len(perm.samples) + elif perm.compression.enum_name == "ogg": + # oggvorbis NEEDS this set for proper playback ingame + perm.buffer_size = ( + ogg.get_ogg_pcm_sample_count(perm.samples.data) + if sound_const.OGGVORBIS_AVAILABLE else + # oh well. default to whatever it's set to + (perm.buffer_size // bytes_per_sample) + ) * bytes_per_sample + + # default sound min and max distance by class + sound_class = tagdata.sound_class.enum_name + distance_defaults = None + if sound_class in ( + "device_door", "device_force_field", "device_machinery", + "device_nature", "music", "ambient_nature", "ambient_machinery" + ): + distance_defaults = (0.9, 5.0) + elif sound_class in ( + "weapon_ready", "weapon_reload", "weapon_empty", + "weapon_charge", "weapon_overheat", "weapon_idle" + ): + distance_defaults = (1.0, 9.0) + elif sound_class in ( + "unit_dialog", "scripted_dialog_player", + "scripted_dialog_other", + "scripted_dialog_force_unspatialized", "game_event" + ): + distance_defaults = (3.0, 20.0) + elif sound_class in ( + "object_impacts", "particle_impacts", "slow_particle_impacts", + "device_computers", "ambient_computers", "first_person_damage", + ): + distance_defaults = (0.5, 3.0) + elif sound_class in ( + "projectile_impact", "vehicle_collision", "vehicle_engine" + ): + distance_defaults = (1.4, 8.0) + elif sound_class == "weapon_fire": + distance_defaults = (4.0, 70.0) + elif sound_class == "scripted_effect": + distance_defaults = (2.0, 5.0) + elif sound_class == "projectile_detonation": + distance_defaults = (8.0, 120.0) + elif sound_class == "unit_footsteps": + distance_defaults = (0.9, 10.0) + + zero_gain_modifier_default = 1.0 + if sound_class in ( + "object_impacts", "particle_impacts", "slow_particle_impacts", + "unit_dialog", "music", "ambient_nature", "ambient_machinery", + "ambient_computers", "scripted_dialog_player", + "scripted_effect", "scripted_dialog_other", + "scripted_dialog_force_unspatialized" + ): + zero_gain_modifier_default = 0.0 + + if distance_defaults: + if not tagdata.minimum_distance: + tagdata.minimum_distance = distance_defaults[0] + + if not tagdata.maximum_distance: + tagdata.maximum_distance = distance_defaults[1] + + if (not tagdata.modifiers_when_scale_is_zero.gain and + not tagdata.modifiers_when_scale_is_one.gain + ): + tagdata.modifiers_when_scale_is_zero.gain = zero_gain_modifier_default + tagdata.modifiers_when_scale_is_one.gain = 1.0 diff --git a/reclaimer/hek/defs/part.py b/reclaimer/hek/defs/part.py index 9ee519f7..0b1a86f5 100644 --- a/reclaimer/hek/defs/part.py +++ b/reclaimer/hek/defs/part.py @@ -8,7 +8,7 @@ # from ...common_descs import * -from .objs.tag import HekTag +from .objs.part import PartTag from supyr_struct.defs.tag_def import TagDef part_body = Struct("tagdata", @@ -50,7 +50,7 @@ Float("to", UNIT_SCALE=per_sec_unit_scale), ORIENT='h', SIDETIP='frames/sec' ), - Float("contact_deterioration"), + Float("contact_deterioration", VISIBLE=False, DEFAULT=0.0), Float("fade_start_size", SIDETIP="pixels"), Float("fade_end_size", SIDETIP="pixels"), @@ -60,7 +60,8 @@ SInt16("looping_sequence_count"), SInt16("final_sequence_count"), - Pad(12), + Pad(8), + FlFloat("sprite_size", VISIBLE=False), SEnum16("orientation", *render_mode), Pad(38), @@ -101,5 +102,5 @@ def get(): blam_header("part", 2), part_body, - ext=".particle", endian=">", tag_cls=HekTag, + ext=".particle", endian=">", tag_cls=PartTag, ) diff --git a/reclaimer/hek/defs/plac.py b/reclaimer/hek/defs/plac.py index d9befd91..9c1ad5a5 100644 --- a/reclaimer/hek/defs/plac.py +++ b/reclaimer/hek/defs/plac.py @@ -9,15 +9,8 @@ from .obje import * from .objs.obje import ObjeTag -from supyr_struct.defs.tag_def import TagDef -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(10)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "plac") plac_body = Struct("tagdata", obje_attrs, SIZE=508, diff --git a/reclaimer/hek/defs/proj.py b/reclaimer/hek/defs/proj.py index baf1f83b..1f73c4d3 100644 --- a/reclaimer/hek/defs/proj.py +++ b/reclaimer/hek/defs/proj.py @@ -9,13 +9,8 @@ from .obje import * from .objs.obje import ObjeTag -from supyr_struct.util import desc_variant -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(5)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "proj") responses = ( "disappear", @@ -25,6 +20,18 @@ "attach" ) +# split out to be reused in mcc_hek +potential_response = Struct("potential_response", + SEnum16('response', *responses), + Bool16("flags", + "only_against_units", + ), + float_zero_to_one("skip_fraction"), + from_to_rad("impact_angle"), # radians + from_to_wu_sec("impact_velocity"), # world units/second + dependency('effect', "effe"), + ) + material_response = Struct("material_response", Bool16("flags", "cannot_be_overpenetrated", @@ -33,16 +40,7 @@ dependency('effect', "effe"), Pad(16), - Struct("potential_response", - SEnum16('response', *responses), - Bool16("flags", - "only_against_units", - ), - float_zero_to_one("skip_fraction"), - from_to_rad("impact_angle"), # radians - from_to_wu_sec("impact_velocity"), # world units/second - dependency('effect', "effe"), - ), + potential_response, Pad(16), SEnum16("scale_effects_by", diff --git a/reclaimer/hek/defs/rain.py b/reclaimer/hek/defs/rain.py index 39372bda..4e4132c5 100644 --- a/reclaimer/hek/defs/rain.py +++ b/reclaimer/hek/defs/rain.py @@ -53,10 +53,11 @@ Pad(32), QStruct("color_lower_bound", INCLUDE=argb_float), QStruct("color_upper_bound", INCLUDE=argb_float), + FlFloat("sprite_size", VISIBLE=False), #Shader Struct("shader", - Pad(64), + Pad(60), dependency("sprite_bitmap", "bitm"), SEnum16("render_mode", *render_mode), SEnum16("render_direction_source", @@ -64,7 +65,8 @@ "from_acceleration" ), - Pad(40), + Pad(36), + FlUInt32("unknown", VISIBLE=False), Bool16("shader_flags", *shader_flags), SEnum16("framebuffer_blend_function", *framebuffer_blend_functions), SEnum16("framebuffer_fade_mode", *render_fade_mode), diff --git a/reclaimer/hek/defs/sbsp.py b/reclaimer/hek/defs/sbsp.py index 42d29085..9d610f5e 100644 --- a/reclaimer/hek/defs/sbsp.py +++ b/reclaimer/hek/defs/sbsp.py @@ -17,6 +17,13 @@ "Add 0x8000 to get fog index." ) +# calculated when compiled into map +material_type_cache_enum = FlSEnum16("material_type", + *(tuple((materials_list[i], i) for i in + range(len(materials_list))) + (("NONE", -1), )), + VISIBLE=False + ) + # the order is an array of vertices first, then an array of lightmap vertices. # @@ -52,8 +59,8 @@ # this normal is the direction the light is hitting from, and # is used for calculating dynamic shadows on dynamic objects UInt32('normal'), - SInt16('u', UNIT_SCALE=1/32767, MIN=-32767, WIDGET_WIDTH=10), - SInt16('v', UNIT_SCALE=1/32767, MIN=-32767, WIDGET_WIDTH=10), + SInt16('u', UNIT_SCALE=1/SINT16_MAX, MIN=-SINT16_MAX, WIDGET_WIDTH=10), + SInt16('v', UNIT_SCALE=1/SINT16_MAX, MIN=-SINT16_MAX, WIDGET_WIDTH=10), SIZE=8 ) @@ -69,7 +76,8 @@ collision_material = Struct("collision_material", dependency("shader", valid_shaders), - FlUInt32("unknown", VISIBLE=False), + Pad(2), + material_type_cache_enum, SIZE=20 ) @@ -143,7 +151,9 @@ QStruct("shadow_color", INCLUDE=rgb_float), QStruct("plane", INCLUDE=plane), SInt16("breakable_surface", EDITABLE=False), - Pad(6), + Pad(2), + UEnum8("vertex_type", *H1_VERTEX_BUFFER_TYPES, VISIBLE=False), + Pad(3), SInt32("vertices_count", EDITABLE=False), SInt32("vertices_offset", EDITABLE=False, VISIBLE=False), @@ -155,13 +165,8 @@ "bspmagic relative pointer to the vertices."), VISIBLE=False ), - FlUEnum16("vertex_type", # name is a guess - ("unknown", 0), - ("uncompressed", 2), - ("compressed", 3), - VISIBLE=False, - ), - Pad(2), + UEnum8("lightmap_vertex_type", *H1_VERTEX_BUFFER_TYPES, VISIBLE=False), + Pad(3), SInt32("lightmap_vertices_count", EDITABLE=False), SInt32("lightmap_vertices_offset", EDITABLE=False, VISIBLE=False), @@ -250,13 +255,14 @@ # almost certain this is padding, though a value in the third # and fourth bytes is non-zero in meta, but not in a tag, so idk. + # also this is a bunch of floats in stubbs, so there's also that. Pad(24), reflexive("predicted_resources", predicted_resource, 1024, VISIBLE=False), reflexive("subclusters", subcluster, 4096), SInt16("first_lens_flare_marker_index"), SInt16("lens_flare_marker_count"), - reflexive("surface_indices", surface_index, 32768), + reflexive("surface_indices", surface_index, SINT16_INDEX_MAX), reflexive("mirrors", mirror, 16, DYN_NAME_PATH=".shader.filepath"), reflexive("portals", portal, 128), SIZE=104 @@ -289,10 +295,7 @@ fog_plane = Struct("fog_plane", SInt16("front_region"), - FlSEnum16("material_type", - *(tuple((materials_list[i], i) for i in - range(len(materials_list))) + (("NONE", -1), )), - VISIBLE=False), # calculated when compiled into map + material_type_cache_enum, QStruct("plane", INCLUDE=plane), reflexive("vertices", vertex, 4096), SIZE=32 @@ -397,7 +400,7 @@ detail_object = Struct("detail_object", reflexive("cells", detail_object_cell, 262144), reflexive("instances", detail_object_instance, 2097152), - reflexive("counts", detail_object_count, 8388608), + reflexive("counts", detail_object_count, SINT24_INDEX_MAX), reflexive("z_reference_vectors", detail_object_z_reference_vector, 262144), Bool8("flags", "enabled", # required to be set on map compile. @@ -420,8 +423,8 @@ ) leaf_map_leaf = Struct("leaf_map_leaf", - reflexive("faces", face, 256), - reflexive("portal_indices", portal_index, 256), + reflexive("faces", face, UINT8_INDEX_MAX), + reflexive("portal_indices", portal_index, UINT8_INDEX_MAX), SIZE=24 ) @@ -467,23 +470,23 @@ QStruct("world_bounds_x", INCLUDE=from_to), QStruct("world_bounds_y", INCLUDE=from_to), QStruct("world_bounds_z", INCLUDE=from_to), - reflexive("leaves", leaf, 65535), + reflexive("leaves", leaf, UINT16_INDEX_MAX), reflexive("leaf_surfaces", leaf_surface, 262144), reflexive("surfaces", surface, 131072), reflexive("lightmaps", lightmap, 128), Pad(12), - reflexive("lens_flares", lens_flare, 256, + reflexive("lens_flares", lens_flare, UINT8_INDEX_MAX, DYN_NAME_PATH='.shader.filepath'), - reflexive("lens_flare_markers", lens_flare_marker, 65535), + reflexive("lens_flare_markers", lens_flare_marker, UINT16_INDEX_MAX), reflexive("clusters", cluster, 8192), # this is an array of 8 byte structs for each cluster - rawdata_ref("cluster_data", max_size=65536), + rawdata_ref("cluster_data", max_size=UINT16_INDEX_MAX), reflexive("cluster_portals", cluster_portal, 512), Pad(12), - reflexive("breakable_surfaces", breakable_surface, 256), + reflexive("breakable_surfaces", breakable_surface, UINT8_INDEX_MAX), reflexive("fog_planes", fog_plane, 32), reflexive("fog_regions", fog_region, 32), reflexive("fog_palettes", fog_palette, 32, @@ -513,7 +516,7 @@ reflexive("runtime_decals", runtime_decal, 6144, VISIBLE=False), Pad(12), - reflexive("leaf_map_leaves", leaf_map_leaf, 65536, VISIBLE=False), + reflexive("leaf_map_leaves", leaf_map_leaf, UINT16_INDEX_MAX, VISIBLE=False), reflexive("leaf_map_portals", leaf_map_portal, 524288, VISIBLE=False), SIZE=648, ) @@ -521,11 +524,11 @@ fast_sbsp_body = desc_variant(sbsp_body, ("collision_bsp", reflexive("collision_bsp", fast_collision_bsp, 1)), ("nodes", raw_reflexive("nodes", node, 131072)), - ("leaves", raw_reflexive("leaves", leaf, 65535)), + ("leaves", raw_reflexive("leaves", leaf, UINT16_INDEX_MAX)), ("leaf_surfaces", raw_reflexive("leaf_surfaces", leaf_surface, 262144)), ("surfaces", raw_reflexive("surface", surface, 131072)), - ("lens_flare_markers", raw_reflexive("lens_flare_markers", lens_flare_marker, 65535)), - ("breakable_surfaces", raw_reflexive("breakable_surfaces", breakable_surface, 256)), + ("lens_flare_markers", raw_reflexive("lens_flare_markers", lens_flare_marker, UINT16_INDEX_MAX)), + ("breakable_surfaces", raw_reflexive("breakable_surfaces", breakable_surface, UINT8_INDEX_MAX)), ("pathfinding_surfaces", raw_reflexive("pathfinding_surfaces", pathfinding_surface, 131072)), ("pathfinding_edges", raw_reflexive("pathfinding_edges", pathfinding_edge, 262144)), ("markers", raw_reflexive("markers", marker, 1024, DYN_NAME_PATH='.name')), diff --git a/reclaimer/hek/defs/scen.py b/reclaimer/hek/defs/scen.py index 24238809..87c4513f 100644 --- a/reclaimer/hek/defs/scen.py +++ b/reclaimer/hek/defs/scen.py @@ -9,15 +9,8 @@ from .obje import * from .objs.obje import ObjeTag -from supyr_struct.defs.tag_def import TagDef -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(6)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "scen") scen_body = Struct("tagdata", obje_attrs, SIZE=508, diff --git a/reclaimer/hek/defs/scnr.py b/reclaimer/hek/defs/scnr.py index 78f11b6f..9e5b4f10 100644 --- a/reclaimer/hek/defs/scnr.py +++ b/reclaimer/hek/defs/scnr.py @@ -8,9 +8,21 @@ # from ...common_descs import * -from .objs.tag import HekTag +from .objs.scnr import ScnrTag from supyr_struct.defs.tag_def import TagDef from supyr_struct.util import desc_variant +from reclaimer.misc.defs.recorded_animations import build_r_a_stream_block + + +def compute_decompiled_ra_stream(parent=None, **kwargs): + try: + if parent is not None: + parent.decompiled_stream = build_r_a_stream_block( + parent.parent.unit_control_data_version, + parent.parent.recorded_animation_event_stream.STEPTREE + ) + except Exception: + pass def object_reference(name, *args, **kwargs): @@ -42,47 +54,6 @@ def object_swatch(name, def_id, size=48): SIZE=size ) -fl_float_xyz = QStruct("", - FlFloat("x"), - FlFloat("y"), - FlFloat("z"), - ORIENT="h" - ) - -stance_flags = FlBool16("stance", - "walk", - "look_only", - "primary_fire", - "secondary_fire", - "jump", - "crouch", - "melee", - "flashlight", - "action1", - "action2", - "action_hold", - ) - -unit_control_packet = Struct("unit_control_packet", - - ) - -r_a_stream_header = Struct("r_a_stream_header", - UInt8("move_index", DEFAULT=3, MAX=6), - UInt8("bool_index"), - stance_flags, - FlSInt16("weapon", DEFAULT=-1), - QStruct("speed", FlFloat("x"), FlFloat("y"), ORIENT="h"), - QStruct("feet", INCLUDE=fl_float_xyz), - QStruct("body", INCLUDE=fl_float_xyz), - QStruct("head", INCLUDE=fl_float_xyz), - QStruct("change", INCLUDE=fl_float_xyz), - FlUInt16("unknown1"), - FlUInt16("unknown2"), - FlUInt16("unknown3", DEFAULT=0xFFFF), - FlUInt16("unknown4", DEFAULT=0xFFFF), - SIZE=60 - ) device_flags = ( "initially_open", # value of 1.0 @@ -235,11 +206,21 @@ def object_swatch(name, def_id, size=48): SIZE=36 ) +# 8 bytes of padding, with the 5th byte being defined separately +# so it can be replaced with the appearance_player_index in mcc_hek +_object_ref_pad_fields = tuple( + Pad(n) for n in (2, 2, 1, 3) + ) + # Object references -scenery = object_reference("scenery", SIZE=72, block_name="sceneries") +scenery = object_reference("scenery", + *_object_ref_pad_fields, + SIZE=72, block_name="sceneries" + ) biped = object_reference("biped", - Pad(40), + *_object_ref_pad_fields, + Pad(32), float_zero_to_one("body_vitality"), Bool32("flags", "dead", @@ -248,7 +229,8 @@ def object_swatch(name, def_id, size=48): ) vehicle = object_reference("vehicle", - Pad(40), + *_object_ref_pad_fields, + Pad(32), float_zero_to_one("body_vitality"), Bool32("flags", "dead", @@ -285,11 +267,14 @@ def object_swatch(name, def_id, size=48): "obsolete", {NAME: "can_accelerate", GUI_NAME:"moves due to explosions"}, ), + Pad(1), # replaced with appearance_player_index in mcc_hek + Pad(3), SIZE=40 ) weapon = object_reference("weapon", - Pad(40), + *_object_ref_pad_fields, + Pad(32), SInt16("rounds_left"), SInt16("rounds_loaded"), Bool16("flags", @@ -310,7 +295,7 @@ def object_swatch(name, def_id, size=48): ) machine = object_reference("machine", - Pad(8), + *_object_ref_pad_fields, dyn_senum16("power_group", DYN_NAME_PATH=".....device_groups.STEPTREE[DYN_I].name"), dyn_senum16("position_group", @@ -326,7 +311,7 @@ def object_swatch(name, def_id, size=48): ) control = object_reference("control", - Pad(8), + *_object_ref_pad_fields, dyn_senum16("power_group", DYN_NAME_PATH=".....device_groups.STEPTREE[DYN_I].name"), dyn_senum16("position_group", @@ -340,7 +325,10 @@ def object_swatch(name, def_id, size=48): ) light_fixture = object_reference("light_fixture", - Pad(8), + FlUInt16("bsp_indices_mask", VISIBLE=False), + Pad(2), + Pad(1), + Pad(3), dyn_senum16("power_group", DYN_NAME_PATH=".....device_groups.STEPTREE[DYN_I].name"), dyn_senum16("position_group", @@ -353,7 +341,10 @@ def object_swatch(name, def_id, size=48): SIZE=88 ) -sound_scenery = object_reference("sound_scenery", SIZE=40, block_name="sound_sceneries") +sound_scenery = object_reference("sound_scenery", + *_object_ref_pad_fields, + SIZE=40, block_name="sound_sceneries" + ) # Object swatches scenery_swatch = object_swatch("scenery_swatch", "scen") @@ -378,6 +369,8 @@ def object_swatch(name, def_id, size=48): SInt16("secondary_rounds_total"), SInt8("starting_frag_grenade_count", MIN=0), SInt8("starting_plasma_grenade_count", MIN=0), + Pad(1), # replaced with starting_grenade_type2_count in mcc_hek + Pad(1), # replaced with starting_grenade_type3_count in mcc_hek SIZE=104 ) @@ -434,7 +427,19 @@ def object_swatch(name, def_id, size=48): SInt16("length_of_animation", SIDETIP="ticks"), # ticks Pad(6), rawdata_ref("recorded_animation_event_stream", max_size=2097152), - SIZE=64 + Pad(0), + SIZE=64, + ) + +recorded_animation_with_ra_stream = desc_variant(recorded_animation, + ("pad_8", QStruct("decompiled_stream", + # NOTE: making this a steptree to ensure the data in the + # recorded_animation_event_stream is parsed and available + STEPTREE=Computed("decompiled_stream", + COMPUTE_READ=compute_decompiled_ra_stream, WIDGET=ContainerFrame + ), + SIZE=0, + )) ) netgame_flag = Struct("netgame_flag", @@ -467,8 +472,9 @@ def object_swatch(name, def_id, size=48): SInt16("team_index"), SInt16("spawn_time", SIDETIP="seconds(0 = default)", UNIT_SCALE=sec_unit_scale), # seconds + FlUInt32("unknown", VISIBLE=False), - Pad(48), + Pad(44), QStruct("position", INCLUDE=xyz_float), float_rad("facing"), # radians dependency("item_collection", "itmc"), @@ -542,6 +548,8 @@ def object_swatch(name, def_id, size=48): SEnum16("return_type", *script_object_types, EDITABLE=False), UInt32("root_expression_index", EDITABLE=False), Computed("decompiled_script", WIDGET=HaloScriptTextFrame), + Pad(40), + Pad(12), # replaced with parameters in mcc_hek SIZE=92, ) @@ -575,7 +583,7 @@ def object_swatch(name, def_id, size=48): ) cutscene_camera_point = Struct("cutscene_camera_point", - Pad(4), + FlUInt32("unknown", VISIBLE=False), ascii_str32("name"), Pad(4), QStruct("position", INCLUDE=xyz_float), @@ -585,7 +593,7 @@ def object_swatch(name, def_id, size=48): ) cutscene_title = Struct("cutscene_title", - Pad(4), + FlUInt32("unknown", VISIBLE=False), ascii_str32("name"), Pad(4), QStruct("text_bounds", @@ -606,7 +614,8 @@ def object_swatch(name, def_id, size=48): "center", ), - Pad(6), + Pad(2), + Pad(4), # replaced with flags in mcc_hek #QStruct("text_color", INCLUDE=argb_byte), #QStruct("shadow_color", INCLUDE=argb_byte), UInt32("text_color", INCLUDE=argb_uint32), @@ -626,8 +635,11 @@ def object_swatch(name, def_id, size=48): dyn_senum16("animation", DYN_NAME_PATH="tagdata.ai_animation_references.STEPTREE[DYN_I].animation_name"), SInt8("sequence_id"), + Pad(1), + Pad(8), - Pad(45), + FlUInt16("cluster_index", VISIBLE=False), + Pad(34), SInt32("surface_index"), SIZE=80 ) @@ -635,7 +647,7 @@ def object_swatch(name, def_id, size=48): actor_starting_location = Struct("starting_location", QStruct("position", INCLUDE=xyz_float), float_rad("facing"), # radians - Pad(2), + FlUInt16("cluster_index", VISIBLE=False), SInt8("sequence_id"), Bool8("flags", "required", @@ -704,8 +716,8 @@ def object_swatch(name, def_id, size=48): from_to_sec("respawn_delay"), Pad(48), - reflexive("move_positions", move_position, 31), - reflexive("starting_locations", actor_starting_location, 31), + reflexive("move_positions", move_position, 32), + reflexive("starting_locations", actor_starting_location, 32), SIZE=232 ) @@ -759,7 +771,7 @@ def object_swatch(name, def_id, size=48): {NAME: "unused8", GUI_NAME: "8 / unused8"}, {NAME: "unused9", GUI_NAME: "9 / unused9"} ), - SInt16('unknown', VISIBLE=False), + FlUInt16('unknown', VISIBLE=False, DEFAULT=1), SEnum16("search_behavior", "normal", "never", @@ -799,6 +811,7 @@ def object_swatch(name, def_id, size=48): point = Struct("point", QStruct("position", INCLUDE=xyz_float), + FlUInt32("surface_index", VISIBLE=False), SIZE=20 ) @@ -847,8 +860,14 @@ def object_swatch(name, def_id, size=48): dyn_senum16("set_new_name", DYN_NAME_PATH="tagdata.object_names.STEPTREE[DYN_I].name"), Pad(12), - BytesRaw("unknown", DEFAULT=b"\xFF"*12, SIZE=12, VISIBLE=False), + FlUInt16("variant_1_id", VISIBLE=False), + FlUInt16("variant_2_id", VISIBLE=False), + FlUInt16("variant_3_id", VISIBLE=False), + FlUInt16("variant_4_id", VISIBLE=False), + FlUInt16("variant_5_id", VISIBLE=False), + FlUInt16("variant_6_id", VISIBLE=False), ascii_str32("encounter_name"), + FlUInt32("encounter_index", VISIBLE=False), SIZE=84 ) @@ -939,60 +958,61 @@ def object_swatch(name, def_id, size=48): reflexive("predicted_resources", predicted_resource, 1024, VISIBLE=False), reflexive("functions", function, 32, DYN_NAME_PATH='.name'), - rawdata_ref("scenario_editor_data", max_size=65536), + rawdata_ref("scenario_editor_data", max_size=UINT16_MAX), reflexive("comments", comment, 1024), - Pad(224), + Pad(12), # replaced with scavenger_hunt_objects in mcc_hek + Pad(212), reflexive("object_names", object_name, 512, - DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True), - reflexive("sceneries", scenery, 2000, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("sceneries", scenery, 2000, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("sceneries_palette", scenery_swatch, 100, - DYN_NAME_PATH='.name.filepath'), - reflexive("bipeds", biped, 128, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("bipeds", biped, 128, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("bipeds_palette", biped_swatch, 100, - DYN_NAME_PATH='.name.filepath'), - reflexive("vehicles", vehicle, 80, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("vehicles", vehicle, 80, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("vehicles_palette", vehicle_swatch, 100, - DYN_NAME_PATH='.name.filepath'), - reflexive("equipments", equipment, 256, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("equipments", equipment, 256, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("equipments_palette", equipment_swatch, 100, - DYN_NAME_PATH='.name.filepath'), - reflexive("weapons", weapon, 128, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("weapons", weapon, 128, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("weapons_palette", weapon_swatch, 100, - DYN_NAME_PATH='.name.filepath'), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), reflexive("device_groups", device_group, 128, DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True), - reflexive("machines", machine, 400, IGNORE_SAFE_MODE=True), + reflexive("machines", machine, 400, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("machines_palette", machine_swatch, 100, - DYN_NAME_PATH='.name.filepath'), - reflexive("controls", control, 100, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("controls", control, 100, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("controls_palette", control_swatch, 100, - DYN_NAME_PATH='.name.filepath'), - reflexive("light_fixtures", light_fixture, 500, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("light_fixtures", light_fixture, 500, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("light_fixtures_palette", light_fixture_swatch, 100, - DYN_NAME_PATH='.name.filepath'), - reflexive("sound_sceneries", sound_scenery, 256, IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("sound_sceneries", sound_scenery, 256, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("sound_sceneries_palette", sound_scenery_swatch, 100, - DYN_NAME_PATH='.name.filepath'), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), Pad(84), reflexive("player_starting_profiles", player_starting_profile, 256, - DYN_NAME_PATH='.name'), - reflexive("player_starting_locations", player_starting_location, 256), + DYN_NAME_PATH='.name', EXT_MAX=SINT16_MAX), + reflexive("player_starting_locations", player_starting_location, 256, EXT_MAX=SINT16_MAX), reflexive("trigger_volumes", trigger_volume, 256, - DYN_NAME_PATH='.name'), + DYN_NAME_PATH='.name', EXT_MAX=SINT16_MAX), reflexive("recorded_animations", recorded_animation, 1024, - DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), reflexive("netgame_flags", netgame_flag, 200, - DYN_NAME_PATH='.type.enum_name'), + DYN_NAME_PATH='.type.enum_name', EXT_MAX=SINT16_MAX), reflexive("netgame_equipments", netgame_equipment, 200, - DYN_NAME_PATH='.item_collection.filepath'), - reflexive("starting_equipments", starting_equipment, 200), + DYN_NAME_PATH='.item_collection.filepath', EXT_MAX=SINT16_MAX), + reflexive("starting_equipments", starting_equipment, 200, EXT_MAX=SINT16_MAX), reflexive("bsp_switch_trigger_volumes", bsp_switch_trigger_volume, 256), - reflexive("decals", decal, 65535), + reflexive("decals", decal, 65535, EXT_MAX=UINT16_MAX), reflexive("decals_palette", decal_swatch, 128, - DYN_NAME_PATH='.name.filepath'), + DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), reflexive("detail_object_collection_palette", detail_object_collection_swatch, 32, DYN_NAME_PATH='.name.filepath'), @@ -1033,6 +1053,11 @@ def object_swatch(name, def_id, size=48): SIZE=1456, ) +scnr_body_with_decomp_ra_stream = desc_variant(scnr_body, + reflexive("recorded_animations", recorded_animation_with_ra_stream, 1024, + DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX) + ) + def get(): return scnr_def @@ -1040,5 +1065,11 @@ def get(): blam_header('scnr', 2), scnr_body, - ext=".scenario", endian=">", tag_cls=HekTag + ext=".scenario", endian=">", tag_cls=ScnrTag ) + +scnr_with_decomp_ra_stream_def = TagDef("scnr", + blam_header('scnr', 2), + scnr_body_with_decomp_ra_stream, + ext=".scenario", endian=">", tag_cls=ScnrTag + ) \ No newline at end of file diff --git a/reclaimer/hek/defs/senv.py b/reclaimer/hek/defs/senv.py index 42aabecb..476c18d7 100644 --- a/reclaimer/hek/defs/senv.py +++ b/reclaimer/hek/defs/senv.py @@ -88,6 +88,7 @@ "blended_base_specular", COMMENT=environment_shader_type_comment ), + SIZE=4 ) diffuse = Struct("diffuse", diff --git a/reclaimer/hek/defs/snd_.py b/reclaimer/hek/defs/snd_.py index ab29cd73..b6d072e0 100644 --- a/reclaimer/hek/defs/snd_.py +++ b/reclaimer/hek/defs/snd_.py @@ -67,22 +67,20 @@ """ ) - permutation = Struct('permutation', ascii_str32("name"), Float("skip_fraction"), Float("gain", DEFAULT=1.0), compression, SInt16("next_permutation_index", DEFAULT=-1), - FlSInt32("unknown0", VISIBLE=False), - FlUInt32("unknown1", VISIBLE=False), # always zero? - FlUInt32("unknown2", VISIBLE=False), - # this is actually the required length of the ogg + UInt32("sample_data_pointer", VISIBLE=False), + UInt32("unknown", VISIBLE=False), # always zero? + UInt32("parent_tag_id", VISIBLE=False), + # for ogg vorbis, this is the required length of the # decompression buffer. For "none" compression, this - # mirrors samples.size, so a more appropriate name - # for this field should be pcm_buffer_size - FlUInt32("ogg_sample_count", EDITABLE=False), - FlUInt32("unknown3", VISIBLE=False), # seems to always be == unknown2 + # mirrors samples.size + FlUInt32("buffer_size", EDITABLE=False), + UInt32("parent_tag_id2", VISIBLE=False), rawdata_ref("samples", max_size=4194304, widget=SoundSampleFrame), rawdata_ref("mouth_data", max_size=8192), rawdata_ref("subtitle_data", max_size=512), @@ -101,7 +99,8 @@ SInt32("unknown2", VISIBLE=False, DEFAULT=-1), reflexive("permutations", permutation, 256, - DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True), + DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX + ), SIZE=72, ) @@ -150,12 +149,15 @@ compression, dependency("promotion_sound", "snd!"), SInt16("promotion_count"), - SInt16("unknown1", VISIBLE=False), - Struct("unknown2", INCLUDE=rawdata_ref_struct, VISIBLE=False), + Pad(2), + FlUInt32("max_play_length", VISIBLE=False), + Pad(8), + FlUInt32("unknown1", VISIBLE=False, DEFAULT=0xFFFFFFFF), + FlUInt32("unknown2", VISIBLE=False, DEFAULT=0xFFFFFFFF), reflexive("pitch_ranges", pitch_range, 8, DYN_NAME_PATH='.name'), - SIZE=164, + SIZE=164, WIDGET=SoundPlayerFrame ) @@ -170,6 +172,6 @@ def get(): ) snd__meta_stub = desc_variant( - snd__body, ("pitch_ranges", Pad(12)) + snd__body, ("pitch_ranges", reflexive_struct) ) snd__meta_stub_blockdef = BlockDef(snd__meta_stub) diff --git a/reclaimer/hek/defs/snde.py b/reclaimer/hek/defs/snde.py index 1a5ad2dc..8ce07ee3 100644 --- a/reclaimer/hek/defs/snde.py +++ b/reclaimer/hek/defs/snde.py @@ -12,7 +12,7 @@ from supyr_struct.defs.tag_def import TagDef snde_body = QStruct("tagdata", - Pad(4), + FlUInt32("unknown", VISIBLE=False), UInt16("priority"), Pad(2), Float("room_intensity"), diff --git a/reclaimer/hek/defs/soso.py b/reclaimer/hek/defs/soso.py index ca8f22de..073ed047 100644 --- a/reclaimer/hek/defs/soso.py +++ b/reclaimer/hek/defs/soso.py @@ -61,7 +61,7 @@ ), Pad(14), Float("translucency"), - COMMENT=soso_comment + SIZE=20, COMMENT=soso_comment ) self_illumination = Struct("self_illumination", @@ -113,39 +113,35 @@ QStruct("parallel_tint_color", INCLUDE=rgb_float), dependency("cube_map", "bitm"), - #COMMENT=reflection_prop_comment + Pad(16), + + COMMENT=reflection_prop_comment ) soso_attrs = Struct("soso_attrs", - #Model Shader Properties model_shader, Pad(16), - #Color-Change SEnum16("color_change_source", *function_names, COMMENT=cc_comment), - Pad(30), - #Self-Illumination self_illumination, Pad(12), - #Diffuse, Multipurpose, and Detail Maps maps, # this padding is the reflexive for the OS shader model extension Pad(12), - #Texture Scrolling Animation texture_scrolling, Pad(8), - #Reflection Properties reflection, - Pad(16), - Float("unknown0", VISIBLE=False), - BytesRaw("unknown1", SIZE=16, VISIBLE=False), # little endian dependency + # NOTE: these aren't actually used, but they were at one point in + # development. keeping these for documentation purposes. + Pad(4), #Float("reflection_bump_scale", VISIBLE=False), + Pad(16), #dependency("reflection_bump_map", "bitm", VISIBLE=False), SIZE=400 ) diff --git a/reclaimer/hek/defs/ssce.py b/reclaimer/hek/defs/ssce.py index 78591bd8..052a129e 100644 --- a/reclaimer/hek/defs/ssce.py +++ b/reclaimer/hek/defs/ssce.py @@ -9,15 +9,8 @@ from .obje import * from .objs.obje import ObjeTag -from supyr_struct.defs.tag_def import TagDef -from supyr_struct.util import desc_variant - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(11)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "ssce") ssce_body = Struct("tagdata", obje_attrs, SIZE=508, diff --git a/reclaimer/hek/defs/unhi.py b/reclaimer/hek/defs/unhi.py index 097c352a..0830f039 100644 --- a/reclaimer/hek/defs/unhi.py +++ b/reclaimer/hek/defs/unhi.py @@ -126,8 +126,11 @@ ) auxilary_meter = Struct("auxilary_meter", + SEnum16("type", + "integrated_light", + VISIBLE=False + ), Pad(18), - SEnum16("type", "integrated_light", VISIBLE=False), Struct("background", INCLUDE=hud_background), QStruct("anchor_offset", diff --git a/reclaimer/hek/defs/unit.py b/reclaimer/hek/defs/unit.py index 949e9a98..aa73c3f0 100644 --- a/reclaimer/hek/defs/unit.py +++ b/reclaimer/hek/defs/unit.py @@ -10,6 +10,8 @@ from ...common_descs import * from .objs.tag import HekTag from supyr_struct.defs.tag_def import TagDef +# import here so it can be reused in all variants of unit +from supyr_struct.util import desc_variant def get(): return unit_def @@ -174,10 +176,8 @@ def get(): ), Pad(2), - Struct("mcc_additions", # replaced with opensauce unit extension in os_v4 - SEnum16("mcc_scoring_type", TOOLTIP="Used to determine score in MCC", *mcc_actor_types), - Pad(10), - ), + Pad(12), # replaced with opensauce unit extension in os_v4 and mcc_additions in mcc_hek + reflexive("new_hud_interfaces", new_hud_interface, 2, 'default/solo', 'multiplayer'), reflexive("dialogue_variants", dialogue_variant, 16, @@ -188,7 +188,8 @@ def get(): SEnum16('grenade_type', *grenade_types), SInt16('grenade_count', MIN=0), - Pad(4), + FlUInt16("soft_ping_stun_ticks", VISIBLE=False), # set to soft_ping_interrupt_time * 30 + FlUInt16("hard_ping_stun_ticks", VISIBLE=False), # set to hard_ping_interrupt_time * 30 reflexive("powered_seats", powered_seat, 2, "driver", "gunner"), reflexive("weapons", weapon, 4, DYN_NAME_PATH='.weapon.filepath'), diff --git a/reclaimer/hek/defs/weap.py b/reclaimer/hek/defs/weap.py index 27afab29..6ad1d756 100644 --- a/reclaimer/hek/defs/weap.py +++ b/reclaimer/hek/defs/weap.py @@ -10,13 +10,8 @@ from .obje import * from .item import * from .objs.weap import WeapTag -from supyr_struct.util import desc_variant -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = desc_variant(obje_attrs, - ("object_type", object_type(2)) - ) +obje_attrs = obje_attrs_variant(obje_attrs, "weap") magazine_item = Struct("magazine_item", SInt16("rounds"), @@ -67,6 +62,36 @@ SIZE=132 ) +# split out to be reused in mcc_hek +firing = Struct("firing", + QStruct("rounds_per_second", + Float("from", UNIT_SCALE=per_sec_unit_scale, GUI_NAME=''), + Float("to", UNIT_SCALE=per_sec_unit_scale), + ORIENT='h' + ), + float_sec("acceleration_time"), + float_sec("deceleration_time"), + Float("blurred_rate_of_fire", UNIT_SCALE=per_sec_unit_scale), + + Pad(8), + SEnum16("magazine", + 'primary', + 'secondary', + ('NONE', -1), + DEFAULT=-1 + ), + SInt16("rounds_per_shot"), + SInt16("minimum_rounds_loaded"), + SInt16("rounds_between_tracers"), + + Pad(6), + SEnum16("firing_noise", *sound_volumes), + from_to_zero_to_one("error"), + float_sec("error_acceleration_time"), + float_sec("error_deceleration_time"), + SIZE=60 + ) + trigger = Struct("trigger", Bool32("flags", "tracks_fired_projectile", @@ -84,33 +109,7 @@ "projectiles_have_identical_error", "projectile_is_client_side_only", ), - Struct("firing", - QStruct("rounds_per_second", - Float("from", UNIT_SCALE=per_sec_unit_scale, GUI_NAME=''), - Float("to", UNIT_SCALE=per_sec_unit_scale), - ORIENT='h' - ), - float_sec("acceleration_time"), - float_sec("deceleration_time"), - Float("blurred_rate_of_fire", UNIT_SCALE=per_sec_unit_scale), - - Pad(8), - SEnum16("magazine", - 'primary', - 'secondary', - ('NONE', -1), - DEFAULT=-1 - ), - SInt16("rounds_per_shot"), - SInt16("minimum_rounds_loaded"), - SInt16("rounds_between_tracers"), - - Pad(6), - SEnum16("firing_noise", *sound_volumes), - from_to_zero_to_one("error"), - float_sec("error_acceleration_time"), - float_sec("error_deceleration_time"), - ), + firing, Pad(8), Struct("charging", diff --git a/reclaimer/hek/handler.py b/reclaimer/hek/handler.py index 3414faf5..80a0a4ee 100644 --- a/reclaimer/hek/handler.py +++ b/reclaimer/hek/handler.py @@ -14,6 +14,8 @@ from traceback import format_exc from pathlib import Path, PureWindowsPath +# NOTE: this is a pretty tough dependency to move to make +# reclaimer able to operate without binilla installed. from binilla.handler import Handler from reclaimer.data_extraction import h1_data_extractors @@ -83,8 +85,6 @@ def __init__(self, *args, **kwargs): self.datadir = Path( kwargs.get("datadir", self.tagsdir.parent.joinpath("data"))) - # These break on Python 3.9 - if self.tag_ref_cache is None: self.tag_ref_cache = self.build_loc_caches(TagRef) @@ -112,16 +112,10 @@ def _build_loc_cache(self, cond, desc={}): if f_type is None: return NO_LOC_REFS - # python 3.9 band-aid - - try: - nodepath_ref = NodepathRef(cond(desc)) - except Exception: - print("Ignore me if you're not a developer") - print(format_exc()) - return NO_LOC_REFS - + nodepath_ref = NodepathRef(cond(desc)) for key in desc: + if not isinstance(desc[key], dict): + continue sub_nodepath_ref = self._build_loc_cache(cond, desc[key]) if sub_nodepath_ref.is_ref or sub_nodepath_ref: nodepath_ref[key] = sub_nodepath_ref @@ -176,6 +170,10 @@ def get_nodes_by_paths(self, paths, node, cond=lambda x, y: True): def get_def_id(self, filepath): filepath = Path(filepath) + if is_path_empty(filepath): + # return None instead of throwing an error about a non-existent file + return + if self.tagsdir_relative and not filepath.is_absolute(): filepath = self.tagsdir.joinpath(filepath) @@ -189,8 +187,10 @@ def get_def_id(self, filepath): engine_id = f.read(4).decode(encoding='latin-1') if def_id in self.defs and engine_id == self.tag_header_engine_id: return def_id + except FileNotFoundError: + pass except Exception: - print(format_exc()); + print(format_exc()) return self.ext_id_map.get(filepath.suffix.lower()) diff --git a/reclaimer/hek/hardcoded_ce_tag_paths.py b/reclaimer/hek/hardcoded_ce_tag_paths.py index a3907a9f..08c286dd 100644 --- a/reclaimer/hek/hardcoded_ce_tag_paths.py +++ b/reclaimer/hek/hardcoded_ce_tag_paths.py @@ -183,3 +183,15 @@ HARDCODED_matg_TAG_PATHS + HARDCODED_scnr_TAG_PATHS + HARDCODED_lsnd_TAG_PATHS + HARDCODED_snd__TAG_PATHS ) + +HARDCODED_TAG_PATHS_BY_TYPE = { + "ustr": HARDCODED_ustr_TAG_PATHS, + "DeLa": HARDCODED_DeLa_TAG_PATHS, + "font": HARDCODED_font_TAG_PATHS, + "vcky": HARDCODED_vcky_TAG_PATHS, + "bitm": HARDCODED_bitm_TAG_PATHS, + "matg": HARDCODED_matg_TAG_PATHS, + "scnr": HARDCODED_scnr_TAG_PATHS, + "lsnd": HARDCODED_lsnd_TAG_PATHS, + "snd!": HARDCODED_snd__TAG_PATHS, + } \ No newline at end of file diff --git a/reclaimer/mcc_hek/__init__.py b/reclaimer/mcc_hek/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reclaimer/mcc_hek/defs/DeLa.py b/reclaimer/mcc_hek/defs/DeLa.py new file mode 100644 index 00000000..a0762da4 --- /dev/null +++ b/reclaimer/mcc_hek/defs/DeLa.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.DeLa import * diff --git a/reclaimer/mcc_hek/defs/Soul.py b/reclaimer/mcc_hek/defs/Soul.py new file mode 100644 index 00000000..f5e0804d --- /dev/null +++ b/reclaimer/mcc_hek/defs/Soul.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.Soul import * diff --git a/reclaimer/mcc_hek/defs/__init__.py b/reclaimer/mcc_hek/defs/__init__.py new file mode 100644 index 00000000..50ea1ec1 --- /dev/null +++ b/reclaimer/mcc_hek/defs/__init__.py @@ -0,0 +1,22 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +# DO NOT REMOVE! __all__ is used to locate frozen definitions +__all__ = ( + "actr", "actv", "ant_", "antr", "bipd", "bitm", "boom", "cdmg", "coll", + "colo", "cont", "ctrl", "deca", "DeLa", "devc", "devi", "dobc", "effe", + "elec", "eqip", "flag", "fog_", "font", "foot", "garb", "glw_", "grhi", + "hmt_", "hud_", "hudg", "item", "itmc", "jpt_", "lens", "lifi", "ligh", + "mach", "matg", "metr", "mgs2", "mod2", "mode", "mply", "ngpr", "obje", + "part", "pctl", "phys", "plac", "pphy", "proj", "rain", "sbsp", "scen", + "scex", "schi", "scnr", "senv", "sgla", "shdr", "sky_", "smet", "snd_", + "snde", "lsnd", "soso", "sotr", "Soul", "spla", "ssce", "str_", "swat", + "tagc", "trak", "udlg", "unhi", "unit", "ustr", "vcky", "vehi", "weap", + "wind", "wphi", + ) \ No newline at end of file diff --git a/reclaimer/mcc_hek/defs/actr.py b/reclaimer/mcc_hek/defs/actr.py new file mode 100644 index 00000000..71f1409c --- /dev/null +++ b/reclaimer/mcc_hek/defs/actr.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.actr import * diff --git a/reclaimer/mcc_hek/defs/actv.py b/reclaimer/mcc_hek/defs/actv.py new file mode 100644 index 00000000..ac9bc2a0 --- /dev/null +++ b/reclaimer/mcc_hek/defs/actv.py @@ -0,0 +1,34 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.actv import * + +metagame_scoring = Struct("metagame_scoring", + SEnum16("metagame_type", *actor_types_mcc), + SEnum16("metagame_class", *actor_classes_mcc), + SIZE=4, ORIENT="H", COMMENT="Used to determine score in MCC", + ) + +actv_grenades = desc_variant(actv_grenades, + SEnum16("grenade_type", *grenade_types_mcc) + ) +actv_body = desc_variant(actv_body, + actv_grenades, + ("pad_4", metagame_scoring), + ) + +def get(): + return actv_def + +actv_def = TagDef("actv", + blam_header('actv'), + actv_body, + + ext=".actor_variant", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/mcc_hek/defs/ant_.py b/reclaimer/mcc_hek/defs/ant_.py new file mode 100644 index 00000000..a03db3c2 --- /dev/null +++ b/reclaimer/mcc_hek/defs/ant_.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.ant_ import * diff --git a/reclaimer/mcc_hek/defs/antr.py b/reclaimer/mcc_hek/defs/antr.py new file mode 100644 index 00000000..d3fbd080 --- /dev/null +++ b/reclaimer/mcc_hek/defs/antr.py @@ -0,0 +1,46 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.antr import * + +unit_weapon_desc = desc_variant(unit_weapon_desc, + reflexive("ik_points", ik_point_desc, 8, DYN_NAME_PATH=".marker"), + reflexive("weapon_types", weapon_types_desc, 64, DYN_NAME_PATH=".label"), + ) +unit_desc = desc_variant(unit_desc, + reflexive("ik_points", ik_point_desc, 8, DYN_NAME_PATH=".marker"), + reflexive("weapons", unit_weapon_desc, 64, DYN_NAME_PATH=".name"), + ) +vehicle_desc = desc_variant(vehicle_desc, + reflexive("suspension_animations", suspension_desc, 32), + ) +fp_animation_desc = desc_variant(fp_animation_desc, + reflexive("animations", anim_enum_desc, 30, *fp_animation_names_mcc), + ) +animation_desc = desc_variant(animation_desc, + rawdata_ref("frame_data", max_size=4194304), + ) + +antr_body = desc_variant(antr_body, + reflexive("units", unit_desc, 2048, DYN_NAME_PATH=".label"), + reflexive("vehicles", vehicle_desc, 1), + reflexive("fp_animations", fp_animation_desc, 1), + reflexive("sound_references", sound_reference_desc, 512, DYN_NAME_PATH=".sound.filepath"), + reflexive("animations", animation_desc, 2048, DYN_NAME_PATH=".name"), + ) + +def get(): + return antr_def + +antr_def = TagDef("antr", + blam_header('antr', 4), + antr_body, + + ext=".model_animations", endian=">", tag_cls=AntrTag + ) diff --git a/reclaimer/mcc_hek/defs/bipd.py b/reclaimer/mcc_hek/defs/bipd.py new file mode 100644 index 00000000..e66fa11e --- /dev/null +++ b/reclaimer/mcc_hek/defs/bipd.py @@ -0,0 +1,31 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.bipd import * +from .obje import * +from .unit import * + +obje_attrs = obje_attrs_variant(obje_attrs, "bipd") + +bipd_body = Struct("tagdata", + obje_attrs, + unit_attrs, + bipd_attrs, + SIZE=1268, + ) + +def get(): + return bipd_def + +bipd_def = TagDef("bipd", + blam_header('bipd', 3), + bipd_body, + + ext=".biped", endian=">", tag_cls=BipdTag + ) diff --git a/reclaimer/mcc_hek/defs/bitm.py b/reclaimer/mcc_hek/defs/bitm.py new file mode 100644 index 00000000..2d674a07 --- /dev/null +++ b/reclaimer/mcc_hek/defs/bitm.py @@ -0,0 +1,105 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.bitm import * +from .objs.bitm import MccBitmTag + +format_comment_parts = format_comment.split("NOTE: ", 1) +format_comment = "".join(( + format_comment_parts[0], + """\ +*HIGH QUALITY COMPRESSION: Block compression format similar to DXT3 and DXT5, + with same size as DXT3/DXT5(8-bits per pixel), but with higher quality + results. The format is far too complex to describe short-hand here, but + for those interested in learning about it, it is described here: + https://learn.microsoft.com/en-us/windows/win32/direct3d11/bc7-format + +NOTE:""", + format_comment_parts[1], + )) + +bitmap_format = SEnum16("format", + "a8", + "y8", + "ay8", + "a8y8", + ("r5g6b5", 6), + ("a1r5g5b5", 8), + ("a4r4g4b4", 9), + ("x8r8g8b8", 10), + ("a8r8g8b8", 11), + ("dxt1", 14), + ("dxt3", 15), + ("dxt5", 16), + ("p8_bump", 17), + ("bc7", 18), + ) +bitmap_flags = Bool16("flags", + "power_of_2_dim", + "compressed", + "palletized", + "swizzled", + "linear", + "v16u16", + {NAME: "unknown", VISIBLE: False}, + {NAME: "prefer_low_detail", VISIBLE: False}, + {NAME: "data_in_resource_map", VISIBLE: False}, + {NAME: "environment", VISIBLE: False}, + ) +bitmap = desc_variant(bitmap, bitmap_format, bitmap_flags) +body_format = SEnum16("format", + "color_key_transparency", + "explicit_alpha", + "interpolated_alpha", + "color_16bit", + "color_32bit", + "monochrome", + "high_quality_compression", + COMMENT=format_comment + ) +body_flags = Bool16("flags", + "enable_diffusion_dithering", + "disable_height_map_compression", + "uniform_sprite_sequences", + "sprite_bug_fix", + {NAME: "hud_scale_half", GUI_NAME: "hud scale 50%"}, + "invert_detail_fade", + "use_average_color_for_detail_fade" + ) +sprite_processing = Struct("sprite_processing", + SEnum16("sprite_budget_size", + {NAME: "x32", VALUE: 0, GUI_NAME: "32x32"}, + {NAME: "x64", VALUE: 1, GUI_NAME: "64x64"}, + {NAME: "x128", VALUE: 2, GUI_NAME: "128x128"}, + {NAME: "x256", VALUE: 3, GUI_NAME: "256x256"}, + {NAME: "x512", VALUE: 4, GUI_NAME: "512x512"}, + {NAME: "x1024", VALUE: 5, GUI_NAME: "1024x1024"}, + ), + UInt16("sprite_budget_count"), + SIZE=4, COMMENT=sprite_processing_comment + ) +bitm_body = desc_variant(bitm_body, + body_format, + body_flags, + sprite_processing, + rawdata_ref("compressed_color_plate_data", max_size=1073741824), + rawdata_ref("processed_pixel_data", max_size=1073741824), + reflexive("bitmaps", bitmap, 65536, IGNORE_SAFE_MODE=True), + ) + +def get(): + return bitm_def + +bitm_def = TagDef("bitm", + blam_header('bitm', 7), + bitm_body, + + ext=".bitmap", endian=">", tag_cls=MccBitmTag, + subdefs = {'pixel_root':pixel_root} + ) diff --git a/reclaimer/mcc_hek/defs/boom.py b/reclaimer/mcc_hek/defs/boom.py new file mode 100644 index 00000000..9f928092 --- /dev/null +++ b/reclaimer/mcc_hek/defs/boom.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.boom import * diff --git a/reclaimer/mcc_hek/defs/cdmg.py b/reclaimer/mcc_hek/defs/cdmg.py new file mode 100644 index 00000000..a7b7814f --- /dev/null +++ b/reclaimer/mcc_hek/defs/cdmg.py @@ -0,0 +1,51 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.cdmg import * + +damage_flags = Bool32("flags", + "does_not_hurt_owner", + {NAME: "headshot", GUI_NAME: "can cause headshots"}, + "pings_resistant_units", + "does_not_hurt_friends", + "does_not_ping_units", + "detonates_explosives", + "only_hurts_shields", + "causes_flaming_death", + {NAME: "indicator_points_down", GUI_NAME: "damage indicator always points down"}, + "skips_shields", + "only_hurts_one_infection_form", + {NAME: "multiplayer_headshot", GUI_NAME: "can cause multiplayer headshots"}, + "infection_form_pop", + "ignore_seat_scale_for_dir_dmg", + "forces_hard_ping", + "does_not_hurt_players", + "use_3d_instantaneous_acceleration", + "allow_any_non_zero_acceleration_value", + ) + +damage = desc_variant(damage, + damage_flags, + QStruct("instantaneous_acceleration", INCLUDE=ijk_float, SIDETIP="[-inf,+inf]", ORIENT="h"), + ("pad_13", Pad(0)), + # we're doing some weird stuff to make this work, so we're turning off verify + verify=False + ) + +cdmg_body = desc_variant(cdmg_body, damage) + +def get(): + return cdmg_def + +cdmg_def = TagDef("cdmg", + blam_header('cdmg'), + cdmg_body, + + ext=".continuous_damage_effect", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/mcc_hek/defs/coll.py b/reclaimer/mcc_hek/defs/coll.py new file mode 100644 index 00000000..0adb7301 --- /dev/null +++ b/reclaimer/mcc_hek/defs/coll.py @@ -0,0 +1,34 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.coll import * + +coll_body = desc_variant(coll_body, + reflexive("pathfinding_spheres", pathfinding_sphere, 256), + ) +fast_coll_body = desc_variant(coll_body, + reflexive("nodes", fast_node, 64, DYN_NAME_PATH='.name'), + ) + +def get(): + return coll_def + +coll_def = TagDef("coll", + blam_header("coll", 10), + coll_body, + + ext=".model_collision_geometry", endian=">", tag_cls=CollTag, + ) + +fast_coll_def = TagDef("coll", + blam_header("coll", 10), + fast_coll_body, + + ext=".model_collision_geometry", endian=">", tag_cls=CollTag, + ) diff --git a/reclaimer/mcc_hek/defs/colo.py b/reclaimer/mcc_hek/defs/colo.py new file mode 100644 index 00000000..d124ebc1 --- /dev/null +++ b/reclaimer/mcc_hek/defs/colo.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.colo import * diff --git a/reclaimer/mcc_hek/defs/cont.py b/reclaimer/mcc_hek/defs/cont.py new file mode 100644 index 00000000..246a8f05 --- /dev/null +++ b/reclaimer/mcc_hek/defs/cont.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.cont import * diff --git a/reclaimer/mcc_hek/defs/ctrl.py b/reclaimer/mcc_hek/defs/ctrl.py new file mode 100644 index 00000000..9a8a7301 --- /dev/null +++ b/reclaimer/mcc_hek/defs/ctrl.py @@ -0,0 +1,31 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.ctrl import * +from .obje import * +from .devi import * + +obje_attrs = obje_attrs_variant(obje_attrs, "ctrl") +ctrl_body = Struct("tagdata", + obje_attrs, + devi_attrs, + ctrl_attrs, + + SIZE=792, + ) + +def get(): + return ctrl_def + +ctrl_def = TagDef("ctrl", + blam_header('ctrl'), + ctrl_body, + + ext=".device_control", endian=">", tag_cls=CtrlTag + ) diff --git a/reclaimer/mcc_hek/defs/deca.py b/reclaimer/mcc_hek/defs/deca.py new file mode 100644 index 00000000..1c5983c7 --- /dev/null +++ b/reclaimer/mcc_hek/defs/deca.py @@ -0,0 +1,36 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.deca import * + +flags = Bool16("flags", + "geometry_inherited_by_next_decal_in_chain", + "interpolate_color_in_hsv", + "more_colors", + "no_random_rotation", + "water_effect", + "SAPIEN_snap_to_axis", + "SAPIEN_incremental_counter", + "animation_loop", + "preserve_aspect", + "disabled_in_remastered_by_blood_setting", + COMMENT=decal_comment + ) + +deca_body = desc_variant(deca_body, flags) + +def get(): + return deca_def + +deca_def = TagDef("deca", + blam_header('deca'), + deca_body, + + ext=".decal", endian=">", tag_cls=DecaTag + ) diff --git a/reclaimer/mcc_hek/defs/devc.py b/reclaimer/mcc_hek/defs/devc.py new file mode 100644 index 00000000..a32a44d5 --- /dev/null +++ b/reclaimer/mcc_hek/defs/devc.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.devc import * diff --git a/reclaimer/mcc_hek/defs/devi.py b/reclaimer/mcc_hek/defs/devi.py new file mode 100644 index 00000000..48429a52 --- /dev/null +++ b/reclaimer/mcc_hek/defs/devi.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.devi import * diff --git a/reclaimer/mcc_hek/defs/dobc.py b/reclaimer/mcc_hek/defs/dobc.py new file mode 100644 index 00000000..d97d047c --- /dev/null +++ b/reclaimer/mcc_hek/defs/dobc.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.dobc import * diff --git a/reclaimer/mcc_hek/defs/effe.py b/reclaimer/mcc_hek/defs/effe.py new file mode 100644 index 00000000..988adc3c --- /dev/null +++ b/reclaimer/mcc_hek/defs/effe.py @@ -0,0 +1,29 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.effe import * + +flags = Bool32("flags", + {NAME: "deleted_when_inactive", GUI_NAME: "deleted when attachment deactivates"}, + {NAME: "required", GUI_NAME: "required for gameplay (cannot optimize out)"}, + {NAME: "must_be_deterministic", VISIBLE: VISIBILITY_HIDDEN}, + "disabled_in_remastered_by_blood_setting" + ) + +effe_body = desc_variant(effe_body, flags) + +def get(): + return effe_def + +effe_def = TagDef("effe", + blam_header("effe", 4), + effe_body, + + ext=".effect", endian=">", tag_cls=EffeTag, + ) diff --git a/reclaimer/mcc_hek/defs/elec.py b/reclaimer/mcc_hek/defs/elec.py new file mode 100644 index 00000000..78521eea --- /dev/null +++ b/reclaimer/mcc_hek/defs/elec.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.elec import * diff --git a/reclaimer/mcc_hek/defs/eqip.py b/reclaimer/mcc_hek/defs/eqip.py new file mode 100644 index 00000000..0482b9ff --- /dev/null +++ b/reclaimer/mcc_hek/defs/eqip.py @@ -0,0 +1,36 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.eqip import * +from .obje import * +from .item import * + +eqip_attrs = desc_variant(eqip_attrs, + SEnum16("grenade_type", *grenade_types_mcc) + ) + +obje_attrs = obje_attrs_variant(obje_attrs, "eqip") + +eqip_body = Struct("tagdata", + obje_attrs, + item_attrs, + eqip_attrs, + + SIZE=944, + ) + +def get(): + return eqip_def + +eqip_def = TagDef("eqip", + blam_header('eqip', 2), + eqip_body, + + ext=".equipment", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/flag.py b/reclaimer/mcc_hek/defs/flag.py new file mode 100644 index 00000000..705bcc63 --- /dev/null +++ b/reclaimer/mcc_hek/defs/flag.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.flag import * diff --git a/reclaimer/mcc_hek/defs/fog_.py b/reclaimer/mcc_hek/defs/fog_.py new file mode 100644 index 00000000..da390e5f --- /dev/null +++ b/reclaimer/mcc_hek/defs/fog_.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.fog_ import * diff --git a/reclaimer/mcc_hek/defs/font.py b/reclaimer/mcc_hek/defs/font.py new file mode 100644 index 00000000..8ab600d4 --- /dev/null +++ b/reclaimer/mcc_hek/defs/font.py @@ -0,0 +1,25 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.font import * + +flags = Bool32("flags", + "never_override_with_remastered_font_under_mcc", + ) +font_body = desc_variant(font_body, flags) + +def get(): + return font_def + +font_def = TagDef("font", + blam_header('font'), + font_body, + + ext=".font", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/mcc_hek/defs/foot.py b/reclaimer/mcc_hek/defs/foot.py new file mode 100644 index 00000000..e359bc3a --- /dev/null +++ b/reclaimer/mcc_hek/defs/foot.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.foot import * diff --git a/reclaimer/mcc_hek/defs/garb.py b/reclaimer/mcc_hek/defs/garb.py new file mode 100644 index 00000000..53b8b965 --- /dev/null +++ b/reclaimer/mcc_hek/defs/garb.py @@ -0,0 +1,30 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.garb import * +from .obje import * +from .item import * + +obje_attrs = obje_attrs_variant(obje_attrs, "garb") +garb_body = Struct("tagdata", + obje_attrs, + item_attrs, + + SIZE=944, + ) + +def get(): + return garb_def + +garb_def = TagDef("garb", + blam_header('garb'), + garb_body, + + ext=".garbage", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/glw_.py b/reclaimer/mcc_hek/defs/glw_.py new file mode 100644 index 00000000..057d76e1 --- /dev/null +++ b/reclaimer/mcc_hek/defs/glw_.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.glw_ import * diff --git a/reclaimer/mcc_hek/defs/grhi.py b/reclaimer/mcc_hek/defs/grhi.py new file mode 100644 index 00000000..963427bb --- /dev/null +++ b/reclaimer/mcc_hek/defs/grhi.py @@ -0,0 +1,25 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.grhi import * + +# NOTE: used by unhi and wphi +mcc_hud_anchor = SEnum16("anchor", *hud_anchors_mcc) + +grhi_body = desc_variant(grhi_body, mcc_hud_anchor) + +def get(): + return grhi_def + +grhi_def = TagDef("grhi", + blam_header("grhi"), + grhi_body, + + ext=".grenade_hud_interface", endian=">", tag_cls=HekTag, + ) diff --git a/reclaimer/mcc_hek/defs/hmt_.py b/reclaimer/mcc_hek/defs/hmt_.py new file mode 100644 index 00000000..5bc951be --- /dev/null +++ b/reclaimer/mcc_hek/defs/hmt_.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.hmt_ import * diff --git a/reclaimer/mcc_hek/defs/hud_.py b/reclaimer/mcc_hek/defs/hud_.py new file mode 100644 index 00000000..2b759083 --- /dev/null +++ b/reclaimer/mcc_hek/defs/hud_.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.hud_ import * diff --git a/reclaimer/mcc_hek/defs/hudg.py b/reclaimer/mcc_hek/defs/hudg.py new file mode 100644 index 00000000..47f129c9 --- /dev/null +++ b/reclaimer/mcc_hek/defs/hudg.py @@ -0,0 +1,54 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.hudg import * + +targets = Struct("targets", + dependency("target_bitmap", "bitm"), + SEnum16("language", + "english", + "french", + "spanish", + "italian", + "german", + "tchinese", + "japanese", + "korean", + "portuguese", + "latam_spanish", + "polish", + "russian", + "schinese" + ), + Bool16("flags", + "legacy_mode" + ), + SIZE=20 + ) + +remap = Struct("remap", + dependency("original_bitmap", "bitm"), + reflexive("targets", targets, 26, DYN_NAME_PATH='.target_bitmap.filepath'), + SIZE=28 + ) + +misc_hud_crap = desc_variant(misc_hud_crap, + ("unknown0", reflexive("remaps", remap, 32, DYN_NAME_PATH='.original_bitmap.filepath')) + ) +hudg_body = desc_variant(hudg_body, misc_hud_crap) + +def get(): + return hudg_def + +hudg_def = TagDef("hudg", + blam_header("hudg"), + hudg_body, + + ext=".hud_globals", endian=">", tag_cls=HekTag, + ) diff --git a/reclaimer/mcc_hek/defs/item.py b/reclaimer/mcc_hek/defs/item.py new file mode 100644 index 00000000..3c7c221f --- /dev/null +++ b/reclaimer/mcc_hek/defs/item.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.item import * diff --git a/reclaimer/mcc_hek/defs/itmc.py b/reclaimer/mcc_hek/defs/itmc.py new file mode 100644 index 00000000..e067cdab --- /dev/null +++ b/reclaimer/mcc_hek/defs/itmc.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.itmc import * diff --git a/reclaimer/mcc_hek/defs/jpt_.py b/reclaimer/mcc_hek/defs/jpt_.py new file mode 100644 index 00000000..509cfb84 --- /dev/null +++ b/reclaimer/mcc_hek/defs/jpt_.py @@ -0,0 +1,31 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.jpt_ import * +from .cdmg import damage_flags + +damage = desc_variant(damage, + damage_flags, + QStruct("instantaneous_acceleration", INCLUDE=ijk_float, SIDETIP="[-inf,+inf]", ORIENT="h"), + ("pad_13", Pad(0)), + # we're doing some weird stuff to make this work, so we're turning off verify + verify=False + ) + +jpt__body = desc_variant(jpt__body, damage) + +def get(): + return jpt__def + +jpt__def = TagDef("jpt!", + blam_header('jpt!', 6), + jpt__body, + + ext=".damage_effect", endian=">", tag_cls=HekTag, + ) diff --git a/reclaimer/mcc_hek/defs/lens.py b/reclaimer/mcc_hek/defs/lens.py new file mode 100644 index 00000000..300c08e3 --- /dev/null +++ b/reclaimer/mcc_hek/defs/lens.py @@ -0,0 +1,33 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.lens import * + +bitmaps = desc_variant(bitmaps, + Bool16("flags", + "sun", + "no_occlusion_test", + "only_render_in_first_person", + "only_render_in_third_person", + "fade_in_more_quickly", + "fade_out_more_quickly", + "scale_by_marker", + ) + ) +lens_body = desc_variant(lens_body, bitmaps) + +def get(): + return lens_def + +lens_def = TagDef("lens", + blam_header("lens", 2), + lens_body, + + ext=".lens_flare", endian=">", tag_cls=LensTag, + ) diff --git a/reclaimer/mcc_hek/defs/lifi.py b/reclaimer/mcc_hek/defs/lifi.py new file mode 100644 index 00000000..a8261754 --- /dev/null +++ b/reclaimer/mcc_hek/defs/lifi.py @@ -0,0 +1,30 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.lifi import * +from .obje import * +from .devi import * + +obje_attrs = obje_attrs_variant(obje_attrs, "lifi") +lifi_body = Struct("tagdata", + obje_attrs, + devi_attrs, + + SIZE=720, + ) + +def get(): + return lifi_def + +lifi_def = TagDef("lifi", + blam_header('lifi'), + lifi_body, + + ext=".device_light_fixture", endian=">", tag_cls=LifiTag + ) diff --git a/reclaimer/mcc_hek/defs/ligh.py b/reclaimer/mcc_hek/defs/ligh.py new file mode 100644 index 00000000..ac32dcef --- /dev/null +++ b/reclaimer/mcc_hek/defs/ligh.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.ligh import * diff --git a/reclaimer/mcc_hek/defs/lsnd.py b/reclaimer/mcc_hek/defs/lsnd.py new file mode 100644 index 00000000..aafc7ba1 --- /dev/null +++ b/reclaimer/mcc_hek/defs/lsnd.py @@ -0,0 +1,29 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.lsnd import * + +lsnd_flags = Bool32("flags", + "deafening_to_ai", + "not_a_loop", + "stops_music", + "siege_of_the_madrigal", + ) + +lsnd_body = desc_variant(lsnd_body, lsnd_flags) + +def get(): + return lsnd_def + +lsnd_def = TagDef("lsnd", + blam_header("lsnd", 3), + lsnd_body, + + ext=".sound_looping", endian=">", tag_cls=HekTag, + ) diff --git a/reclaimer/mcc_hek/defs/mach.py b/reclaimer/mcc_hek/defs/mach.py new file mode 100644 index 00000000..9d440179 --- /dev/null +++ b/reclaimer/mcc_hek/defs/mach.py @@ -0,0 +1,31 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.mach import * +from .obje import * +from .devi import * + +obje_attrs = obje_attrs_variant(obje_attrs, "mach") +mach_body = Struct("tagdata", + obje_attrs, + devi_attrs, + mach_attrs, + + SIZE=804, + ) + +def get(): + return mach_def + +mach_def = TagDef("mach", + blam_header('mach'), + mach_body, + + ext=".device_machine", endian=">", tag_cls=MachTag + ) diff --git a/reclaimer/mcc_hek/defs/matg.py b/reclaimer/mcc_hek/defs/matg.py new file mode 100644 index 00000000..2e36286e --- /dev/null +++ b/reclaimer/mcc_hek/defs/matg.py @@ -0,0 +1,24 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.matg import * + +matg_body = desc_variant(matg_body, + reflexive("grenades", grenade, 4, *grenade_types_mcc), + ) + +def get(): + return matg_def + +matg_def = TagDef("matg", + blam_header('matg', 3), + matg_body, + + ext=".globals", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/mcc_hek/defs/metr.py b/reclaimer/mcc_hek/defs/metr.py new file mode 100644 index 00000000..0f59bdcc --- /dev/null +++ b/reclaimer/mcc_hek/defs/metr.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.metr import * diff --git a/reclaimer/mcc_hek/defs/mgs2.py b/reclaimer/mcc_hek/defs/mgs2.py new file mode 100644 index 00000000..516735cc --- /dev/null +++ b/reclaimer/mcc_hek/defs/mgs2.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.mgs2 import * diff --git a/reclaimer/mcc_hek/defs/mod2.py b/reclaimer/mcc_hek/defs/mod2.py new file mode 100644 index 00000000..a3aa0bbd --- /dev/null +++ b/reclaimer/mcc_hek/defs/mod2.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.mod2 import * diff --git a/reclaimer/mcc_hek/defs/mode.py b/reclaimer/mcc_hek/defs/mode.py new file mode 100644 index 00000000..5c8d9d1c --- /dev/null +++ b/reclaimer/mcc_hek/defs/mode.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.mode import * diff --git a/reclaimer/mcc_hek/defs/mply.py b/reclaimer/mcc_hek/defs/mply.py new file mode 100644 index 00000000..ba79de26 --- /dev/null +++ b/reclaimer/mcc_hek/defs/mply.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.mply import * diff --git a/reclaimer/mcc_hek/defs/ngpr.py b/reclaimer/mcc_hek/defs/ngpr.py new file mode 100644 index 00000000..e0d01cd8 --- /dev/null +++ b/reclaimer/mcc_hek/defs/ngpr.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.ngpr import * diff --git a/reclaimer/mcc_hek/defs/obje.py b/reclaimer/mcc_hek/defs/obje.py new file mode 100644 index 00000000..78890b94 --- /dev/null +++ b/reclaimer/mcc_hek/defs/obje.py @@ -0,0 +1,38 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.obje import * + +obje_flags = Bool16('flags', + 'does_not_cast_shadow', + 'transparent_self_occlusion', + 'brighter_than_it_should_be', + 'not_a_pathfinding_obstacle', + "extension_of_parent", + "cast_shadow_by_default", + "does_not_have_remastered_geometry", + {NAME: 'xbox_unknown_bit_8', VALUE: 1<<8, VISIBLE: False}, + {NAME: 'xbox_unknown_bit_11', VALUE: 1<<11, VISIBLE: False}, + ) + +obje_attrs = desc_variant(obje_attrs, obje_flags) +obje_body = Struct('tagdata', + obje_attrs, + SIZE=380 + ) + +def get(): + return obje_def + +obje_def = TagDef("obje", + blam_header('obje'), + obje_body, + + ext=".object", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/objs/bitm.py b/reclaimer/mcc_hek/defs/objs/bitm.py new file mode 100644 index 00000000..48f643c5 --- /dev/null +++ b/reclaimer/mcc_hek/defs/objs/bitm.py @@ -0,0 +1,27 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# +from ....hek.defs.objs.bitm import * +from reclaimer.constants import MCC_FORMAT_NAME_MAP + +class MccBitmTag(BitmTag): + @property + def format_name_map(self): + return MCC_FORMAT_NAME_MAP + + def fix_top_format(self): + top_format = None + if self.bitmap_count() > 0: + pixel_format = self.data.tagdata.bitmaps.bitmaps_array[0].format.enum_name + if pixel_format == "bc7": + top_format = "high_quality_compression" + + if top_format is None: + return super().fix_top_format() + + self.data.tagdata.format.set_to(top_format) \ No newline at end of file diff --git a/reclaimer/mcc_hek/defs/part.py b/reclaimer/mcc_hek/defs/part.py new file mode 100644 index 00000000..1125f046 --- /dev/null +++ b/reclaimer/mcc_hek/defs/part.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.part import * diff --git a/reclaimer/mcc_hek/defs/pctl.py b/reclaimer/mcc_hek/defs/pctl.py new file mode 100644 index 00000000..9bed8582 --- /dev/null +++ b/reclaimer/mcc_hek/defs/pctl.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.pctl import * diff --git a/reclaimer/mcc_hek/defs/phys.py b/reclaimer/mcc_hek/defs/phys.py new file mode 100644 index 00000000..e4b8d637 --- /dev/null +++ b/reclaimer/mcc_hek/defs/phys.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.phys import * diff --git a/reclaimer/mcc_hek/defs/plac.py b/reclaimer/mcc_hek/defs/plac.py new file mode 100644 index 00000000..4d6bb051 --- /dev/null +++ b/reclaimer/mcc_hek/defs/plac.py @@ -0,0 +1,28 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.plac import * +from .obje import * + +obje_attrs = obje_attrs_variant(obje_attrs, "plac") +plac_body = Struct("tagdata", + obje_attrs, + + SIZE=508, + ) + +def get(): + return plac_def + +plac_def = TagDef("plac", + blam_header('plac', 2), + plac_body, + + ext=".placeholder", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/pphy.py b/reclaimer/mcc_hek/defs/pphy.py new file mode 100644 index 00000000..8e02981c --- /dev/null +++ b/reclaimer/mcc_hek/defs/pphy.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.pphy import * diff --git a/reclaimer/mcc_hek/defs/proj.py b/reclaimer/mcc_hek/defs/proj.py new file mode 100644 index 00000000..499bbe0c --- /dev/null +++ b/reclaimer/mcc_hek/defs/proj.py @@ -0,0 +1,40 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.proj import * +from .obje import * + +potential_response_flags = Bool16("flags", + "only_against_units", + "never_against_units" + ) +potential_response = desc_variant(potential_response, potential_response_flags) +material_response = desc_variant(material_response, potential_response) +material_responses = reflexive("material_responses", + material_response, len(materials_list), *materials_list + ) + +obje_attrs = obje_attrs_variant(obje_attrs, "proj") +proj_attrs = desc_variant(proj_attrs, material_responses) + +proj_body = Struct("tagdata", + obje_attrs, + proj_attrs, + SIZE=588, + ) + +def get(): + return proj_def + +proj_def = TagDef("proj", + blam_header('proj', 5), + proj_body, + + ext=".projectile", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/rain.py b/reclaimer/mcc_hek/defs/rain.py new file mode 100644 index 00000000..eeff8261 --- /dev/null +++ b/reclaimer/mcc_hek/defs/rain.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.rain import * diff --git a/reclaimer/mcc_hek/defs/sbsp.py b/reclaimer/mcc_hek/defs/sbsp.py new file mode 100644 index 00000000..1f64cb7e --- /dev/null +++ b/reclaimer/mcc_hek/defs/sbsp.py @@ -0,0 +1,22 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.sbsp import * + +sbsp_meta_header_def = BlockDef("sbsp_meta_header", + # to convert the meta pointer to offsets, do: pointer - bsp_magic + UInt32("meta_pointer"), + # the rest of these are literal pointers in the map + UInt32("uncompressed_render_vertices_size"), + UInt32("uncompressed_render_vertices_pointer"), + UInt32("compressed_render_vertices_size"), # name is a guess + UInt32("compressed_render_vertices_pointer"), # name is a guess + UInt32("sig", DEFAULT="sbsp"), + SIZE=24, TYPE=QStruct + ) \ No newline at end of file diff --git a/reclaimer/mcc_hek/defs/scen.py b/reclaimer/mcc_hek/defs/scen.py new file mode 100644 index 00000000..15bb80e6 --- /dev/null +++ b/reclaimer/mcc_hek/defs/scen.py @@ -0,0 +1,27 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.scen import * +from .obje import * + +obje_attrs = obje_attrs_variant(obje_attrs, "scen") +scen_body = Struct("tagdata", + obje_attrs, + SIZE=508, + ) + +def get(): + return scen_def + +scen_def = TagDef("scen", + blam_header('scen'), + scen_body, + + ext=".scenery", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/scex.py b/reclaimer/mcc_hek/defs/scex.py new file mode 100644 index 00000000..6a771580 --- /dev/null +++ b/reclaimer/mcc_hek/defs/scex.py @@ -0,0 +1,29 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.scex import * +from .schi import * + +scex_attrs = desc_variant(scex_attrs, extra_flags) +scex_body = Struct("tagdata", + shdr_attrs, + scex_attrs, + SIZE=120 + ) + +def get(): + return scex_def + +scex_def = TagDef("scex", + blam_header('scex'), + scex_body, + + ext=".shader_transparent_chicago_extended", + endian=">", tag_cls=ShdrTag + ) diff --git a/reclaimer/mcc_hek/defs/schi.py b/reclaimer/mcc_hek/defs/schi.py new file mode 100644 index 00000000..15b08bd8 --- /dev/null +++ b/reclaimer/mcc_hek/defs/schi.py @@ -0,0 +1,35 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.schi import * +from .shdr import * + +extra_flags = Bool32("extra_flags", + "dont_fade_active_camouflage", + "numeric_countdown_timer", + "custom_edition_blending", + ) +schi_attrs = desc_variant(schi_attrs, extra_flags) + +schi_body = Struct("tagdata", + shdr_attrs, + schi_attrs, + SIZE=108 + ) + +def get(): + return schi_def + +schi_def = TagDef("schi", + blam_header('schi'), + schi_body, + + ext=".shader_transparent_chicago", + endian=">", tag_cls=ShdrTag + ) diff --git a/reclaimer/mcc_hek/defs/scnr.py b/reclaimer/mcc_hek/defs/scnr.py new file mode 100644 index 00000000..579d8272 --- /dev/null +++ b/reclaimer/mcc_hek/defs/scnr.py @@ -0,0 +1,130 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.scnr import * + +object_ref_flags = Bool16('not_placed', + "automatically", + "on_easy", + "on_normal", + "on_hard", + "use_player_appearance", + ) + +starting_equipment_flags = Bool32("flags", + "no_grenades", + "plasma_grenades_only", + "type2_grenades_only", + "type3_grenades_only", + ) +parameter = Struct("parameter", + ascii_str32("name", EDITABLE=False), + SEnum16("return_type", *script_object_types, EDITABLE=False), + Pad(2), + SIZE=36, + ) +cutscene_title_flags = Bool32("flags", + "wrap_horizontally", + "wrap_vertically", + "center_vertically", + "bottom_justify", + ) +scavenger_hunt_objects = Struct("scavenger_hunt_objects", + ascii_str32("exported_name"), + dyn_senum16("scenario_object_name_index", DYN_NAME_PATH="tagdata.object_names.STEPTREE[DYN_I].name"), + Pad(2), + SIZE=36 + ) + +# Object references +object_ref_replacements = ( + ('not_placed', object_ref_flags), + ("pad_8", SInt8("appearance_player_index")), + ) + +scenery = desc_variant(scenery, *object_ref_replacements) +equipment = desc_variant(equipment, *object_ref_replacements) +sound_scenery = desc_variant(sound_scenery, *object_ref_replacements) +biped = desc_variant(biped, *object_ref_replacements) +vehicle = desc_variant(vehicle, *object_ref_replacements) +weapon = desc_variant(weapon, *object_ref_replacements) +machine = desc_variant(machine, *object_ref_replacements) +control = desc_variant(control, *object_ref_replacements) +light_fixture = desc_variant(light_fixture, *object_ref_replacements) + +player_starting_profile = desc_variant(player_starting_profile, + ("pad_11", SInt8("starting_grenade_type2_count", MIN=0)), + ("pad_12", SInt8("starting_grenade_type3_count", MIN=0)), + ) +netgame_equipment = desc_variant(netgame_equipment, + ("team_index", SInt16("usage_id")), + ) +starting_equipment = desc_variant(starting_equipment, + starting_equipment_flags + ) +halo_script = desc_variant(halo_script, + ("pad_6", reflexive("parameters", parameter, 16)) + ) +source_file = desc_variant(source_file, + rawdata_ref("source", max_size=1048576, widget=HaloScriptSourceFrame), + ) +cutscene_title = desc_variant(cutscene_title, + ("pad_8", cutscene_title_flags), + ) + +scnr_flags = Bool16("flags", + "cortana_hack", + "use_demo_ui", + {NAME: "color_correction", GUI_NAME: "color correction (ntsc->srgb)"}, + "DO_NOT_apply_bungie_campaign_tag_patches", + ) + +scnr_body = desc_variant(scnr_body, + scnr_flags, + ("pad_13", reflexive("scavenger_hunt_objects", scavenger_hunt_objects, 16)), + reflexive("object_names", object_name, 640, DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("sceneries", scenery, 2000, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("bipeds", biped, 128, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("vehicles", vehicle, 256, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("equipments", equipment, 256, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("weapons", weapon, 128, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("machines", machine, 400, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("controls", control, 100, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("light_fixtures", light_fixture, 500, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("sound_sceneries", sound_scenery, 256, IGNORE_SAFE_MODE=True, EXT_MAX=SINT16_MAX), + reflexive("sceneries_palette", scenery_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("bipeds_palette", biped_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("vehicles_palette", vehicle_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("equipments_palette", equipment_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("weapons_palette", weapon_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("machines_palette", machine_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("controls_palette", control_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("light_fixtures_palette", light_fixture_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("sound_sceneries_palette", sound_scenery_swatch, 256, DYN_NAME_PATH='.name.filepath', EXT_MAX=SINT16_MAX), + reflexive("player_starting_profiles", player_starting_profile, 256, DYN_NAME_PATH='.name', EXT_MAX=SINT16_MAX), + reflexive("netgame_equipments", netgame_equipment, 200, DYN_NAME_PATH='.item_collection.filepath', EXT_MAX=SINT16_MAX), + reflexive("starting_equipments", starting_equipment, 200, EXT_MAX=SINT16_MAX), + rawdata_ref("script_syntax_data", max_size=655396, IGNORE_SAFE_MODE=True), + rawdata_ref("script_string_data", max_size=819200, IGNORE_SAFE_MODE=True), + reflexive("scripts", halo_script, 1024, DYN_NAME_PATH='.name'), + reflexive("globals", halo_global, 512, DYN_NAME_PATH='.name'), + reflexive("references", reference, 512, DYN_NAME_PATH='.reference.filepath'), + reflexive("source_files", source_file, 16, DYN_NAME_PATH='.source_name'), + reflexive("cutscene_titles", cutscene_title, 64, DYN_NAME_PATH='.name'), + ) + +def get(): + return scnr_def + +scnr_def = TagDef("scnr", + blam_header('scnr', 2), + scnr_body, + + ext=".scenario", endian=">", tag_cls=ScnrTag + ) diff --git a/reclaimer/mcc_hek/defs/senv.py b/reclaimer/mcc_hek/defs/senv.py new file mode 100644 index 00000000..88cde9d7 --- /dev/null +++ b/reclaimer/mcc_hek/defs/senv.py @@ -0,0 +1,38 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.senv import * +from .shdr import * + +environment_shader_flags = Bool16("flags", + "alpha_tested", + "bump_map_is_specular_mask", + "true_atmospheric_fog", + "use_variant_2_for_calculation_bump_attention", + COMMENT=environment_shader_comment + ) + +environment_shader = desc_variant(environment_shader, environment_shader_flags) +senv_attrs = desc_variant(senv_attrs, environment_shader) +senv_body = Struct("tagdata", + shdr_attrs, + senv_attrs, + SIZE=836, + ) + + +def get(): + return senv_def + +senv_def = TagDef("senv", + blam_header('senv', 2), + senv_body, + + ext=".shader_environment", endian=">", tag_cls=ShdrTag + ) diff --git a/reclaimer/mcc_hek/defs/sgla.py b/reclaimer/mcc_hek/defs/sgla.py new file mode 100644 index 00000000..7b93274a --- /dev/null +++ b/reclaimer/mcc_hek/defs/sgla.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.sgla import * diff --git a/reclaimer/mcc_hek/defs/shdr.py b/reclaimer/mcc_hek/defs/shdr.py new file mode 100644 index 00000000..152ef509 --- /dev/null +++ b/reclaimer/mcc_hek/defs/shdr.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.shdr import * diff --git a/reclaimer/mcc_hek/defs/sky_.py b/reclaimer/mcc_hek/defs/sky_.py new file mode 100644 index 00000000..eff94d12 --- /dev/null +++ b/reclaimer/mcc_hek/defs/sky_.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.sky_ import * diff --git a/reclaimer/mcc_hek/defs/smet.py b/reclaimer/mcc_hek/defs/smet.py new file mode 100644 index 00000000..b079796d --- /dev/null +++ b/reclaimer/mcc_hek/defs/smet.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.smet import * diff --git a/reclaimer/mcc_hek/defs/snd_.py b/reclaimer/mcc_hek/defs/snd_.py new file mode 100644 index 00000000..f16321a9 --- /dev/null +++ b/reclaimer/mcc_hek/defs/snd_.py @@ -0,0 +1,52 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.snd_ import * + + +mcc_snd__flags = Bool32("flags", + "fit_to_adpcm_blocksize", + "split_long_sound_into_permutations", + "thirsty_grunt", + ) +# found from sapien error log: +# EXCEPTION halt in c:\mcc\qfe1\h1\code\h1a2\sources\cache\pc_sound_cache.c,#157: !TEST_FLAG(sound->runtime_flags, _sound_permutation_cached_bit) +mcc_runtime_perm_flags = UInt32("runtime_flags", + # NOTES: + # this is set, even if the samples are NOT cached. not sure why + # also, seems like the above exception trips if ANY bits are set + VISIBLE=False + ) +permutation = desc_variant(permutation, + ("parent_tag_id", mcc_runtime_perm_flags), + ("parent_tag_id2", UInt32("parent_tag_id")), + ) +pitch_range = desc_variant(pitch_range, + reflexive("permutations", permutation, 256, DYN_NAME_PATH='.name', IGNORE_SAFE_MODE=True), + ) + +mcc_snd__body = desc_variant(snd__body, + mcc_snd__flags, + reflexive("pitch_ranges", pitch_range, 8, DYN_NAME_PATH='.name'), + ) + +def get(): + return snd__def + +snd__def = TagDef("snd!", + blam_header('snd!', 4), + mcc_snd__body, + + ext=".sound", endian=">", tag_cls=Snd_Tag, + ) + +snd__meta_stub = desc_variant(mcc_snd__body, + ("pitch_ranges", Pad(12)), + ) +snd__meta_stub_blockdef = BlockDef(snd__meta_stub) diff --git a/reclaimer/mcc_hek/defs/snde.py b/reclaimer/mcc_hek/defs/snde.py new file mode 100644 index 00000000..f0767f1b --- /dev/null +++ b/reclaimer/mcc_hek/defs/snde.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.snde import * diff --git a/reclaimer/mcc_hek/defs/soso.py b/reclaimer/mcc_hek/defs/soso.py new file mode 100644 index 00000000..8cdf4f2e --- /dev/null +++ b/reclaimer/mcc_hek/defs/soso.py @@ -0,0 +1,39 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.soso import * +from .shdr import * + +model_shader_flags = Bool16("flags", + "detail_after_reflection", + "two_sided", + "not_alpha_tested", + "alpha_blended_decal", + "true_atmospheric_fog", + "disable_two_sided_culling", + "multipurpose_map_uses_og_xbox_channel_order", + ) + +model_shader = desc_variant(model_shader, model_shader_flags) +soso_attrs = desc_variant(soso_attrs, model_shader) + +soso_body = Struct("tagdata", + shdr_attrs, + soso_attrs + ) + +def get(): + return soso_def + +soso_def = TagDef("soso", + blam_header('soso', 2), + soso_body, + + ext=".shader_model", endian=">", tag_cls=ShdrTag + ) diff --git a/reclaimer/mcc_hek/defs/sotr.py b/reclaimer/mcc_hek/defs/sotr.py new file mode 100644 index 00000000..ae77c445 --- /dev/null +++ b/reclaimer/mcc_hek/defs/sotr.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.sotr import * diff --git a/reclaimer/mcc_hek/defs/spla.py b/reclaimer/mcc_hek/defs/spla.py new file mode 100644 index 00000000..b05fef80 --- /dev/null +++ b/reclaimer/mcc_hek/defs/spla.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.spla import * diff --git a/reclaimer/mcc_hek/defs/ssce.py b/reclaimer/mcc_hek/defs/ssce.py new file mode 100644 index 00000000..a962b2b3 --- /dev/null +++ b/reclaimer/mcc_hek/defs/ssce.py @@ -0,0 +1,27 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.ssce import * +from .obje import * + +obje_attrs = obje_attrs_variant(obje_attrs, "ssce") +ssce_body = Struct("tagdata", + obje_attrs, + SIZE=508, + ) + +def get(): + return ssce_def + +ssce_def = TagDef("ssce", + blam_header('ssce'), + ssce_body, + + ext=".sound_scenery", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/str_.py b/reclaimer/mcc_hek/defs/str_.py new file mode 100644 index 00000000..6a1411ae --- /dev/null +++ b/reclaimer/mcc_hek/defs/str_.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.str_ import * diff --git a/reclaimer/mcc_hek/defs/swat.py b/reclaimer/mcc_hek/defs/swat.py new file mode 100644 index 00000000..b35bb169 --- /dev/null +++ b/reclaimer/mcc_hek/defs/swat.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.swat import * diff --git a/reclaimer/mcc_hek/defs/tagc.py b/reclaimer/mcc_hek/defs/tagc.py new file mode 100644 index 00000000..d0be72ab --- /dev/null +++ b/reclaimer/mcc_hek/defs/tagc.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.tagc import * \ No newline at end of file diff --git a/reclaimer/mcc_hek/defs/trak.py b/reclaimer/mcc_hek/defs/trak.py new file mode 100644 index 00000000..c5ed1bb6 --- /dev/null +++ b/reclaimer/mcc_hek/defs/trak.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.trak import * diff --git a/reclaimer/mcc_hek/defs/udlg.py b/reclaimer/mcc_hek/defs/udlg.py new file mode 100644 index 00000000..9a6059b1 --- /dev/null +++ b/reclaimer/mcc_hek/defs/udlg.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.udlg import * diff --git a/reclaimer/mcc_hek/defs/unhi.py b/reclaimer/mcc_hek/defs/unhi.py new file mode 100644 index 00000000..379a2e1f --- /dev/null +++ b/reclaimer/mcc_hek/defs/unhi.py @@ -0,0 +1,108 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.unhi import * +from .grhi import hud_background, mcc_hud_anchor + +# to reduce a lot of code, these have been snipped out +meter_xform_common = ( # NOTE: used by wphi + QStruct("anchor_offset", + SInt16("x"), SInt16("y"), ORIENT='h', + ), + Float("width_scale"), + Float("height_scale"), + Bool16("scaling_flags", *hud_scaling_flags), + Pad(22), + ) +meter_common = ( # NOTE: used by wphi + dependency("meter_bitmap", "bitm"), + UInt32("color_at_meter_minimum", INCLUDE=xrgb_uint32), + UInt32("color_at_meter_maximum", INCLUDE=xrgb_uint32), + UInt32("flash_color", INCLUDE=xrgb_uint32), + UInt32("empty_color", INCLUDE=argb_uint32), + Bool8("flags", + "use_min_max_for_state_changes", + "interpolate_between_min_max_flash_colors_as_state_changes", + "interpolate_color_along_hsv_space", + "more_colors_for_hsv_interpolation", + "invert_interpolation", + "use_xbox_shading", + ), + SInt8("minimum_meter_value"), + SInt16("sequence_index"), + SInt8("alpha_multiplier"), + SInt8("alpha_bias"), + SInt16("value_scale"), + Float("opacity"), + Float("translucency"), + UInt32("disabled_color", INCLUDE=argb_uint32), + ) + +shield_panel_meter = Struct("shield_panel_meter", + *meter_xform_common, + *meter_common, + Float("min_alpha"), + Pad(12), + UInt32("overcharge_minimum_color", INCLUDE=xrgb_uint32), + UInt32("overcharge_maximum_color", INCLUDE=xrgb_uint32), + UInt32("overcharge_flash_color", INCLUDE=xrgb_uint32), + UInt32("overcharge_empty_color", INCLUDE=xrgb_uint32), + Pad(16), + SIZE=136 + ) + +health_panel_meter = Struct("health_panel_meter", + *meter_xform_common, + *meter_common, + Float("min_alpha"), + Pad(12), + UInt32("medium_health_left_color", INCLUDE=xrgb_uint32), + Float("max_color_health_fraction_cutoff"), + Float("min_color_health_fraction_cutoff"), + Pad(20), + SIZE=136 + ) + +auxilary_meter = Struct("auxilary_meter", + Pad(18), + SEnum16("type", "integrated_light", VISIBLE=False), + Struct("background", INCLUDE=hud_background), + + *meter_xform_common, + *meter_common, + Float("min_alpha"), + Pad(12), + Float("minimum_fraction_cutoff"), + Bool32("overlay_flags", + "show_only_when_active", + "flash_once_if_activated_while_disabled", + ), + Pad(24), + Pad(64), + + SIZE=324, + COMMENT="\nThis auxilary meter is meant for the flashlight.\n" + ) + +unhi_body = desc_variant(unhi_body, + SEnum16("anchor", *hud_anchors_mcc), + shield_panel_meter, + health_panel_meter, + reflexive("auxilary_meters", auxilary_meter, 16), + ) + +def get(): + return unhi_def + +unhi_def = TagDef("unhi", + blam_header("unhi"), + unhi_body, + + ext=".unit_hud_interface", endian=">", tag_cls=HekTag, + ) diff --git a/reclaimer/mcc_hek/defs/unit.py b/reclaimer/mcc_hek/defs/unit.py new file mode 100644 index 00000000..523bed71 --- /dev/null +++ b/reclaimer/mcc_hek/defs/unit.py @@ -0,0 +1,36 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.unit import * + +metagame_scoring = Struct("metagame_scoring", + SEnum16("metagame_type", TOOLTIP="Used to determine score in MCC", *actor_types_mcc), + SEnum16("metagame_class", TOOLTIP="Used to determine score in MCC", *actor_classes_mcc), + Pad(8), + ) + +unit_attrs = desc_variant(unit_attrs, + ("pad_45", metagame_scoring), + SEnum16("grenade_type", *grenade_types_mcc), + ) + +unit_body = Struct('tagdata', + unit_attrs, + SIZE=372 + ) + +def get(): + return unit_def + +unit_def = TagDef("unit", + blam_header('unit', 2), + unit_body, + + ext=".unit", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/mcc_hek/defs/ustr.py b/reclaimer/mcc_hek/defs/ustr.py new file mode 100644 index 00000000..c012a53c --- /dev/null +++ b/reclaimer/mcc_hek/defs/ustr.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.ustr import * diff --git a/reclaimer/mcc_hek/defs/vcky.py b/reclaimer/mcc_hek/defs/vcky.py new file mode 100644 index 00000000..4d56b87f --- /dev/null +++ b/reclaimer/mcc_hek/defs/vcky.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.vcky import * diff --git a/reclaimer/mcc_hek/defs/vehi.py b/reclaimer/mcc_hek/defs/vehi.py new file mode 100644 index 00000000..e51c6ae7 --- /dev/null +++ b/reclaimer/mcc_hek/defs/vehi.py @@ -0,0 +1,57 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.vehi import * +from .obje import * +from .unit import * + +vehi_flags = Bool32("flags", + "speed_wakes_physics", + "turn_wakes_physics", + "driver_power_wakes_physics", + "gunner_power_wakes_physics", + "control_opposite_sets_brake", + "slide_wakes_physics", + "kills_riders_at_terminal_velocity", + "causes_collision_damage", + "ai_weapon_cannot_rotate", + "ai_does_not_require_driver", + "ai_unused", + "ai_driver_enable", + "ai_driver_flying", + "ai_driver_can_sidestep", + "ai_driver_hovering", + "vehicle_steers_directly", + "unused", + "has_e_brake", + "noncombat_vehicle", + "no_friction_with_driver", + "can_trigger_automatic_opening_doors", + "autoaim_when_teamless" + ) + +obje_attrs = obje_attrs_variant(obje_attrs, "vehi") +vehi_attrs = desc_variant(vehi_attrs, vehi_flags) + +vehi_body = Struct("tagdata", + obje_attrs, + unit_attrs, + vehi_attrs, + SIZE=1008, + ) + +def get(): + return vehi_def + +vehi_def = TagDef("vehi", + blam_header('vehi'), + vehi_body, + + ext=".vehicle", endian=">", tag_cls=ObjeTag + ) diff --git a/reclaimer/mcc_hek/defs/weap.py b/reclaimer/mcc_hek/defs/weap.py new file mode 100644 index 00000000..b7c9900b --- /dev/null +++ b/reclaimer/mcc_hek/defs/weap.py @@ -0,0 +1,68 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.weap import * +from .obje import * +from .item import * + +trigger_flags = Bool32("flags", + "tracks_fired_projectile", + "random_firing_effects", + "can_fire_with_partial_ammo", + "does_not_repeat_automatically", + "locks_in_on_off_state", + "projectiles_use_weapon_origin", + "sticks_when_dropped", + "ejects_during_chamber", + "discharging_spews", + "analog_rate_of_fire", + "use_error_when_unzoomed", + "projectile_vector_cannot_be_adjusted", + "projectiles_have_identical_error", + "projectile_is_client_side_only", + "use_unit_adjust_projectile_ray_from_halo1", + ) +mcc_upgrades = Struct("mcc_upgrades", + Pad(4), + SEnum16("prediction_type", + 'none', + 'continuous', + 'instant', + ), + SIZE=6 + ) + +firing = desc_variant(firing, + ("pad_9", mcc_upgrades) + ) +trigger = desc_variant(trigger, trigger_flags, firing) + +obje_attrs = obje_attrs_variant(obje_attrs, "weap") +weap_attrs = desc_variant(weap_attrs, + reflexive("triggers", trigger, 2, "primary", "secondary"), + SEnum16('weapon_type', *weapon_types_mcc) + ) + +weap_body = Struct("tagdata", + obje_attrs, + item_attrs, + weap_attrs, + SIZE=1288, + ) + + +def get(): + return weap_def + +weap_def = TagDef("weap", + blam_header('weap', 2), + weap_body, + + ext=".weapon", endian=">", tag_cls=WeapTag + ) diff --git a/reclaimer/mcc_hek/defs/wind.py b/reclaimer/mcc_hek/defs/wind.py new file mode 100644 index 00000000..721a2615 --- /dev/null +++ b/reclaimer/mcc_hek/defs/wind.py @@ -0,0 +1,10 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.wind import * diff --git a/reclaimer/mcc_hek/defs/wphi.py b/reclaimer/mcc_hek/defs/wphi.py new file mode 100644 index 00000000..2f234bfe --- /dev/null +++ b/reclaimer/mcc_hek/defs/wphi.py @@ -0,0 +1,100 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from ...hek.defs.wphi import * +from .grhi import multitex_overlay, mcc_hud_anchor +from .unhi import meter_xform_common, meter_common + +# to reduce a lot of code, these have been snipped out +element_common = ( + attached_state, + Pad(2), + use_on_map_type, + mcc_hud_anchor, + Pad(28), + ) +element_flash_common = ( + UInt32("default_color", INCLUDE=argb_uint32), + UInt32("flashing_color", INCLUDE=argb_uint32), + float_sec("flash_period"), + float_sec("flash_delay"), + SInt16("number_of_flashes"), + Bool16("flash_flags", *hud_flash_flags), + float_sec("flash_length"), + UInt32("disabled_color", INCLUDE=argb_uint32), + ) + +static_element = Struct("static_element", + *element_common, + *meter_xform_common, + dependency("interface_bitmap", "bitm"), + *element_flash_common, + + Pad(4), + SInt16("sequence_index"), + + Pad(2), + reflexive("multitex_overlays", multitex_overlay, 30), + SIZE=180 + ) + +meter_element = Struct("meter_element", + *element_common, + *meter_xform_common, + *meter_common, + SIZE=180 + ) + +number_element = Struct("number_element", + *element_common, + *meter_xform_common, + *element_flash_common, + + Pad(4), + SInt8("maximum_number_of_digits"), + Bool8("flags", + "show_leading_zeros", + "only_show_when_zoomed", + "draw_a_trailing_m", + ), + SInt8("number_of_fractional_digits"), + Pad(1), + + Pad(12), + Bool16("weapon_specific_flags", + "divide_number_by_magazine_size" + ), + SIZE=160 + ) + +overlay_element = Struct("overlay_element", + *element_common, + dependency("overlay_bitmap", "bitm"), + reflexive("overlays", overlay, 16), + SIZE=104 + ) + +wphi_body = desc_variant(wphi_body, + mcc_hud_anchor, + reflexive("static_elements", static_element, 16), + reflexive("meter_elements", meter_element, 16), + reflexive("number_elements", number_element, 16), + reflexive("overlay_elements", overlay_element, 16), + ) + + +def get(): + return wphi_def + +wphi_def = TagDef("wphi", + blam_header("wphi", 2), + wphi_body, + + ext=".weapon_hud_interface", endian=">", tag_cls=WphiTag, + ) diff --git a/reclaimer/mcc_hek/handler.py b/reclaimer/mcc_hek/handler.py new file mode 100644 index 00000000..3d838a86 --- /dev/null +++ b/reclaimer/mcc_hek/handler.py @@ -0,0 +1,17 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from pathlib import Path + +from reclaimer.hek.handler import HaloHandler +from reclaimer.mcc_hek.defs import __all__ as all_def_names + +class MCCHaloHandler(HaloHandler): + frozen_imp_paths = all_def_names + default_defs_path = "reclaimer.mcc_hek.defs" diff --git a/reclaimer/meta/class_repair.py b/reclaimer/meta/class_repair.py index ebb63043..08ef85e1 100644 --- a/reclaimer/meta/class_repair.py +++ b/reclaimer/meta/class_repair.py @@ -12,6 +12,7 @@ repair_dependency, repair_dependency_array from reclaimer.halo_script.hsc import get_hsc_data_block,\ HSC_IS_SCRIPT_OR_GLOBAL +from reclaimer.constants import * #from supyr_struct.util import * MAX_MATERIAL_COUNT = 33 @@ -84,6 +85,8 @@ def repair_item_attrs(offset, index_array, map_data, magic, repair, engine): def repair_unit_attrs(offset, index_array, map_data, magic, repair, engine): + is_yelo = engine == "halo1yelo" + # struct size is 372 args = (index_array, map_data, magic, repair, engine) repair_dependency(*(args + (b'effe', offset + 12))) @@ -119,7 +122,7 @@ def repair_unit_attrs(offset, index_array, map_data, magic, repair, engine): repair_dependency(*(args + (b'vtca', moff + 248))) - if "yelo" in engine: + if is_yelo: # seat extension for moff2 in iter_reflexive_offs(map_data, moff + 264 - magic, 100, 1, magic): # seat boarding @@ -134,7 +137,7 @@ def repair_unit_attrs(offset, index_array, map_data, magic, repair, engine): repair_dependency(*(args + (b'!tpj', moff3 + 4))) repair_dependency(*(args + (b'!tpj', moff3 + 96))) - if "yelo" in engine: + if is_yelo: # unit extension for moff in iter_reflexive_offs(map_data, offset + 288 - magic, 60, 1, magic): # mounted states @@ -167,12 +170,16 @@ def repair_ant_(tag_id, index_array, map_data, magic, repair, engine, safe_mode= def repair_antr(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): + is_mcc = engine == "halo1mcc" + + # sound references ct, moff, _ = read_reflexive( - map_data, index_array[tag_id].meta_offset + 0x54 - magic, 257, 20, magic) + map_data, index_array[tag_id].meta_offset + 0x54 - magic, + 257*(2 if is_mcc else 1), 20, magic + ) repair_dependency_array(index_array, map_data, magic, repair, engine, b'!dns', moff, ct, 20) - def repair_coll(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): tag_offset = index_array[tag_id].meta_offset args = (index_array, map_data, magic, repair, engine, b'effe') @@ -193,8 +200,15 @@ def repair_coll(tag_id, index_array, map_data, magic, repair, engine, safe_mode= def repair_cont(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): tag_offset = index_array[tag_id].meta_offset args = (index_array, map_data, magic, repair, engine) + is_yelo = engine == "halo1yelo" repair_dependency(*(args + (b'mtib', tag_offset + 0x30))) + # OS v4 shader extension + ct, _, __ = read_reflexive(map_data, tag_offset + 0xB4 - magic) + if is_yelo and ct > 1: + map_data.seek(tag_offset + 0xB4 - magic) + map_data.write(b'\x01\x00\x00\x00') + repair_dependency(*(args + (b'mtib', tag_offset + 0xD0))) # point states @@ -220,13 +234,12 @@ def repair_DeLa(tag_id, index_array, map_data, magic, repair, engine, safe_mode= repair_dependency(*(args + (b'!dns', moff + 24))) # search and replace functions - # Not needed anymore when tags are parsed in safe-mode - #ct, _, __ = read_reflexive(map_data, tag_offset + 96 - magic) - #if ct > 32: - # # some people apparently think its cute to set this reflexive - # # count so high so that tool just fails to compile the tag - # map_data.seek(tag_offset + 96 - magic) - # map_data.write(b'\x20\x00\x00\x00') + ct, _, __ = read_reflexive(map_data, tag_offset + 96 - magic) + if ct > 32: + # some people apparently think its cute to set this reflexive + # count so high so that tool just fails to compile the tag + map_data.seek(tag_offset + 96 - magic) + map_data.write(b'\x20\x00\x00\x00') repair_dependency(*(args + (b'rtsu', tag_offset + 236))) repair_dependency(*(args + (b'tnof', tag_offset + 252))) @@ -395,6 +408,8 @@ def repair_lsnd(tag_id, index_array, map_data, magic, repair, engine, safe_mode= def repair_matg(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): tag_offset = index_array[tag_id].meta_offset args = (index_array, map_data, magic, repair, engine) + is_yelo = engine == "halo1yelo" + is_mcc = engine == "halo1mcc" # sounds ct, moff, _ = read_reflexive(map_data, tag_offset + 0xF8 - magic, 2, 16, magic) @@ -405,7 +420,10 @@ def repair_matg(tag_id, index_array, map_data, magic, repair, engine, safe_mode= repair_dependency_array(*(args + (b'kart', moff, ct))) # grenades - for moff in iter_reflexive_offs(map_data, tag_offset + 0x128 - magic, 68, 4, magic): + for moff in iter_reflexive_offs( + map_data, tag_offset + 0x128 - magic, 68, + 4 if (is_yelo or is_mcc) else 2, magic + ): repair_dependency(*(args + (b'effe', moff + 4))) repair_dependency(*(args + (b'ihrg', moff + 20))) repair_dependency(*(args + (b'piqe', moff + 36))) @@ -554,10 +572,11 @@ def repair_object(tag_id, index_array, map_data, magic, repair, engine, safe_mod # not an object return + is_yelo = engine == "halo1yelo" # obje_attrs struct size is 380 args = (index_array, map_data, magic, repair, engine) repair_dependency(*(args + (b'2dom', tag_offset + 40))) - repair_dependency(*(args + (b'rtna', tag_offset + 56))) + repair_dependency(*(args + (None if is_yelo else b'rtna', tag_offset + 56))) repair_dependency(*(args + (b'lloc', tag_offset + 112))) repair_dependency(*(args + (b'syhp', tag_offset + 128))) @@ -607,7 +626,7 @@ def repair_object(tag_id, index_array, map_data, magic, repair, engine, safe_mod repair_dependency(*(args + (None, tag_offset + 280))) repair_dependency(*(args + (None, tag_offset + 296))) repair_dependency(*(args + (b'2dom', tag_offset + 340))) - repair_dependency(*(args + (b'rtna', tag_offset + 356))) + repair_dependency(*(args + (None if is_yelo else b'rtna', tag_offset + 356))) repair_dependency(*(args + (b'ihpw', tag_offset + 376))) repair_dependency(*(args + (b'!dns', tag_offset + 392))) repair_dependency(*(args + (b'!dns', tag_offset + 408))) @@ -623,7 +642,9 @@ def repair_object(tag_id, index_array, map_data, magic, repair, engine, safe_mod repair_dependency(*(args + (None, moff + 72))) # magazine items - ct, moff2, _ = read_reflexive(map_data, moff + 100 - magic, 2, 28, magic) + ct, moff2, _ = read_reflexive( + map_data, moff + 100 - magic, 8 if is_yelo else 2, 28, magic + ) repair_dependency_array(*(args + (b'piqe', moff2 + 12, ct, 28))) # triggers @@ -689,18 +710,27 @@ def repair_object(tag_id, index_array, map_data, magic, repair, engine, safe_mod def repair_part(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): tag_offset = index_array[tag_id].meta_offset args = (index_array, map_data, magic, repair, engine) + is_yelo = engine == "halo1yelo" repair_dependency(*(args + (b'mtib', tag_offset + 0x4))) repair_dependency(*(args + (b'yhpp', tag_offset + 0x14))) repair_dependency(*(args + (b'toof', tag_offset + 0x24))) repair_dependency(*(args + (None, tag_offset + 0x48))) repair_dependency(*(args + (None, tag_offset + 0x58))) + # OS v4 shader extension + ct, _, __ = read_reflexive(map_data, tag_offset + 0xE0 - magic) + if is_yelo and ct > 1: + map_data.seek(tag_offset + 0xE0 - magic) + map_data.write(b'\x01\x00\x00\x00') + repair_dependency(*(args + (b'mtib', tag_offset + 0xFC))) def repair_pctl(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): tag_offset = index_array[tag_id].meta_offset args = (index_array, map_data, magic, repair, engine) + is_yelo = engine == "halo1yelo" + repair_dependency(*(args + (b'yhpp', tag_offset + 56))) # particle types @@ -709,6 +739,12 @@ def repair_pctl(tag_id, index_array, map_data, magic, repair, engine, safe_mode= for moff2 in iter_reflexive_offs(map_data, moff + 116 - magic, 376, 8, magic): repair_dependency(*(args + (b'mtib', moff2 + 48))) repair_dependency(*(args + (b'yhpp', moff2 + 132))) + # OS v4 shader extension + ct, _, __ = read_reflexive(map_data, moff2 + 0xE8 - magic) + if is_yelo and ct > 1: + map_data.seek(moff2 + 0xE8 - magic) + map_data.write(b'\x01\x00\x00\x00') + repair_dependency(*(args + (b'mtib', moff2 + 260))) @@ -764,12 +800,13 @@ def repair_sbsp(tag_offset, index_array, map_data, magic, repair, engine, def repair_scnr(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): - ### Need to finish this up. not all the limits specified here - # should be as low as they are because open sauce is a thing tag_offset = index_array[tag_id].meta_offset args = (index_array, map_data, magic, repair, engine) - if "yelo" in engine: + is_yelo = engine == "halo1yelo" + is_mcc = engine == "halo1mcc" + + if is_yelo: repair_dependency(*(args + (b'oley', tag_offset))) # bsp modifiers @@ -818,7 +855,7 @@ def repair_scnr(tag_id, index_array, map_data, magic, repair, engine, safe_mode= # ai animation reference ct, moff, _ = read_reflexive(map_data, tag_offset + 1092 - magic, 128, 60, magic) - repair_dependency_array(*(args + (b'rtna', moff + 32, ct, 60))) + repair_dependency_array(*(args + (None if is_yelo else b'rtna', moff + 32, ct, 60))) # ai conversations for moff in iter_reflexive_offs(map_data, tag_offset + 1128 - magic, 116, 128, magic): @@ -832,7 +869,7 @@ def repair_scnr(tag_id, index_array, map_data, magic, repair, engine, safe_mode= repair_dependency(*(args + (b'!dns', moff2 + 108))) # tag references - ct, moff, _ = read_reflexive(map_data, tag_offset + 1204 - magic, 256, 40, magic) + ct, moff, _ = read_reflexive(map_data, tag_offset + 1204 - magic, 512 if is_mcc else 256, 40, magic) repair_dependency_array(*(args + (None, moff + 24, ct, 40))) # structure bsps @@ -842,11 +879,14 @@ def repair_scnr(tag_id, index_array, map_data, magic, repair, engine, safe_mode= # palettes # NOTE: Can't trust that these palettes are valid. # Need to check what the highest one used by all instances + max_pal_ct = 256 if is_mcc else 100 for off, inst_size in ( (540, 72), (564, 120), (588, 120), # scen bipd vehi (612, 40), (636, 92), (672, 64), # eqip weap mach (696, 64), (720, 88), (744, 40)): # ctrl lifi ssce - pal_ct, pal_moff, _ = read_reflexive(map_data, tag_offset + off - magic) + pal_ct, pal_moff, _ = read_reflexive( + map_data, tag_offset + off - magic, max_pal_ct + ) if safe_mode: used_pal_indices = set() @@ -867,7 +907,7 @@ def repair_scnr(tag_id, index_array, map_data, magic, repair, engine, safe_mode= # script syntax data references size, _, __, moff, ___ = read_rawdata_ref(map_data, tag_offset + 1140 - magic, magic) map_data.seek(moff - magic) - script_syntax_data_nodes = get_hsc_data_block(map_data.read(size)).nodes + script_syntax_data_nodes = get_hsc_data_block(map_data.read(size), engine).nodes for node in script_syntax_data_nodes: tag_cls = { 24: 'snd!', 25: 'effe', 26: 'jpt!', 27: 'lsnd', @@ -911,6 +951,7 @@ def repair_shader(tag_id, index_array, map_data, magic, repair, engine, safe_mod tag_offset = index_array[tag_id].meta_offset map_data.seek(tag_offset + 36 - magic) shader_type = int.from_bytes(map_data.read(2), 'little') + is_yelo = engine == "halo1yelo" if shader_type != -1 and shader_type not in range(len(shader_class_bytes) - 1): # not a shader @@ -931,7 +972,7 @@ def repair_shader(tag_id, index_array, map_data, magic, repair, engine, safe_mod repair_dependency(*(args + (b'mtib', tag_offset + 0x22C))) repair_dependency(*(args + (b'mtib', tag_offset + 0x2FC))) # shader environment os extension - if "yelo" in engine: + if is_yelo: ct, moff, _ = read_reflexive(map_data, tag_offset + 0xC8 - magic, 1, 100, magic) repair_dependency_array(*(args + (b'mtib', moff + 8, ct, 100))) @@ -941,7 +982,7 @@ def repair_shader(tag_id, index_array, map_data, magic, repair, engine, safe_mod repair_dependency(*(args + (b'mtib', tag_offset + 0xB4))) repair_dependency(*(args + (b'mtib', tag_offset + 0x13C))) # shader model os extension - if "yelo" in engine: + if is_yelo: for moff in iter_reflexive_offs( map_data, tag_offset + 0xC8 - magic, 192, 1, magic): repair_dependency(*(args + (b'mtib', moff))) @@ -993,9 +1034,10 @@ def repair_shader(tag_id, index_array, map_data, magic, repair, engine, safe_mod def repair_sky(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): tag_offset = index_array[tag_id].meta_offset args = (index_array, map_data, magic, repair, engine) + is_yelo = engine == "halo1yelo" repair_dependency(*(args + (b'2dom', tag_offset))) - repair_dependency(*(args + (b'rtna', tag_offset + 0x10))) + repair_dependency(*(args + (None if is_yelo else b'rtna', tag_offset + 0x10))) repair_dependency(*(args + (b' gof', tag_offset + 0x98))) # lights ct, moff, _ = read_reflexive(map_data, tag_offset + 0xC4 - magic, 8, 116, magic) @@ -1163,16 +1205,16 @@ def repair_avto(tag_id, index_array, map_data, magic, repair, engine, safe_mode= args = (index_array, map_data, magic, repair, engine) # instigators - for moff in iter_reflexive_offs(map_data, tag_offset + 44, 32, 16, magic): + for moff in iter_reflexive_offs(map_data, tag_offset + 52, 32, 16, magic): repair_dependency(*(args + (b'tinu', moff))) # keyframe actions - for moff in iter_reflexive_offs(map_data, tag_offset + 88, 72, 9, magic): + for moff in iter_reflexive_offs(map_data, tag_offset + 96, 72, 9, magic): repair_dependency(*(args + (b'!tpj', moff + 8))) repair_dependency(*(args + (b'effe', moff + 24))) # attachments - for moff in iter_reflexive_offs(map_data, tag_offset + 104, 120, 16, magic): + for moff in iter_reflexive_offs(map_data, tag_offset + 112, 120, 16, magic): repair_dependency(*(args + (b'ejbo', moff))) @@ -1189,7 +1231,7 @@ def repair_efpg(tag_id, index_array, map_data, magic, repair, engine, safe_mode= ct, moff, _ = read_reflexive( map_data, index_array[tag_id].meta_offset + 60 - magic, 12, 16, magic) repair_dependency_array( - index_array, map_data, magic, repair, engine, b'gphs', moff, ct) + index_array, map_data, magic, repair, engine, b'gphs', moff, ct, 16) def repair_gelc(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): @@ -1232,7 +1274,7 @@ def repair_gelo(tag_id, index_array, map_data, magic, repair, engine, safe_mode= def repair_magy(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): repair_antr(tag_id, index_array, map_data, magic, repair, engine) repair_dependency(index_array, map_data, magic, repair, engine, - b'rtna', index_array[tag_id].meta_offset + 0x80) + None, index_array[tag_id].meta_offset + 0x80) def repair_shpg(tag_id, index_array, map_data, magic, repair, engine, safe_mode=True): diff --git a/reclaimer/meta/halo1_map.py b/reclaimer/meta/halo1_map.py index 4ed29f6a..d40e6335 100644 --- a/reclaimer/meta/halo1_map.py +++ b/reclaimer/meta/halo1_map.py @@ -176,34 +176,21 @@ def tag_index_array_pointer(parent=None, new_value=None, magic=0, **kwargs): Pad(256), STEPTREE=Array("blocks", SIZE=".block_count", SUB_STRUCT=vap_block - ) + ), + SIZE=480 ) - # Halo Demo maps have a different header # structure with garbage filling the padding map_header_demo = Struct("map header", Pad(2), - UEnum16("map type", - "sp", - "mp", - "ui", - ), + gen1_map_type, # NOTE: in common_descs.py Pad(700), UEnum32('head', ('head', 'Ehed'), EDITABLE=False, DEFAULT='Ehed'), UInt32("tag data size"), ascii_str32("build date", EDITABLE=False), Pad(672), - UEnum32("version", - ("halo1xbox", 5), - ("halo1pcdemo", 6), - ("halo1pc", 7), - ("halo2", 8), - ("halo3beta", 9), - ("halo3", 11), - ("halo1ce", 609), - ("halo1vap", 134), - ), + map_version, # NOTE: in common_descs.py ascii_str32("map name"), UInt32("unknown"), UInt32("crc32"), @@ -217,16 +204,7 @@ def tag_index_array_pointer(parent=None, new_value=None, magic=0, **kwargs): map_header = Struct("map header", UEnum32('head', ('head', 'head'), DEFAULT='head'), - UEnum32("version", - ("halo1xbox", 5), - ("halo1pcdemo", 6), - ("halo1pc", 7), - ("halo2", 8), - ("halo3beta", 9), - ("halo3", 11), - ("halo1ce", 609), - ("halo1vap", 134), - ), + map_version, # NOTE: in common_descs.py UInt32("decomp len"), UInt32("unknown"), UInt32("tag index header offset"), @@ -234,31 +212,41 @@ def tag_index_array_pointer(parent=None, new_value=None, magic=0, **kwargs): Pad(8), ascii_str32("map name"), ascii_str32("build date", EDITABLE=False), - UEnum16("map type", - "sp", - "mp", - "ui", - ), + gen1_map_type, # NOTE: in common_descs.py Pad(2), UInt32("crc32"), - Pad(8), + Pad(1), + Pad(3), + Pad(4), yelo_header, UEnum32('foot', ('foot', 'foot'), DEFAULT='foot', OFFSET=2044), SIZE=2048 ) +mcc_flags = Bool8("mcc_flags", + "use_bitmaps_map", + "use_sounds_map", + "disable_remastered_sync", + ) + +map_header_mcc = desc_variant( + map_header, + ("pad_12", mcc_flags), + ) + map_header_vap = desc_variant( map_header, ("yelo_header", Struct("vap_header", INCLUDE=vap_header, OFFSET=128)), + verify=False, ) -tag_header = Struct("tag header", - UEnum32("class 1", GUI_NAME="primary tag class", INCLUDE=valid_tags_os), - UEnum32("class 2", GUI_NAME="secondary tag class", INCLUDE=valid_tags_os), - UEnum32("class 3", GUI_NAME="tertiary tag class", INCLUDE=valid_tags_os), +tag_header = Struct("tag_header", + UEnum32("class_1", GUI_NAME="primary tag class", INCLUDE=valid_tags_os), + UEnum32("class_2", GUI_NAME="secondary tag class", INCLUDE=valid_tags_os), + UEnum32("class_3", GUI_NAME="tertiary tag class", INCLUDE=valid_tags_os), UInt32("id"), - UInt32("path offset"), - UInt32("meta offset"), + UInt32("path_offset"), + UInt32("meta_offset"), UInt8("indexed"), Pad(3), # if indexed is non-zero, the meta_offset is the literal index in @@ -269,51 +257,33 @@ def tag_index_array_pointer(parent=None, new_value=None, magic=0, **kwargs): SIZE=32 ) -tag_index_array = TagIndex("tag index", +tag_index_array = TagIndex("tag_index", SIZE=".tag_count", SUB_STRUCT=tag_header, POINTER=tag_index_array_pointer ) -tag_index_xbox = Struct("tag index", - UInt32("tag index offset"), - UInt32("scenario tag id"), - UInt32("map id"), # normally unused, but can be used - # for spoofing the maps checksum. - UInt32("tag count"), - - UInt32("vertex parts count"), - UInt32("model data offset"), - - UInt32("index parts count"), - UInt32("index parts offset"), - UInt32("tag sig", EDITABLE=False, DEFAULT='tags'), - - SIZE=36, - STEPTREE=tag_index_array - ) - -tag_index_pc = Struct("tag index", - UInt32("tag index offset"), - UInt32("scenario tag id"), - UInt32("map id"), # normally unused, but can be used +tag_index_pc = Struct("tag_index", + UInt32("tag_index_offset"), + UInt32("scenario_tag_id"), + UInt32("map_id"), # normally unused, but can be used # for spoofing the maps checksum. - UInt32("tag count"), + UInt32("tag_count"), - UInt32("vertex parts count"), - UInt32("model data offset"), + UInt32("vertex_parts_count"), + UInt32("model_data_offset"), - UInt32("index parts count"), - UInt32("vertex data size"), - UInt32("model data size"), - UInt32("tag sig", EDITABLE=False, DEFAULT='tags'), + UInt32("index_parts_count"), + UInt32("index_parts_offset"), + UInt32("model_data_size"), + UInt32("tag_sig", EDITABLE=False, DEFAULT='tags'), SIZE=40, STEPTREE=tag_index_array ) -#tag_index_pc = tipc = dict(tag_index_xbox) -#tipc['ENTRIES'] += 1; tipc['SIZE'] += 4 -#tipc[7] = UInt32("vertex data size") -#tipc[9] = tipc[8]; tipc[8] = UInt32("model data size") +tag_index_xbox = desc_variant(tag_index_pc, + ("model_data_size", Pad(0)), + SIZE=36, verify=False + ) map_header_def = BlockDef(map_header) map_header_anni_def = BlockDef(map_header, endian=">") @@ -325,3 +295,4 @@ def tag_index_array_pointer(parent=None, new_value=None, magic=0, **kwargs): tag_index_anni_def = BlockDef(tag_index_pc, endian=">") map_header_vap_def = BlockDef(map_header_vap) +map_header_mcc_def = BlockDef(map_header_mcc) diff --git a/reclaimer/meta/halo1_map_fast_functions.py b/reclaimer/meta/halo1_map_fast_functions.py index 61f453e5..05c5b364 100644 --- a/reclaimer/meta/halo1_map_fast_functions.py +++ b/reclaimer/meta/halo1_map_fast_functions.py @@ -86,6 +86,9 @@ def read_reflexive(map_data, refl_offset, max_count=0xFFffFFff, map_data.seek(0, 2) max_count = min(max_count, (map_data.tell() - (start - tag_magic)) // struct_size) + if count > max_count: + print("Warning: Clipped %s reflexive size from %s to %s" % (count, max_count)) + return min(count, max_count), start, id @@ -173,8 +176,10 @@ def repair_dependency(index_array, map_data, tag_magic, repair, engine, cls, return cls = shader_class_bytes[shader_type] + elif cls == b'lcpw': + cls = b'cmti' elif cls in (b'2dom', b'edom'): - if "xbox" in engine: + if "xbox" in engine or "halo" not in engine: cls = b'edom' else: cls = b'2dom' diff --git a/reclaimer/meta/halo2_map.py b/reclaimer/meta/halo2_map.py index 3e0f7ae4..b48f3c1c 100644 --- a/reclaimer/meta/halo2_map.py +++ b/reclaimer/meta/halo2_map.py @@ -55,15 +55,7 @@ def string_id_table_name_pointer(parent=None, new_value=None, **kwargs): h2x_map_header = Struct("map header", UEnum32('head', ('head', 'head'), EDITABLE=False, DEFAULT='head'), - UEnum32("version", - ("halo1xbox", 5), - ("halo1pcdemo", 6), - ("halo1pc", 7), - ("halo2", 8), - ("halo3beta", 9), - ("halo3", 11), - ("halo1ce", 609), - ), + map_version, # NOTE: in common_descs.py UInt32("decomp len"), UInt32("unknown0"), UInt32("tag index header offset"), @@ -73,13 +65,7 @@ def string_id_table_name_pointer(parent=None, new_value=None, **kwargs): Pad(256), ascii_str32("build date"), - UEnum16("map type", - "sp", - "mp", - "ui", - "shared", - "sharedsp", - ), + gen2_map_type, # NOTE: in common_descs.py Pad(2), UInt32("crc32"), Pad(16), @@ -120,14 +106,7 @@ def string_id_table_name_pointer(parent=None, new_value=None, **kwargs): h2v_map_header = Struct("map header", UEnum32('head', ('head', 'head'), EDITABLE=False, DEFAULT='head'), - UEnum32("version", - ("halo1xbox", 5), - ("halo1pcdemo", 6), - ("halo1pc", 7), - ("halo2", 8), - ("halo3", 11), - ("halo1ce", 609), - ), + map_version, # NOTE: in common_descs.py UInt32("decomp len"), UInt32("unknown0"), UInt32("tag index header offset"), @@ -140,13 +119,7 @@ def string_id_table_name_pointer(parent=None, new_value=None, **kwargs): Pad(256), ascii_str32("build date"), - UEnum16("map type", - "sp", - "mp", - "ui", - "shared", - "sharedsp", - ), + gen2_map_type, # NOTE: in common_descs.py Pad(2), UInt32("crc32"), Pad(16), diff --git a/reclaimer/meta/halo3_map.py b/reclaimer/meta/halo3_map.py index eb7d150d..704ad10a 100644 --- a/reclaimer/meta/halo3_map.py +++ b/reclaimer/meta/halo3_map.py @@ -174,16 +174,7 @@ def root_tags_array_pointer(parent=None, new_value=None, **kwargs): UEnum32('head', ('head', 'head'), EDITABLE=False, VISIBLE=False, DEFAULT='head' ), - UEnum32("version", - ("halo1xbox", 5), - ("halo1pcdemo", 6), - ("halo1pc", 7), - ("halo2", 8), - ("halo3beta", 9), - ("halo3", 11), - ("haloreach", 12), - ("halo1ce", 609), - ), + map_version, # NOTE: in common_descs.py UInt32("decomp len"), UInt32("unknown0", VISIBLE=False), UInt32("tag index header offset"), @@ -192,13 +183,7 @@ def root_tags_array_pointer(parent=None, new_value=None, **kwargs): Pad(256), ascii_str32("build date"), - UEnum16("map type", - "sp", - "mp", - "ui", - "shared", - "sharedsp", - ), + gen2_map_type, # NOTE: in common_descs.py UInt16("unknown2", VISIBLE=False), UInt8("unknown3", VISIBLE=False), UInt8("unknown4", VISIBLE=False), diff --git a/reclaimer/meta/halo_map.py b/reclaimer/meta/halo_map.py index cdbc9034..eb1b7a84 100644 --- a/reclaimer/meta/halo_map.py +++ b/reclaimer/meta/halo_map.py @@ -15,8 +15,8 @@ from reclaimer.constants import map_build_dates, map_magics, GEN_3_ENGINES from reclaimer.meta.halo1_map import map_header_def, map_header_vap_def,\ - map_header_anni_def, map_header_demo_def, tag_index_pc_def,\ - tag_index_xbox_def, tag_index_anni_def + map_header_anni_def, map_header_demo_def, map_header_mcc_def,\ + tag_index_pc_def, tag_index_xbox_def, tag_index_anni_def from reclaimer.meta.halo2_alpha_map import h2_alpha_map_header_def,\ h2_alpha_tag_index_def from reclaimer.meta.halo2_map import h2v_map_header_full_def,\ @@ -24,7 +24,8 @@ h2_tag_index_def from reclaimer.meta.halo3_map import h3_map_header_def, h3_tag_index_def from reclaimer.meta.shadowrun_map import sr_tag_index_def -from reclaimer.meta.stubbs_map import stubbs_tag_index_def +from reclaimer.meta.stubbs_map import stubbs_tag_index_def,\ + stubbs_64bit_tag_index_def from supyr_struct.defs.tag_def import TagDef from supyr_struct.buffer import get_rawdata @@ -58,8 +59,9 @@ def get_map_version(header): # Xbox maps don't have a build date, but they do have this bit of data if header.unknown in (11, 1033): version = "halo1xboxdemo" - # Stubs PC headers match xbox headers without the unknown data - else: + else: # Stubs PC headers match xbox headers without the unknown data + # unfortunately, its impossible to tell apart 32 and 64bit stubbs + # maps from just the header alone. you need to look at the tag index version = "stubbspc" elif build_date == map_build_dates["shadowrun_proto"]: version = "shadowrun_proto" @@ -92,6 +94,19 @@ def get_map_version(header): return version +def get_engine_name(map_header, map_data): + # NOTE: this is basically just get_map_version, but with the ability to + # further refine the choice by looking at the rest of the map data. + engine = get_map_version(map_header) + if engine == "stubbspc": + map_data.seek(map_header.tag_index_header_offset + 52) + # thank god for the tags sig + if map_data.read(4) == b'sgat': + engine = "stubbspc64bit" + + return engine + + def get_map_header(map_file, header_only=False): if hasattr(map_file, "read"): orig_pos = map_file.tell() @@ -123,6 +138,9 @@ def get_map_header(map_file, header_only=False): elif ver_little in (5, 6, 609): header_def = map_header_def + elif ver_little == 13: + header_def = map_header_mcc_def + elif ver_little == 134: header_def = map_header_vap_def @@ -169,16 +187,20 @@ def get_tag_index(map_data, header=None): base_address = header.tag_index_header_offset tag_index_def = tag_index_pc_def - version = get_map_version(header) - if "shadowrun" in version: + engine = get_engine_name(header, map_data) + if "shadowrun" in engine: tag_index_def = sr_tag_index_def - elif "stubbs" in version: - tag_index_def = stubbs_tag_index_def + elif "stubbs" in engine: + tag_index_def = ( + stubbs_64bit_tag_index_def + if engine == "stubbspc64bit" else + stubbs_tag_index_def + ) elif header.version.data < 6: tag_index_def = tag_index_xbox_def - elif version == "halo2alpha": + elif engine == "halo2alpha": tag_index_def = h2_alpha_tag_index_def - elif version == "halo1anni": + elif engine == "halo1anni": tag_index_def = tag_index_anni_def elif header.version.enum_name == "halo2": tag_index_def = h2_tag_index_def @@ -227,9 +249,9 @@ def get_map_magic(header): def get_is_compressed_map(comp_data, header): if header.version.data == 134: return header.vap_header.compression_type.data != 0 - elif header.version.data not in (7, 609): + elif header.version.data not in (7, 13, 609): decomp_len = header.decomp_len - if get_map_version(header) == "pcstubbs": + if get_map_version(header) == "stubbspc": decomp_len -= 2048 return decomp_len > len(comp_data) @@ -296,7 +318,7 @@ def decompress_map_deflate(comp_data, header, decomp_path="", writable=False): decomp_start = 2048 decomp_len = header.decomp_len version = get_map_version(header) - if version == "pcstubbs": + if version == "stubbspc": decomp_len -= 2048 elif version == "halo2vista": decomp_start = 0 diff --git a/reclaimer/meta/objs/halo1_rsrc_map.py b/reclaimer/meta/objs/halo1_rsrc_map.py index 4624a927..b3c188d9 100644 --- a/reclaimer/meta/objs/halo1_rsrc_map.py +++ b/reclaimer/meta/objs/halo1_rsrc_map.py @@ -333,11 +333,15 @@ def _add_sound(self, tag_path, new_tag, base_pointer, depreciate): meta_head, samp_head = rsrc_tags[-1], rsrc_tags[-2] for pr in tag_data.pitch_ranges.STEPTREE: - pr.unknown0 = 1.0 + pr.playback_rate = 1.0 pr.unknown1 = pr.unknown2 = -1 for perm in pr.permutations.STEPTREE: - perm.unknown1 = 0 - perm.unknown3 = perm.unknown2 = 0xFFffFFff + perm.sample_data_pointer = perm.parent_tag_id = perm.unknown = 0 + if hasattr(perm, "runtime_flags"): # mcc + perm.runtime_flags = 0 + else: # non-mcc + perm.parent_tag_id2 = 0 + sample_data = perm.samples.data if perm.compression.enum_name == "none": sample_data = array(">h", sample_data) diff --git a/reclaimer/meta/shadowrun_map.py b/reclaimer/meta/shadowrun_map.py index e2c81231..265d0a11 100644 --- a/reclaimer/meta/shadowrun_map.py +++ b/reclaimer/meta/shadowrun_map.py @@ -7,46 +7,20 @@ # See LICENSE for more information. # -from reclaimer.meta.halo1_map import tag_path_pointer, tag_index_array_pointer +from reclaimer.meta.halo1_map import tag_path_pointer, tag_index_xbox,\ + tag_header as tag_index_header, tag_index_array_pointer from reclaimer.shadowrun_prototype.common_descs import * - -sr_tag_header = Struct("tag header", - UEnum32("class 1", GUI_NAME="primary tag class", INCLUDE=sr_valid_tags), - UEnum32("class 2", GUI_NAME="secondary tag class", INCLUDE=sr_valid_tags), - UEnum32("class 3", GUI_NAME="tertiary tag class", INCLUDE=sr_valid_tags), - UInt32("id"), - UInt32("path offset"), - UInt32("meta offset"), - UInt32("indexed"), - # if indexed is 1, the meta_offset is the literal index in the - # bitmaps, sounds, or loc cache that the meta data is located in. - Pad(4), - STEPTREE=CStrTagRef("path", POINTER=tag_path_pointer, MAX=768), - SIZE=32 +sr_tag_header = desc_variant(tag_index_header, + UEnum32("class_1", GUI_NAME="primary tag class", INCLUDE=sr_valid_tags), + UEnum32("class_2", GUI_NAME="secondary tag class", INCLUDE=sr_valid_tags), + UEnum32("class_3", GUI_NAME="tertiary tag class", INCLUDE=sr_valid_tags), ) - sr_tag_index_array = TagIndex("tag index", SIZE=".tag_count", SUB_STRUCT=sr_tag_header, POINTER=tag_index_array_pointer ) - -sr_tag_index = Struct("tag index", - UInt32("tag index offset"), - UInt32("scenario tag id"), - UInt32("map id"), # normally unused, but the scenario tag's header - # can be used for spoofing the maps checksum - UInt32("tag count"), - - UInt32("vertex parts count"), - UInt32("model data offset"), - - UInt32("index parts count"), - UInt32("index parts offset"), - UInt32("tag sig", EDITABLE=False, DEFAULT='tags'), - - SIZE=36, +sr_tag_index = desc_variant(tag_index_xbox, STEPTREE=sr_tag_index_array ) - sr_tag_index_def = BlockDef(sr_tag_index) diff --git a/reclaimer/meta/stubbs_map.py b/reclaimer/meta/stubbs_map.py index b8277344..46e5ddde 100644 --- a/reclaimer/meta/stubbs_map.py +++ b/reclaimer/meta/stubbs_map.py @@ -7,46 +7,55 @@ # See LICENSE for more information. # -from reclaimer.meta.halo1_map import tag_path_pointer, tag_index_array_pointer +from reclaimer.meta.halo1_map import tag_index_xbox, map_version,\ + tag_header as tag_index_header, tag_path_pointer, tag_index_array_pointer from reclaimer.stubbs.common_descs import * +stubbs_tag_index_header = desc_variant(tag_index_header, + UEnum32("class_1", GUI_NAME="primary tag class", INCLUDE=stubbs_valid_tags), + UEnum32("class_2", GUI_NAME="secondary tag class", INCLUDE=stubbs_valid_tags), + UEnum32("class_3", GUI_NAME="tertiary tag class", INCLUDE=stubbs_valid_tags), + ) -stubbs_tag_header = Struct("tag header", - UEnum32("class 1", GUI_NAME="primary tag class", INCLUDE=stubbs_valid_tags), - UEnum32("class 2", GUI_NAME="secondary tag class", INCLUDE=stubbs_valid_tags), - UEnum32("class 3", GUI_NAME="tertiary tag class", INCLUDE=stubbs_valid_tags), - UInt32("id"), - UInt32("path offset"), - UInt32("meta offset"), - UInt32("indexed"), - # if indexed is 1, the meta_offset is the literal index in the - # bitmaps, sounds, or loc cache that the meta data is located in. - Pad(4), - STEPTREE=CStrTagRef("path", POINTER=tag_path_pointer, MAX=768), - SIZE=32 +stubbs_64bit_tag_index_header = desc_variant(stubbs_tag_index_header, + Pointer64("path_offset"), + Pointer64("meta_offset"), + verify=False, + SIZE=40 ) stubbs_tag_index_array = TagIndex("tag index", - SIZE=".tag_count", SUB_STRUCT=stubbs_tag_header, + SIZE=".tag_count", SUB_STRUCT=stubbs_tag_index_header, POINTER=tag_index_array_pointer ) -stubbs_tag_index = Struct("tag index", - UInt32("tag index offset"), - UInt32("scenario tag id"), - UInt32("map id"), # normally unused, but the scenario tag's header - # can be used for spoofing the maps checksum - UInt32("tag count"), +stubbs_64bit_tag_index_array = TagIndex("tag index", + SIZE=".tag_count", SUB_STRUCT=stubbs_64bit_tag_index_header, + POINTER=tag_index_array_pointer + ) - UInt32("vertex parts count"), - UInt32("model data offset"), +stubbs_tag_index = desc_variant(tag_index_xbox, + STEPTREE=stubbs_tag_index_array + ) - UInt32("index parts count"), - UInt32("index parts offset"), - UInt32("tag sig", EDITABLE=False, DEFAULT='tags'), +stubbs_64bit_tag_index = Struct("tag_index", + Pointer64("tag_index_offset"), + UInt32("scenario_tag_id"), + UInt32("map_id"), # normally unused, but can be used + # for spoofing the maps checksum. + UInt32("tag_count"), - SIZE=36, - STEPTREE=stubbs_tag_index_array + UInt32("vertex_parts_count"), + Pointer64("model_data_offset"), + + UInt32("index_parts_count"), + Pad(4), + Pointer64("index_parts_offset"), + UInt32("model_data_size"), + UInt32("tag_sig", EDITABLE=False, DEFAULT='tags'), + STEPTREE=stubbs_64bit_tag_index_array, + SIZE=56 ) -stubbs_tag_index_def = BlockDef(stubbs_tag_index) +stubbs_tag_index_def = BlockDef(stubbs_tag_index) +stubbs_64bit_tag_index_def = BlockDef(stubbs_64bit_tag_index) diff --git a/reclaimer/meta/wrappers/byteswapping.py b/reclaimer/meta/wrappers/byteswapping.py index 2f10c82b..6a6a6467 100644 --- a/reclaimer/meta/wrappers/byteswapping.py +++ b/reclaimer/meta/wrappers/byteswapping.py @@ -14,7 +14,8 @@ ''' import array -from reclaimer.sounds.util import byteswap_pcm16_sample_data +from struct import Struct as PyStruct +from reclaimer.sounds import audioop from supyr_struct.field_types import BytearrayRaw from supyr_struct.defs.block_def import BlockDef @@ -24,12 +25,49 @@ except: fast_byteswapping = False +# These end_swap_XXXX functions are for byteswapping the +# endianness of values parsed from tags as the wrong order. +def end_swap_float(v, packer=PyStruct(">f").pack, + unpacker=PyStruct("= -0x80000000 and v < 0x80000000 + if v < 0: + v += 0x100000000 + v = ((((v << 24) + (v >> 24)) & 0xFF0000FF) + + ((v << 8) & 0xFF0000) + + ((v >> 8) & 0xFF00)) + if v & 0x80000000: + return v - 0x100000000 + return v + +def end_swap_int16(v): + assert v >= -0x8000 and v < 0x8000 + if v < 0: + v += 0x10000 + v = ((v << 8) + (v >> 8)) & 0xFFFF + if v & 0x8000: + return v - 0x10000 + return v + +def end_swap_uint32(v): + assert v >= 0 and v <= 0xFFFFFFFF + return ((((v << 24) + (v >> 24)) & 0xFF0000FF) + + ((v << 8) & 0xFF0000) + + ((v >> 8) & 0xFF00)) + +def end_swap_uint16(v): + assert v >= 0 and v <= 0xFFFF + return ((v << 8) + (v >> 8)) & 0xFFFF + raw_block_def = BlockDef("raw_block", BytearrayRaw('data', SIZE=lambda node, *a, **kw: 0 if node is None else len(node)) ) + def make_mutable_struct_array_copy(data, struct_size): valid_length = struct_size*(len(data)//struct_size) if valid_length == len(data): @@ -118,7 +156,8 @@ def byteswap_coll_bsp(bsp): def byteswap_pcm16_samples(pcm_block): # replace the verts with the byteswapped ones pcm_block.STEPTREE = bytearray( - byteswap_pcm16_sample_data(pcm_block.STEPTREE)) + audioop.byteswap(pcm_block.STEPTREE, 2) + ) def byteswap_sbsp_meta(meta): @@ -132,6 +171,100 @@ def byteswap_sbsp_meta(meta): byteswap_raw_reflexive(b) +def byteswap_anniversary_antr(meta): + # NOTE: don't need to byteswap the uncompresed animation data, as that's + # already handled by the non-anniversary antr byteswapping code. + + for b in meta.animations.STEPTREE: + b.first_permutation_index = end_swap_int16(b.first_permutation_index) + b.chance_to_play = end_swap_float(b.chance_to_play) + if not b.flags.compressed_data: + continue + + # slice out the compressed data and byteswap the + # 11 UInt32 that make up the 44 byte header that + # points to the + comp_data = bytearray(b.frame_data.data[b.offset_to_compressed_data: ]) + unswapped = bytes(comp_data) + byteswap_struct_array( + unswapped, comp_data, count=11, size=4, four_byte_offs=[0], + ) + header = PyStruct("<11i").unpack(comp_data[: 44]) + + # figure out where each array starts, ends, and the item size + starts = (0, *header) + ends = (*header, len(comp_data)) + widths = (4, 2, 2, 2, 4, 2, 4, 4, 4, 2, 4, 4) + + # byteswap each array + for start, end, width in zip(starts, ends, widths): + byteswap_struct_array( + unswapped, comp_data, size=width, + count = (end - start)//width, + four_byte_offs=([0] if width == 4 else []), + two_byte_offs=( [0] if width == 2 else []), + ) + + # replace the frame_data with the compressed data and some + # blank uncompressed default/frame data so tool doesnt cry + frame_data_size = b.frame_count * b.frame_size + default_data_size = b.node_count * (12 + 8 + 4) - b.frame_size + + b.offset_to_compressed_data = frame_data_size + + b.frame_data.data = bytearray(frame_data_size) + comp_data + b.default_data.data += bytearray( + max(0, len(b.default_data.data) - default_data_size) + ) + + +def byteswap_anniversary_sbsp(meta): + # make a copy of the nodes to byteswap to + orig = meta.nodes.STEPTREE + swapped = meta.nodes.STEPTREE = make_mutable_struct_array_copy(orig, 2) + byteswap_struct_array( + orig, swapped, size=2, + two_byte_offs=[0], + ) + + for lm in meta.lightmaps.STEPTREE: + for b in lm.materials.STEPTREE: + # logic is the same, so put comp/uncomp byteswap in a loop + for v_size, lm_v_size, rawdata_ref, sint16_offs in ( + [56, 20, b.uncompressed_vertices, ()], + [32, 8, b.compressed_vertices, (4, 6)] + ): + verts_size = v_size * b.vertices_count + lm_verts_size = lm_v_size * b.lightmap_vertices_count + + # make a copy of the verts to byteswap to + orig = rawdata_ref.STEPTREE[: verts_size+lm_verts_size] + swapped = rawdata_ref.STEPTREE = make_mutable_struct_array_copy(orig, 2) + + # render verts first + byteswap_struct_array( + orig, swapped, size=v_size, + start=0, count=b.vertices_count, + four_byte_offs=range(0, v_size, 4), + ) + + # followed by lightmap verts + byteswap_struct_array( + orig, swapped, size=lm_v_size, + start=verts_size, count=b.lightmap_vertices_count, + four_byte_offs=range(0, lm_v_size, 4), + two_byte_offs=sint16_offs, + ) + + +def byteswap_anniversary_rawdata_ref(rawdata_ref, **kwargs): + if rawdata_ref.size: + orig = rawdata_ref.serialize(attr_index="data") + swapped = bytearray(orig) + byteswap_struct_array(orig, swapped, **kwargs) + rawdata_ref.parse(rawdata=swapped, attr_index="data") + + def byteswap_scnr_script_syntax_data(meta): original = meta.script_syntax_data.data swapped = original[: ((len(original)-56)//20) * 20 + 56] @@ -174,7 +307,7 @@ def byteswap_comp_verts(verts_block): byteswap_struct_array( original, swapped, 32, None, 0, - two_byte_offs=(24, 26, 28, 30), + two_byte_offs=(24, 26, 30), four_byte_offs=(0, 4, 8, 12, 16, 20) ) diff --git a/reclaimer/meta/wrappers/halo1_anni_map.py b/reclaimer/meta/wrappers/halo1_anni_map.py index 017aa2a0..3c3da47e 100644 --- a/reclaimer/meta/wrappers/halo1_anni_map.py +++ b/reclaimer/meta/wrappers/halo1_anni_map.py @@ -8,12 +8,17 @@ # from math import pi, sqrt, log -from struct import Struct as PyStruct from traceback import format_exc +from reclaimer.meta.wrappers.byteswapping import byteswap_anniversary_sbsp,\ + byteswap_anniversary_antr, byteswap_anniversary_rawdata_ref,\ + byteswap_scnr_script_syntax_data, end_swap_float, end_swap_int32,\ + end_swap_int16, end_swap_uint32, end_swap_uint16 + +from reclaimer.misc.defs.recorded_animations import build_r_a_stream_block from reclaimer.meta.wrappers.halo_map import HaloMap from reclaimer.meta.wrappers.halo1_rsrc_map import Halo1RsrcMap -from reclaimer.meta.wrappers.halo1_map import Halo1Map +from reclaimer.meta.wrappers.halo1_mcc_map import Halo1MccMap from reclaimer import data_extraction from reclaimer.halo_script.hsc import h1_script_syntax_data_def from reclaimer.hek.defs.coll import fast_coll_def @@ -26,90 +31,34 @@ __all__ = ("Halo1AnniMap",) -def end_swap_float(v, packer=PyStruct(">f").pack, - unpacker=PyStruct("= -0x80000000 and v < 0x80000000 - if v < 0: - v += 0x100000000 - v = ((((v << 24) + (v >> 24)) & 0xFF0000FF) + - ((v << 8) & 0xFF0000) + - ((v >> 8) & 0xFF00)) - if v & 0x80000000: - return v - 0x100000000 - return v - - -def end_swap_int16(v): - assert v >= -0x8000 and v < 0x8000 - if v < 0: - v += 0x10000 - v = ((v << 8) + (v >> 8)) & 0xFFFF - if v & 0x8000: - return v - 0x10000 - return v - - -def end_swap_uint32(v): - assert v >= 0 and v <= 0xFFFFFFFF - return ((((v << 24) + (v >> 24)) & 0xFF0000FF) + - ((v << 8) & 0xFF0000) + - ((v >> 8) & 0xFF00)) - - -def end_swap_uint16(v): - assert v >= 0 and v <= 0xFFFF - return ((v << 8) + (v >> 8)) & 0xFFFF - - -class Halo1AnniMap(Halo1Map): +class Halo1AnniMap(Halo1MccMap): tag_headers = None + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. defs = None handler_class = HaloHandler - inject_rawdata = Halo1RsrcMap.inject_rawdata - - def __init__(self, maps=None): - HaloMap.__init__(self, maps) - self.setup_tag_headers() - - def get_dependencies(self, meta, tag_id, tag_cls): - if self.is_indexed(tag_id): - if tag_cls != "snd!": - return () - - rsrc_id = meta.promotion_sound.id & 0xFFff - if rsrc_id == 0xFFFF: return () - - sounds = self.maps.get("sounds") - rsrc_id = rsrc_id // 2 - if sounds is None: return () - elif rsrc_id >= len(sounds.tag_index.tag_index): return () - - tag_path = sounds.tag_index.tag_index[rsrc_id].path - inv_snd_map = getattr(self, 'ce_tag_indexs_by_paths', {}) - tag_id = inv_snd_map.get(tag_path, 0xFFFF) - if tag_id >= len(self.tag_index.tag_index): return () - - return [self.tag_index.tag_index[tag_id]] - - if self.handler is None: return () - - dependency_cache = self.handler.tag_ref_cache.get(tag_cls) - if not dependency_cache: return () - - nodes = self.handler.get_nodes_by_paths(dependency_cache, (None, meta)) - dependencies = [] - - for node in nodes: - if node.id & 0xFFff == 0xFFFF: - continue - dependencies.append(node) - return dependencies + @property + def uses_bitmaps_map(self): return False + @property + def uses_sounds_map(self): return False + + def is_indexed(self, tag_id): + return False + + def setup_sbsp_headers(self): + with FieldType.force_big: + super().setup_sbsp_headers() + + def setup_rawdata_pages(self): + # NOTE: for some reason, anniversary maps have overlap between the + # sbsp virtual address range and the tag data address range. + # because of this, we don't setup any pages in the default + # pointer converter for sbsp + # NOTE: we also don't setup pages for the model data section, since + # it's also overlapping with the tag data address range. + pass def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): ''' @@ -138,7 +87,6 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): if self.get_meta_descriptor(tag_cls) is None: return - if tag_cls is None: # couldn't determine the tag class return @@ -186,47 +134,17 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): return meta def byteswap_anniversary_fields(self, meta, tag_cls): + # fix all the forced-little-endian fields that are big-endian, but were + # read as little-endian(that's what FlUInt/FlSInt/FlFloat fields are) + # TODO: use handler build_loc_caches to locate all forced-little-endian + # fields and force them to big-endian without having to write these if tag_cls == "antr": - unpack_header = PyStruct("<11i").unpack for b in meta.animations.STEPTREE: - b.unknown_sint16 = end_swap_int16(b.unknown_sint16) - b.unknown_float = end_swap_float(b.unknown_float) - if not b.flags.compressed_data: - continue - - comp_data = b.frame_data.data[b.offset_to_compressed_data: ] - # byteswap compressed frame data header - for i in range(0, 44, 4): - data = comp_data[i: i + 4] - for j in range(4): - comp_data[i + 3 - j] = data[j] - - header = list(unpack_header(comp_data[: 44])) - header.insert(0, 44) - header.append(len(comp_data)) - - item_sizes = (4, 2, 2, 2, 4, 2, 4, 4, 4, 2, 4, 4) - comp_data_off_len_size = [] - for i in range(len(header) - 1): - comp_data_off_len_size.append([ - header[i], header[i + 1] - header[i], item_sizes[i]]) - - for off, length, size in comp_data_off_len_size: - for i in range(off, off + length, size): - data = comp_data[i: i + size] - for j in range(size): - comp_data[i + size - 1 - j] = data[j] - - # replace the frame_data with the compressed data and some - # blank default / frame data so tool doesnt shit the bed. - default_data_size = b.node_count * (12 + 8 + 4) - b.frame_size - b.default_data.data += bytearray( - max(0, len(b.default_data.data) - default_data_size)) - - b.offset_to_compressed_data = b.frame_count * b.frame_size - b.frame_data.data = bytearray( - b.frame_count * b.frame_size) + comp_data + b.first_permutation_index = end_swap_int16(b.first_permutation_index) + b.chance_to_play = end_swap_float(b.chance_to_play) + + byteswap_anniversary_antr(meta) elif tag_cls == "bitm": for b in meta.bitmaps.STEPTREE: @@ -234,8 +152,8 @@ def byteswap_anniversary_fields(self, meta, tag_cls): elif tag_cls == "coll": for b in meta.nodes.STEPTREE: - b.unknown0 = end_swap_int16(b.unknown0) - b.unknown1 = end_swap_int16(b.unknown1) + b.unknown = end_swap_int16(b.unknown) + b.damage_region = end_swap_int16(b.damage_region) elif tag_cls == "effe": for event in meta.events.STEPTREE: @@ -244,17 +162,11 @@ def byteswap_anniversary_fields(self, meta, tag_cls): b.unknown1 = end_swap_int16(b.unknown1) elif tag_cls == "hmt ": - block_bytes = bytearray(meta.string.serialize()) - for i in range(20, len(block_bytes), 2): - byte = block_bytes[i + 1] - block_bytes[i + 1] = block_bytes[i] - block_bytes[i] = byte - - meta.string.parse(rawdata=block_bytes) + byteswap_anniversary_rawdata_ref(meta.string, size=2, two_byte_offs=[0]) elif tag_cls == "lens": - meta.unknown0 = end_swap_float(meta.unknown0) - meta.unknown1 = end_swap_float(meta.unknown1) + meta.cosine_falloff_angle = end_swap_float(meta.cosine_falloff_angle) + meta.cosine_cutoff_angle = end_swap_float(meta.cosine_cutoff_angle) elif tag_cls == "lsnd": meta.unknown0 = end_swap_float(meta.unknown0) @@ -263,7 +175,7 @@ def byteswap_anniversary_fields(self, meta, tag_cls): meta.unknown3 = end_swap_float(meta.unknown3) meta.unknown4 = end_swap_int16(meta.unknown4) meta.unknown5 = end_swap_int16(meta.unknown5) - meta.unknown6 = end_swap_float(meta.unknown6) + meta.max_distance = end_swap_float(meta.max_distance) elif tag_cls == "metr": meta.screen_x_pos = end_swap_uint16(meta.screen_x_pos) @@ -273,11 +185,11 @@ def byteswap_anniversary_fields(self, meta, tag_cls): elif tag_cls in ("mod2", "mode"): for node in meta.nodes.STEPTREE: - node.unknown = end_swap_float(node.unknown) + node.scale = end_swap_float(node.scale) for b in (node.rot_jj_kk, node.rot_kk_ii, node.rot_ii_jj, node.translation_to_root): - for i in range(len(b)): - b[i] = end_swap_float(b[i]) + for i, val in enumerate(b): + b[i] = end_swap_float(val) elif tag_cls == "part": meta.rendering.unknown0 = end_swap_int32(meta.rendering.unknown0) @@ -285,175 +197,122 @@ def byteswap_anniversary_fields(self, meta, tag_cls): meta.rendering.unknown2 = end_swap_uint32(meta.rendering.unknown2) elif tag_cls == "pphy": - meta.scaled_density = end_swap_float(meta.scaled_density) + meta.scaled_density = end_swap_float(meta.scaled_density) meta.water_gravity_scale = end_swap_float(meta.water_gravity_scale) - meta.air_gravity_scale = end_swap_float(meta.air_gravity_scale) + meta.air_gravity_scale = end_swap_float(meta.air_gravity_scale) elif tag_cls == "sbsp": - # TODO: Might need to byteswap cluster data and sound_pas data + for b in meta.collision_materials.STEPTREE: + b.material_type.data = end_swap_int16(b.material_type.data) + + for b in meta.fog_planes.STEPTREE: + b.material_type.data = end_swap_int16(b.material_type.data) + + for lm in meta.lightmaps.STEPTREE: + for b in lm.materials.STEPTREE: + b.unknown_meta_offset0 = end_swap_uint32(b.unknown_meta_offset0) + b.unknown_meta_offset1 = end_swap_uint32(b.unknown_meta_offset1) + b.vertices_meta_offset = end_swap_uint32(b.vertices_meta_offset) + b.lightmap_vertices_meta_offset = end_swap_uint32(b.lightmap_vertices_meta_offset) + + # byteswap the rawdata + byteswap_anniversary_sbsp(meta) - for coll_mat in meta.collision_materials.STEPTREE: - coll_mat.unknown = end_swap_uint32(coll_mat.unknown) - - node_data = meta.nodes.STEPTREE - for i in range(0, len(node_data), 2): - b0 = node_data[i] - node_data[i] = node_data[i + 1] - node_data[i + 1] = b0 - - leaf_data = meta.leaves.STEPTREE - for i in range(0, len(leaf_data), 16): - b0 = leaf_data[i] - leaf_data[i] = leaf_data[i + 1] - leaf_data[i + 1] = b0 - - b0 = leaf_data[i + 2] - leaf_data[i + 2] = leaf_data[i + 3] - leaf_data[i + 3] = b0 - - b0 = leaf_data[i + 4] - leaf_data[i + 4] = leaf_data[i + 5] - leaf_data[i + 5] = b0 - - b0 = leaf_data[i + 6] - leaf_data[i + 6] = leaf_data[i + 7] - leaf_data[i + 7] = b0 - - for lightmap in meta.lightmaps.STEPTREE: - for b in lightmap.materials.STEPTREE: - vt_ct = b.vertices_count - l_vt_ct = b.lightmap_vertices_count - - u_verts = b.uncompressed_vertices.STEPTREE - c_verts = b.compressed_vertices.STEPTREE - - b.unknown_meta_offset0 = end_swap_uint32( - b.unknown_meta_offset0) - b.vertices_meta_offset = end_swap_uint32( - b.vertices_meta_offset) - - b.vertex_type.data = end_swap_uint16(b.vertex_type.data) - - b.unknown_meta_offset1 = end_swap_uint32( - b.unknown_meta_offset1) - b.lightmap_vertices_meta_offset = end_swap_uint32( - b.lightmap_vertices_meta_offset) - - # byteswap (un)compressed verts and lightmap verts - for data in (u_verts, c_verts): - for i in range(0, len(data), 4): - b0 = data[i] - b1 = data[i+1] - data[i] = data[i+3] - data[i+1] = data[i+2] - data[i+2] = b1 - data[i+3] = b0 - - # since the compressed lightmap u and v coordinates are - # 2 byte fields rather than 4, the above byteswapping - # will have swapped u and v. we need to swap them back. - # multiply vt_ct by 32 to skip non-lightmap verts, and - # add 4 to skip the 4 byte compressed lightmap normal. - for i in range(vt_ct * 32 + 4, len(c_verts), 8): - c_verts[i: i + 1] = c_verts[i+1], c_verts[i] - - for fog_plane in meta.fog_planes.STEPTREE: - fog_plane.material_type.data = end_swap_int16( - fog_plane.material_type.data) + # TODO: Might need to byteswap cluster data and sound_pas data elif tag_cls == "scnr": for b in meta.object_names.STEPTREE: - b.object_type.data = end_swap_uint16(b.object_type.data) - b.reflexive_index = end_swap_int16(b.reflexive_index) + b.object_type.data = end_swap_int16(b.object_type.data) + b.reflexive_index = end_swap_int16(b.reflexive_index) for b in meta.trigger_volumes.STEPTREE: - b.unknown = end_swap_uint16(b.unknown) + b.unknown0 = end_swap_uint16(b.unknown0) for b in meta.encounters.STEPTREE: - b.unknown = end_swap_int16(b.unknown) + b.unknown = end_swap_uint16(b.unknown) - # PROLLY GONNA HAVE TO BYTESWAP RECORDED ANIMS AND MORE SHIT - syntax_data = meta.script_syntax_data.data - with FieldType.force_big: - syntax_header = h1_script_syntax_data_def.build(rawdata=syntax_data) - - i = 56 - for node_i in range(syntax_header.last_node): - n_typ = syntax_data[i + 5] + (syntax_data[i + 4] << 8) - flags = syntax_data[i + 7] + (syntax_data[i + 6] << 8) - if flags & 7 == 1: - # node is a primitive - if n_typ == 5: - # node is a boolean - syntax_data[i + 19] = syntax_data[i + 16] - syntax_data[i + 16: i + 19] = (0, 0, 0) # null these 3 - elif n_typ == 7: - # node is a sint16 - syntax_data[i + 18] = syntax_data[i + 16] - syntax_data[i + 19] = syntax_data[i + 17] - syntax_data[i + 16: i + 18] = (0, 0) # null these 2 - - i += 20 + for ra in meta.recorded_animations.STEPTREE: + # parse the recorded animations as big-endian + # and serialize back as little-endian + try: + with FieldType.force_big: + ra_block = build_r_a_stream_block( + ra.unit_control_data_version, + ra.recorded_animation_event_stream.STEPTREE, + simple=True + ) + ra.recorded_animation_event_stream.STEPTREE = ra_block.serialize() + except Exception: + print(format_exc()) + print("Could not byteswap recorded animation '%s'" % ra.name) + + # NOTE: this is gonna get swapped back when converting to tagdata + byteswap_scnr_script_syntax_data(meta) elif tag_cls == "senv": - meta.senv_attrs.bump_properties.map_scale_x = end_swap_float( - meta.senv_attrs.bump_properties.map_scale_x) - meta.senv_attrs.bump_properties.map_scale_y = end_swap_float( - meta.senv_attrs.bump_properties.map_scale_y) + bump_props = meta.senv_attrs.bump_properties + bump_props.map_scale_x = end_swap_float(bump_props.map_scale_x) + bump_props.map_scale_y = end_swap_float(bump_props.map_scale_y) elif tag_cls == "snd!": for pr in meta.pitch_ranges.STEPTREE: for b in pr.permutations.STEPTREE: - b.ogg_sample_count = end_swap_uint32(b.ogg_sample_count) + b.buffer_size = end_swap_uint32(b.buffer_size) elif tag_cls == "spla": - meta.spla_attrs.primary_noise_map.unknown0 = end_swap_uint16( - meta.spla_attrs.primary_noise_map.unknown0) - meta.spla_attrs.primary_noise_map.unknown1 = end_swap_uint16( - meta.spla_attrs.primary_noise_map.unknown1) - - meta.spla_attrs.secondary_noise_map.unknown0 = end_swap_uint16( - meta.spla_attrs.secondary_noise_map.unknown0) - meta.spla_attrs.secondary_noise_map.unknown1 = end_swap_uint16( - meta.spla_attrs.secondary_noise_map.unknown1) + for noise_map in ( + meta.spla_attrs.primary_noise_map, + meta.spla_attrs.secondary_noise_map + ): + noise_map.unknown0 = end_swap_uint16(noise_map.unknown0) + noise_map.unknown1 = end_swap_uint16(noise_map.unknown1) elif tag_cls == "ustr": + # need to serialize the unicode strings reflexive back to the + # endianness it was read as, and then byteswap the code-points + # of each character(NOTE: 12 is the end of the refelxive header) for b in meta.strings.STEPTREE: - block_bytes = bytearray(b.serialize()) - for i in range(12, len(block_bytes), 2): - byte = block_bytes[i + 1] - block_bytes[i + 1] = block_bytes[i] - block_bytes[i] = byte - - b.parse(rawdata=block_bytes) - + byteswap_anniversary_rawdata_ref(b, size=2, two_byte_offs=[0]) if tag_cls in ("bipd", "vehi", "weap", "eqip", "garb", "proj", "scen", "mach", "ctrl", "lifi", "plac", "ssce", "obje"): meta.obje_attrs.object_type.data = end_swap_int16( - meta.obje_attrs.object_type.data) + meta.obje_attrs.object_type.data + ) elif tag_cls in ("senv", "soso", "sotr", "schi", "scex", "swat", "sgla", "smet", "spla", "shdr"): meta.shdr_attrs.shader_type.data = end_swap_int16( - meta.shdr_attrs.shader_type.data) + meta.shdr_attrs.shader_type.data + ) def inject_rawdata(self, meta, tag_cls, tag_index_ref): - pass + # TODO: Update this with extracting from sabre paks if + # bitmap/sound/model extraction is ever implemented + if tag_cls == "snd!": + # audio samples are ALWAYS in fmod, so fill the with empty padding + for pitches in meta.pitch_ranges.STEPTREE: + for perm in pitches.permutations.STEPTREE: + for b in (perm.samples, perm.mouth_data, perm.subtitle_data): + b.data = b"\x00"*b.size + elif tag_cls == "bitm": + # bitmap pixels are ALWAYS in saber paks, so fill the with empty padding + meta.compressed_color_plate_data.data = b"\x00"*meta.processed_pixel_data.size + meta.processed_pixel_data.data = b"\x00"*meta.processed_pixel_data.size + else: + meta = super().inject_rawdata(meta, tag_cls, tag_index_ref) - def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): - kwargs["byteswap"] = False - Halo1Map.meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs) + return meta + def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): + # no bitmap pixels or sound samples in map. cant extract. + # also, we don't know how to properly byteswap recorded + # animations, so scenarios can't be extracted properly. + if tag_cls == "bitm": + raise ValueError("Bitmap pixel data missing.") + elif tag_cls == "snd!": + raise ValueError("Sound sample data missing.") - # TODO: Remove this if bitmap/sound extraction is ever implemented - if tag_cls in ("snd!", "bitm"): - # these tags don't properly extract due to missing - # pixel data and sound permutation sample data - return - elif tag_cls in ("sbsp", "scnr"): - # renderable geometry is absent from h1 anniversary bsps, and - # we don't know how to properly byteswap recorded animations, - # so scenarios can't be extracted properly either. - return + kwargs["byteswap"] = False + super().meta_to_tag_data(meta, tag_cls, tag_index_ref, **kwargs) return meta diff --git a/reclaimer/meta/wrappers/halo1_map.py b/reclaimer/meta/wrappers/halo1_map.py index bd15ac93..37663674 100644 --- a/reclaimer/meta/wrappers/halo1_map.py +++ b/reclaimer/meta/wrappers/halo1_map.py @@ -8,11 +8,13 @@ # import os +import sys +from array import array as PyArray from copy import deepcopy from math import pi, log from pathlib import Path -from struct import unpack, pack_into +from struct import unpack, unpack_from, pack_into from traceback import format_exc from types import MethodType @@ -20,13 +22,15 @@ from supyr_struct.field_types import FieldType from supyr_struct.defs.frozen_dict import FrozenDict +from reclaimer.halo_script.hsc_decompilation import extract_scripts from reclaimer.halo_script.hsc import get_hsc_data_block,\ - HSC_IS_SCRIPT_OR_GLOBAL, SCRIPT_OBJECT_TYPES_TO_SCENARIO_REFLEXIVES + get_script_syntax_node_tag_refs, clean_script_syntax_nodes,\ + get_script_types, HSC_IS_SCRIPT_OR_GLOBAL,\ + SCRIPT_OBJECT_TYPES_TO_SCENARIO_REFLEXIVES from reclaimer.common_descs import make_dependency_os_block from reclaimer.hek.defs.snd_ import snd__meta_stub_blockdef from reclaimer.hek.defs.sbsp import sbsp_meta_header_def from reclaimer.hek.handler import HaloHandler -from reclaimer.os_hek.defs.gelc import gelc_def from reclaimer.os_v4_hek.defs.coll import fast_coll_def from reclaimer.os_v4_hek.defs.sbsp import fast_sbsp_def from reclaimer.meta.wrappers.byteswapping import raw_block_def, byteswap_animation,\ @@ -38,10 +42,10 @@ from reclaimer.meta.wrappers.map_pointer_converter import MapPointerConverter from reclaimer.meta.wrappers.tag_index_manager import TagIndexManager from reclaimer import data_extraction -from reclaimer.constants import tag_class_fcc_to_ext +from reclaimer.constants import tag_class_fcc_to_ext, GEN_1_HALO_CUSTOM_ENGINES from reclaimer.util.compression import compress_normal32, decompress_normal32 from reclaimer.util import is_overlapping_ranges, is_valid_ascii_name_str,\ - int_to_fourcc + int_to_fourcc, get_block_max from supyr_struct.util import is_path_empty @@ -93,8 +97,14 @@ class Halo1Map(HaloMap): bsp_headers = () bsp_header_offsets = () bsp_pointer_converters = () + + sbsp_meta_header_def = sbsp_meta_header_def data_extractors = data_extraction.h1_data_extractors + + indexable_tag_classes = set(( + "bitm", "snd!", "font", "hmt ", "ustr" + )) def __init__(self, maps=None): HaloMap.__init__(self, maps) @@ -111,14 +121,53 @@ def __init__(self, maps=None): self.setup_tag_headers() + @property + def globals_tag_id(self): + if not self.tag_index: + return None + + for b in self.tag_index.tag_index: + if int_to_fourcc(b.class_1.data) == "matg": + return b.id & 0xFFff + + @property + def resource_map_prefix(self): + return "" + @property + def resource_maps_folder(self): + return self.filepath.parent + @property + def resources_maps_mismatched(self): + maps_dir = self.resource_maps_folder + if not maps_dir: + return False + + for map_name, filepath in self.get_resource_map_paths().items(): + if filepath and filepath.parent != maps_dir: + return True + return False + @property + def uses_bitmaps_map(self): + return not self.is_resource + @property + def uses_loc_map(self): + return not self.is_resource and "pc" not in self.engine + @property + def uses_sounds_map(self): + return not self.is_resource + @property def decomp_file_ext(self): - if self.engine == "halo1yelo": - return ".yelo" - elif self.engine == "halo1vap": - return ".vap" - else: - return ".map" + return ( + ".vap" if self.engine == "halo1vap" else + self._decomp_file_ext + ) + + def is_indexed(self, tag_id): + tag_header = self.tag_index.tag_index[tag_id] + if not tag_header.indexed: + return False + return int_to_fourcc(tag_header.class_1.data) in self.indexable_tag_classes def setup_defs(self): this_class = type(self) @@ -131,7 +180,6 @@ def setup_defs(self): this_class.defs = dict(this_class.handler.defs) this_class.defs["coll"] = fast_coll_def - this_class.defs["gelc"] = gelc_def this_class.defs["sbsp"] = fast_sbsp_def this_class.defs = FrozenDict(this_class.defs) @@ -149,7 +197,7 @@ def ensure_sound_maps_valid(self): return self.sound_rsrc_id = id(sounds) - if self.engine in ("halo1ce", "halo1yelo", "halo1vap"): + if self.engine in GEN_1_HALO_CUSTOM_ENGINES: # ce resource sounds are recognized by tag_path # so we must cache their offsets by their paths rsrc_snd_map = self.ce_rsrc_sound_indexes_by_path = {} @@ -205,26 +253,28 @@ def get_dependencies(self, meta, tag_id, tag_cls): dependencies = [] for node in nodes: - if node.id & 0xFFff == 0xFFFF: + # need to filter to dependencies that are actually valid + tag_id = node.id & 0xFFff + if tag_id not in range(len(tag_index_array)): continue - dependencies.append(node) + + tag_index_ref = tag_index_array[tag_id] + if (node.tag_class.enum_name == tag_index_ref.class_1.enum_name and + node.id == tag_index_ref.id): + dependencies.append(node) if tag_cls == "scnr": # collect the tag references from the scenarios syntax data try: seen_tag_ids = set() syntax_data = get_hsc_data_block(meta.script_syntax_data.data) - for node in syntax_data.nodes: - if (node.flags & HSC_IS_SCRIPT_OR_GLOBAL or - node.type not in range(24, 32)): - # not a tag index ref - continue - + for node in get_script_syntax_node_tag_refs(syntax_data): tag_index_id = node.data & 0xFFff if (tag_index_id in range(len(tag_index_array)) and tag_index_id not in seen_tag_ids): seen_tag_ids.add(tag_index_id) tag_index_ref = tag_index_array[tag_index_id] + dependencies.append(make_dependency_os_block( tag_index_ref.class_1.enum_name, tag_index_ref.id, tag_index_ref.path, tag_index_ref.path_offset)) @@ -269,25 +319,54 @@ def setup_sbsp_pointer_converters(self): i += 1 - # read the sbsp headers - for tag_id, offset in self.bsp_header_offsets.items(): - if self.engine == "halo1anni": - with FieldType.force_big: - header = sbsp_meta_header_def.build( - rawdata=self.map_data, offset=offset) - else: - header = sbsp_meta_header_def.build( - rawdata=self.map_data, offset=offset) - - if header.sig != header.get_desc("DEFAULT", "sig"): - print("Sbsp header is invalid for '%s'" % - self.tag_index.tag_index[tag_id].path) - self.bsp_headers[tag_id] = header - self.tag_index.tag_index[tag_id].meta_offset = header.meta_pointer + self.setup_sbsp_headers() except Exception: print(format_exc()) + def setup_sbsp_headers(self): + # read the sbsp headers + for tag_id, offset in self.bsp_header_offsets.items(): + header = self.sbsp_meta_header_def.build( + rawdata=self.map_data, offset=offset) + + if header.sig != header.get_desc("DEFAULT", "sig"): + print("Sbsp header is invalid for '%s'" % + self.tag_index.tag_index[tag_id].path) + self.bsp_headers[tag_id] = header + self.tag_index.tag_index[tag_id].meta_offset = header.meta_pointer + + def setup_rawdata_pages(self): + tag_index = self.tag_index + + last_bsp_end = 0 + # calculate the start of the rawdata section + for tag_id in self.bsp_headers: + bsp_end = self.bsp_header_offsets[tag_id] + self.bsp_sizes[tag_id] + if last_bsp_end < bsp_end: + last_bsp_end = bsp_end + + # add the rawdata section + self.map_pointer_converter.add_page_info( + last_bsp_end, last_bsp_end, + tag_index.model_data_offset - last_bsp_end, + ) + + # add the model data section + if hasattr(tag_index, "model_data_size"): + # PC tag index + self.map_pointer_converter.add_page_info( + 0, tag_index.model_data_offset, + tag_index.model_data_size, + ) + else: + # XBOX tag index + self.map_pointer_converter.add_page_info( + 0, tag_index.model_data_offset, + (self.map_header.tag_index_header_offset - + tag_index.model_data_offset), + ) + def load_map(self, map_path, **kwargs): HaloMap.load_map(self, map_path, **kwargs) @@ -318,44 +397,11 @@ def load_map(self, map_path, **kwargs): print("Could not read scenario tag") self.setup_sbsp_pointer_converters() - - last_bsp_end = 0 - # calculate the start of the rawdata section - for tag_id in self.bsp_headers: - bsp_end = self.bsp_header_offsets[tag_id] + self.bsp_sizes[tag_id] - if last_bsp_end < bsp_end: - last_bsp_end = bsp_end - - # add the rawdata section - self.map_pointer_converter.add_page_info( - last_bsp_end, last_bsp_end, - tag_index.model_data_offset - last_bsp_end, - ) - - # add the model data section - if tag_index.SIZE == 40: - # PC tag index - self.map_pointer_converter.add_page_info( - 0, tag_index.model_data_offset, - tag_index.model_data_size, - ) - else: - # XBOX tag index - self.map_pointer_converter.add_page_info( - 0, tag_index.model_data_offset, - (self.map_header.tag_index_header_offset - - tag_index.model_data_offset), - ) + self.setup_rawdata_pages() # get the globals meta try: - matg_id = None - for b in tag_index_array: - if int_to_fourcc(b.class_1.data) == "matg": - matg_id = b.id & 0xFFff - break - - self.matg_meta = self.get_meta(matg_id) + self.matg_meta = self.get_meta(self.globals_tag_id) if self.matg_meta is None: print("Could not read globals tag") except Exception: @@ -369,6 +415,15 @@ def load_map(self, map_path, **kwargs): self.clear_map_cache() + if self.resources_maps_mismatched and kwargs.get("unlink_mismatched_resources", True): + # this map reference different resource maps depending on what + # folder its located in. we need to ignore any resource maps + # passed in unless they're in the same folder as this map. + print("Unlinking potentially incompatible resource maps from %s" % + self.map_name + ) + self.maps = {} + def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): ''' Takes a tag reference id as the sole argument. @@ -377,20 +432,19 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): if tag_id is None: return - magic = self.map_magic - engine = self.engine - map_data = self.map_data - tag_index = self.tag_index - tag_index_array = tag_index.tag_index - - # if we are given a 32bit tag id, mask it off - tag_id &= 0xFFFF tag_index_ref = self.tag_index_manager.get_tag_index_ref(tag_id) if tag_index_ref is None: return + # if we are given a 32bit tag id, mask it off + tag_id &= 0xFFFF + magic = self.map_magic + engine = self.engine + map_data = self.map_data + tag_cls = None - if tag_id == (tag_index.scenario_tag_id & 0xFFFF): + is_scenario = (tag_id == (self.tag_index.scenario_tag_id & 0xFFFF)) + if is_scenario: tag_cls = "scnr" elif tag_index_ref.class_1.enum_name not in ("", "NONE"): tag_cls = int_to_fourcc(tag_index_ref.class_1.data) @@ -413,18 +467,20 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): tag_id = offset rsrc_map = None - if tag_cls == "snd!" and "sounds" in self.maps: - rsrc_map = self.maps["sounds"] - sound_mapping = self.ce_rsrc_sound_indexes_by_path - tag_path = tag_index_ref.path - if sound_mapping is None or tag_path not in sound_mapping: - return + if tag_cls == "snd!": + if "sounds" in self.maps: + rsrc_map = self.maps["sounds"] + sound_mapping = self.ce_rsrc_sound_indexes_by_path + tag_path = tag_index_ref.path + if sound_mapping is None or tag_path not in sound_mapping: + return - tag_id = sound_mapping[tag_path]//2 + tag_id = sound_mapping[tag_path]//2 - elif tag_cls == "bitm" and "bitmaps" in self.maps: - rsrc_map = self.maps["bitmaps"] - tag_id = tag_id//2 + elif tag_cls == "bitm": + if "bitmaps" in self.maps: + rsrc_map = self.maps["bitmaps"] + tag_id = tag_id//2 elif "loc" in self.maps: rsrc_map = self.maps["loc"] @@ -439,12 +495,12 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): return meta = rsrc_map.get_meta(tag_id, **kw) + snd_stub = None if tag_cls == "snd!": - # since we're reading the resource tag from the perspective of - # the map referencing it, we have more accurate information - # about which other sound it could be referencing. This is only - # a concern when dealing with open sauce resource maps, as they - # could have additional promotion sounds we cant statically map + # while the sound samples and complete tag are in the + # resource map, the metadata for the body of the sound + # tag is in the main map. Need to copy its values into + # the resource map sound tag we extracted. try: # read the meta data from the map with FieldType.force_little: @@ -452,13 +508,28 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): rawdata=map_data, offset=pointer_converter.v_ptr_to_f_ptr(offset), tag_index_manager=self.tag_index_manager) - meta.promotion_sound = snd_stub.promotion_sound except Exception: print(format_exc()) + if snd_stub: + # copy values over + for name in ( + "flags", "sound_class", "sample_rate", + "minimum_distance", "maximum_distance", + "skip_fraction", "random_pitch_bounds", + "inner_cone_angle", "outer_cone_angle", + "outer_cone_gain", "gain_modifier", + "maximum_bend_per_second", + "modifiers_when_scale_is_zero", + "modifiers_when_scale_is_one", + "encoding", "compression", "promotion_sound", + "promotion_count", "max_play_length", + ): + setattr(meta, name, getattr(snd_stub, name)) + return meta elif not reextract: - if tag_id == tag_index.scenario_tag_id & 0xFFff and self.scnr_meta: + if is_scenario and self.scnr_meta: return self.scnr_meta elif tag_cls == "matg" and self.matg_meta: return self.matg_meta @@ -477,7 +548,7 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): map_pointer_converter=pointer_converter, offset=pointer_converter.v_ptr_to_f_ptr(offset), tag_index_manager=self.tag_index_manager, - safe_mode=not kw.get("disable_safe_mode"), + safe_mode=(self.safe_mode and not kw.get("disable_safe_mode")), parsing_resource=force_parsing_rsrc) except Exception: print(format_exc()) @@ -486,6 +557,7 @@ def get_meta(self, tag_id, reextract=False, ignore_rsrc_sounds=False, **kw): meta = block[0] try: + # TODO: remove this dirty-ass hack if tag_cls == "bitm" and get_is_xbox_map(engine): for bitmap in meta.bitmaps.STEPTREE: # make sure to set this for all xbox bitmaps @@ -534,10 +606,9 @@ def clean_tag_meta(self, meta, tag_id, tag_cls): next_anim = animations[next_anim].next_animation anims_to_remove = [] - + max_anim_count = get_block_max(meta.animations) for i in range(len(animations)): - if self.engine != "halo1yelo" and i >= 256: - # cap it to the non-OS limit of 256 animations + if i >= max_anim_count: break anim = animations[i] @@ -629,6 +700,21 @@ def clean_tag_meta(self, meta, tag_id, tag_cls): except Exception: print("Couldn't read animation data.") + elif tag_cls == "bitm": + bitmaps = [b for b in meta.bitmaps.STEPTREE + if "dxt" in b.format.enum_name] + # correct mipmap count on xbox dxt bitmaps. texels for any + # mipmaps whose dimensions are 2x2 or smaller are pruned + for bitmap in bitmaps: + # figure out largest dimension(clip to 1 to avoid log(0, 2)) + max_dim = max(1, bitmap.width, bitmap.height) + + # subtract 2 to account for width/height of 1 or 2 not having mips + maxmips = int(max(0, log(max_dim, 2) - 2)) + + # clip mipmap count to max and min number that can exist + bitmap.mipmaps = max(0, min(maxmips, bitmap.mipmaps)) + elif tag_cls in ("sbsp", "coll"): if tag_cls == "sbsp" : bsps = meta.collision_bsp.STEPTREE @@ -638,23 +724,24 @@ def clean_tag_meta(self, meta, tag_id, tag_cls): bsps.extend(node.bsps.STEPTREE) for bsp in bsps: - highest_used_vert = -1 - edge_data = bsp.edges.STEPTREE vert_data = bsp.vertices.STEPTREE - for i in range(0, len(edge_data), 24): - v0_i = (edge_data[i] + - (edge_data[i + 1] << 8) + - (edge_data[i + 2] << 16) + - (edge_data[i + 3] << 24)) - v1_i = (edge_data[i + 4] + - (edge_data[i + 5] << 8) + - (edge_data[i + 6] << 16) + - (edge_data[i + 7] << 24)) - highest_used_vert = max(highest_used_vert, v0_i, v1_i) - - if highest_used_vert * 16 < len(vert_data): - del vert_data[(highest_used_vert + 1) * 16: ] - bsp.vertices.size = highest_used_vert + 1 + # first 2 ints in each edge are the vert indices, and theres + # 6 int32s per edge. find the highest vert index being used + if bsp.edges.STEPTREE: + byteorder = 'big' if self.engine == "halo1anni" else 'little' + + edges = PyArray("i", bsp.edges.STEPTREE) + if byteorder != sys.byteorder: + edges.byteswap() + + max_start_vert = max(edges[0: len(edges): 6]) + max_end_vert = max(edges[1: len(edges): 6]) + else: + max_start_vert = max_end_vert = -1 + + if max_start_vert * 16 < len(vert_data): + del vert_data[(max_start_vert + 1) * 16: ] + bsp.vertices.size = max_start_vert + 1 elif tag_cls in ("mode", "mod2"): used_shaders = set() @@ -737,6 +824,10 @@ def clean_tag_meta(self, meta, tag_id, tag_cls): syntax_data = get_hsc_data_block(meta.script_syntax_data.data) script_nodes_modified = False + # lets not use magic numbers here + _, script_object_types = get_script_types(self.engine) + biped_node_enum = script_object_types.index("actor_type") + # clean up any fucked up palettes for pal_block, inst_block in ( (meta.sceneries_palette, meta.sceneries), @@ -759,8 +850,7 @@ def clean_tag_meta(self, meta, tag_id, tag_cls): # determine which palette indices are used by script data for i in range(len(syntax_data.nodes)): node = syntax_data.nodes[i] - # 35 == "actor_type" script type - if node.type == 35 and not(node.flags & HSC_IS_SCRIPT_OR_GLOBAL): + if node.type == biped_node_enum and not(node.flags & HSC_IS_SCRIPT_OR_GLOBAL): script_nodes_to_modify.add(i) used_pal_indices.add(node.data & 0xFFff) @@ -847,62 +937,30 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): for perm in change_color.permutations.STEPTREE: perm.weight, cutoff = perm.weight - cutoff, perm.weight - if tag_cls == "actv": # multiply grenade velocity by 30 meta.grenades.grenade_velocity *= 30 elif tag_cls in ("antr", "magy"): + # try to fix HEK+ extraction bug + for obj in meta.objects.STEPTREE: + for enum in (obj.function, obj.function_controls): + uint16_data = enum.data & 0xFFff + if (uint16_data & 0xFF00 and not uint16_data & 0xFF): + # higher bits are set than lower. this is likely + # a HEK plus extraction bug and should be fixed + uint16_data = ((uint16_data>>8) | (uint16_data<<8)) & 0xFFff + enum.data = uint16_data - ( + 0 if uint16_data < 0x8000 else 0x10000 + ) + # byteswap animation data for anim in meta.animations.STEPTREE: if not byteswap: break byteswap_animation(anim) - elif tag_cls == "bitm": - # set the size of the compressed plate data to nothing - meta.compressed_color_plate_data.STEPTREE = BytearrayBuffer() - - # to enable compatibility with my bitmap converter we'll set the - # base address to a certain constant based on the console platform - is_xbox = get_is_xbox_map(engine) - - new_pixels_offset = 0 - - # uncheck the prefer_low_detail flag and - # set up the pixels_offset correctly. - for bitmap in meta.bitmaps.STEPTREE: - bitmap.flags.prefer_low_detail = is_xbox - bitmap.pixels_offset = new_pixels_offset - new_pixels_offset += bitmap.pixels_meta_size - - # clear some meta-only fields - bitmap.pixels_meta_size = 0 - bitmap.bitmap_id_unknown1 = bitmap.bitmap_id_unknown2 = 0 - bitmap.bitmap_data_pointer = 0 - - if is_xbox: - bitmap.base_address = 1073751810 - if "dxt" in bitmap.format.enum_name: - # need to correct mipmap count on xbox dxt bitmaps. - # the game seems to prune the mipmap texels for any - # mipmaps whose dimensions are 2x2 or smaller - - max_dim = max(bitmap.width, bitmap.height) - if 2 ** bitmap.mipmaps > max_dim: - # make sure the mipmap level isnt higher than the - # number of mipmaps that should be able to exist. - bitmap.mipmaps = int(log(max_dim, 2)) - - last_mip_dim = max_dim // (2 ** bitmap.mipmaps) - if last_mip_dim == 1: - bitmap.mipmaps -= 2 - elif last_mip_dim == 2: - bitmap.mipmaps -= 1 - - if bitmap.mipmaps < 0: - bitmap.mipmaps = 0 - else: - bitmap.base_address = 0 + elif tag_cls in ("bitm", "snd!"): + meta = Halo1RsrcMap.meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs) elif tag_cls == "cdmg": # divide camera shaking wobble period by 30 @@ -917,7 +975,10 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): elif tag_cls == "effe": # mask away the meta-only flags - meta.flags.data &= 3 + # NOTE: xbox has a cache flag in the 2nd + # bit, so it should be masked out too. + meta.flags.data &= (1 if "xbox" in engine else 3) + for event in meta.events.STEPTREE: # tool exceptions if any parts reference a damage effect # tag type, but have an empty filepath for the reference @@ -945,7 +1006,11 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): elif tag_cls == "lens": # DON'T multiply corona rotation by pi/180 # reminder that this is not supposed to be changed - pass # meta.corona_rotation.function_scale *= pi/180 + + if meta.corona_rotation.function_scale == 360.0: + # fix a really old bug(i think its the + # reason the above comment was created) + meta.corona_rotation.function_scale = 0.0 elif tag_cls == "ligh": # divide light time by 30 @@ -972,80 +1037,83 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): meta.stencil_bitmap.filepath = meta.source_bitmap.filepath = '' elif tag_cls in ("mode", "mod2"): - if engine in ("halo1yelo", "halo1ce", "halo1pc", "halo1vap", - "halo1anni", "halo1pcdemo", "stubbspc"): + if engine in ("halo1yelo", "halo1ce", "halo1pc", "halo1vap", "halo1mcc", + "halo1anni", "halo1pcdemo", "stubbspc", "stubbspc64bit"): # model_magic seems to be the same for all pc maps verts_start = tag_index.model_data_offset - tris_start = verts_start + tag_index.vertex_data_size + tris_start = verts_start + tag_index.index_parts_offset model_magic = None else: model_magic = magic + + # need to unset this flag, as it forces map-compile-time processing + # to occur on the model's vertices, which shouldn't be done twice. + meta.flags.blend_shared_normals = False + + # lod cutoffs are swapped between tag and cache form + cutoffs = (meta.superlow_lod_cutoff, meta.low_lod_cutoff, + meta.high_lod_cutoff, meta.superhigh_lod_cutoff) + meta.superlow_lod_cutoff = cutoffs[3] + meta.low_lod_cutoff = cutoffs[2] + meta.high_lod_cutoff = cutoffs[1] + meta.superhigh_lod_cutoff = cutoffs[0] + + # localize the global markers + # ensure all local marker arrays are empty + for region in meta.regions.STEPTREE: + for perm in region.permutations.STEPTREE: + del perm.local_markers.STEPTREE[:] + + for g_marker in meta.markers.STEPTREE: + for g_marker_inst in g_marker.marker_instances.STEPTREE: + try: + region = meta.regions.STEPTREE[g_marker_inst.region_index] + except IndexError: + print("Model marker instance for", g_marker.name, "has invalid region index", g_marker_inst.region_index, "and is skipped.") + continue - if model_magic is None: - verts_attr_name = "uncompressed_vertices" - byteswap_verts = byteswap_uncomp_verts - vert_size = 68 + try: + perm = region.permutations.STEPTREE[g_marker_inst.permutation_index] + except IndexError: + print("Model marker instance for", g_marker.name, "has invalid permutation index", g_marker_inst.permutation_index, "and is skipped.") + continue - if engine != "stubbspc": - # need to swap the lod cutoff and nodes values around - cutoffs = (meta.superlow_lod_cutoff, meta.low_lod_cutoff, - meta.high_lod_cutoff, meta.superhigh_lod_cutoff) - meta.superlow_lod_cutoff = cutoffs[3] - meta.low_lod_cutoff = cutoffs[2] - meta.high_lod_cutoff = cutoffs[1] - meta.superhigh_lod_cutoff = cutoffs[0] + # make a new local marker + perm.local_markers.STEPTREE.append() + l_marker = perm.local_markers.STEPTREE[-1] - else: - verts_attr_name = "compressed_vertices" - byteswap_verts = byteswap_comp_verts - vert_size = 32 - - # If this is a gbxmodel localize the markers. - # We skip this for xbox models for arsenic. - - if tag_cls == "mod2": - # ensure all local marker arrays are empty - for region in meta.regions.STEPTREE: - for perm in region.permutations.STEPTREE: - del perm.local_markers.STEPTREE[:] - - # localize the global markers - for g_marker in meta.markers.STEPTREE: - for g_marker_inst in g_marker.marker_instances.STEPTREE: - try: - region = meta.regions.STEPTREE[g_marker_inst.region_index] - except IndexError: - print("Model marker instance for", g_marker.name, "has invalid region index", g_marker_inst.region_index, "and is skipped.") - continue - - try: - perm = region.permutations.STEPTREE[g_marker_inst.permutation_index] - except IndexError: - print("Model marker instance for", g_marker.name, "has invalid permutation index", g_marker_inst.permutation_index, "and is skipped.") - continue - - # make a new local marker - perm.local_markers.STEPTREE.append() - l_marker = perm.local_markers.STEPTREE[-1] - - # copy the global marker into the local - l_marker.name = g_marker.name - l_marker.node_index = g_marker_inst.node_index - l_marker.translation[:] = g_marker_inst.translation[:] - l_marker.rotation[:] = g_marker_inst.rotation[:] - - # clear the global markers - del meta.markers.STEPTREE[:] + # copy the global marker into the local + l_marker.name = g_marker.name + l_marker.node_index = g_marker_inst.node_index + l_marker.translation[:] = g_marker_inst.translation[:] + l_marker.rotation[:] = g_marker_inst.rotation[:] + + # clear the global markers + del meta.markers.STEPTREE[:] # grab vertices and indices from the map for geom in meta.geometries.STEPTREE: for part in geom.parts.STEPTREE: - verts_block = part[verts_attr_name] tris_block = part.triangles - info = part.model_meta_info + info = part.model_meta_info + + if info.vertex_type.enum_name == "model_comp_verts": + verts_block = part.compressed_vertices + byteswap_verts = byteswap_comp_verts + vert_size = 32 + elif info.vertex_type.enum_name == "model_uncomp_verts": + verts_block = part.uncompressed_vertices + byteswap_verts = byteswap_uncomp_verts + vert_size = 68 + else: + print("Error: Unknown vertex type in model: %s" % info.vertex_type.data) + continue + + if info.index_type.enum_name != "triangle_strip": + print("Error: Unknown index type in model: %s" % info.index_type.data) + continue # null out certain things in the part - part.previous_part_index = part.next_part_index = 0 part.centroid_primary_node = 0 part.centroid_secondary_node = 0 part.centroid_primary_weight = 0.0 @@ -1056,10 +1124,7 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): tris_block.STEPTREE = raw_block_def.build() # read the offsets of the vertices and indices from the map - if engine == "stubbspc": - verts_off = verts_start + info.vertices_reflexive_offset - tris_off = tris_start + info.indices_reflexive_offset - elif model_magic is None: + if model_magic is None: verts_off = verts_start + info.vertices_offset tris_off = tris_start + info.indices_offset else: @@ -1118,6 +1183,10 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): # need to scale velocities by 30 meta.proj_attrs.physics.initial_velocity *= 30 meta.proj_attrs.physics.final_velocity *= 30 + meta.proj_attrs.detonation.minimum_velocity *= 30 + for material_response in meta.proj_attrs.material_responses.STEPTREE: + material_response.potential_response.impact_velocity[0] *= 30 + material_response.potential_response.impact_velocity[1] *= 30 elif tag_cls == "sbsp": if byteswap: @@ -1128,127 +1197,127 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): for cluster in meta.clusters.STEPTREE: predicted_resources.append(cluster.predicted_resources) - - compressed = "xbox" in engine or engine in ("stubbs", "shadowrun_proto") - - if compressed: - generate_verts = kwargs.get("generate_uncomp_verts", False) - else: - generate_verts = kwargs.get("generate_comp_verts", False) - - endian = "<" - if engine == "halo1anni": - endian = ">" + + for coll_mat in meta.collision_materials.STEPTREE: + coll_mat.material_type.data = 0 # supposed to be 0 in tag form comp_norm = compress_normal32 decomp_norm = decompress_normal32 - comp_vert_nbt_unpacker = MethodType(unpack, endian + "3I") - uncomp_vert_nbt_packer = MethodType(pack_into, endian + "12s9f8s") + comp_vert_unpacker = MethodType(unpack_from, "<12s3I8s") + comp_vert_packer = MethodType(pack_into, "<12s3I8s") + uncomp_vert_unpacker = MethodType(unpack_from, "<12s9f8s") + uncomp_vert_packer = MethodType(pack_into, "<12s9f8s") - comp_vert_nuv_unpacker = MethodType(unpack, endian + "I2h") - uncomp_vert_nuv_packer = MethodType(pack_into, endian + "5f") - - uncomp_vert_nbt_unpacker = MethodType(unpack, endian + "9f") - comp_vert_nbt_packer = MethodType(pack_into, endian + "12s3I8s") - - uncomp_vert_nuv_unpacker = MethodType(unpack, endian + "5f") - comp_vert_nuv_packer = MethodType(pack_into, endian + "I2h") + comp_lm_vert_unpacker = MethodType(unpack_from, " 1 else u)*32767), + int((-1 if v < -1 else 1 if v > 1 else v)*32767), + ) + elif (kwargs.get("generate_uncomp_verts") and + lm_vert_type == "sbsp_comp_lightmap_verts" + ): + # generate uncompressed lightmap verts from compressed + u_buffer += bytearray(u_lm_verts_size) + for u_off, c_off in lm_vert_offs: + n, u, v = comp_lm_vert_unpacker(c_buffer, c_off) + uncomp_lm_vert_packer( + u_buffer, u_off, + *decomp_norm(n), u/32767, v/32767 + ) + + # need to null these or original CE sapien could crash + mat.unknown_meta_offset0 = mat.vertices_meta_offset = 0 + mat.unknown_meta_offset1 = mat.lightmap_vertices_meta_offset = 0 + + # set these to the correct vertex types based on what we have + vert_type_str = ( + "sbsp_comp_%s_verts" + if c_verts_size and c_lm_verts_size else + "sbsp_uncomp_%s_verts" + ) + mat.vertex_type.set_to(vert_type_str % "material") + mat.lightmap_vertex_type.set_to(vert_type_str % "lightmap") - # replace the buffers - u_verts.STEPTREE = uncomp_buffer - c_verts.STEPTREE = comp_buffer + mat.uncompressed_vertices.STEPTREE = u_buffer + mat.compressed_vertices.STEPTREE = c_buffer elif tag_cls == "scnr": # need to remove the references to the child scenarios @@ -1269,26 +1338,34 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): string_data = meta.script_string_data.data.decode("latin-1") syntax_data = get_hsc_data_block(raw_syntax_data=meta.script_syntax_data.data) + # lets not use magic numbers here + _, script_object_types = get_script_types(engine) + trigger_volume_enum = script_object_types.index("trigger_volume") + # NOTE: For a list of all the script object types # with their corrosponding enum value, check - # reclaimer.enums.script_object_types - keep_these = {i: set() for i in + # reclaimer.halo_script.hsc.get_script_types + keep_these = {script_object_types.index(typ): set() for typ in SCRIPT_OBJECT_TYPES_TO_SCENARIO_REFLEXIVES} + + # don't de-duplicate trigger volumes for b in meta.bsp_switch_trigger_volumes.STEPTREE: - keep_these[11].add(b.trigger_volume) + keep_these[trigger_volume_enum].add(b.trigger_volume) + # for everything we're keeping, clear the upper 16bits of the data for i in range(min(syntax_data.last_node, len(syntax_data.nodes))): node = syntax_data.nodes[i] - if node.type not in keep_these: - continue - - keep_these[node.type].add(node.data & 0xFFff) + if node.type in keep_these: + keep_these[node.type].add(node.data & 0xFFff) + # for everything else, rename duplicates for script_object_type, reflexive_name in \ SCRIPT_OBJECT_TYPES_TO_SCENARIO_REFLEXIVES.items(): - keep = keep_these[script_object_type] - reflexive = meta[reflexive_name].STEPTREE - counts = {b.name.lower(): 0 for b in reflexive} + + script_object_type_enum = script_object_types.index(script_object_type) + keep = keep_these[script_object_type_enum] + reflexive = meta[reflexive_name].STEPTREE + counts = {b.name.lower(): 0 for b in reflexive} for b in reflexive: counts[b.name.lower()] += 1 @@ -1297,6 +1374,23 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): if counts[name] > 1 and i not in keep: reflexive[i].name = ("DUP%s~%s" % (i, name))[: 31] + # null tag refs after we're done with them + clean_script_syntax_nodes(syntax_data, engine) + + # decompile scripts and put them in the source_files array so + # sapien can recompile them when it opens an extracted scenario + source_files = meta.source_files.STEPTREE + del source_files[:] + script_sources, global_sources = extract_scripts( + engine=engine, tagdata=meta, add_comments=False, minify=True + ) + i = 0 + for source in (*script_sources, *global_sources): + source_files.append() + source_files[-1].source_name = "decompiled_%s.hsc" % i + source_files[-1].source.data = source.encode('latin-1') + i += 1 + # divide the cutscene times by 30(they're in ticks) and # subtract the fade-in time from the up_time(normally added # together as a total up-time in maps, but not in tag form) @@ -1307,15 +1401,6 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): b.fade_out_time /= 30 b.up_time /= 30 - elif tag_cls == "snd!": - meta.maximum_bend_per_second = meta.maximum_bend_per_second ** 30 - for pitch_range in meta.pitch_ranges.STEPTREE: - if not byteswap: break - for permutation in pitch_range.permutations.STEPTREE: - if permutation.compression.enum_name == "none": - # byteswap pcm audio - byteswap_pcm16_samples(permutation.samples) - elif tag_cls == "shpp": predicted_resources.append(meta.predicted_resources) @@ -1363,7 +1448,22 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): # clear the merged values reflexive del shpg_attrs.merged_values.STEPTREE[:] + elif tag_cls == "soso": + # set the mcc multipurpose_map_uses_og_xbox_channel_order flag + if "xbox" in engine or "stubbs" in engine or engine == "shadowrun_proto": + meta.soso_attrs.model_shader.flags.data |= 1<<6 + elif tag_cls == "weap": + # try to fix HEK+ extraction bug + uint16_data = (meta.weap_attrs.aiming.zoom_levels & 0xFFff) + if (uint16_data & 0xFF00 and not uint16_data & 0xFF): + # higher bits are set than lower. this is likely + # a HEK plus extraction bug and should be fixed + uint16_data = ((uint16_data>>8) | (uint16_data<<8)) & 0xFFff + meta.weap_attrs.aiming.zoom_levels = uint16_data - ( + 0 if uint16_data < 0x8000 else 0x10000 + ) + predicted_resources.append(meta.weap_attrs.predicted_resources) # remove any predicted resources @@ -1373,37 +1473,31 @@ def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): return meta def get_resource_map_paths(self, maps_dir=""): - if self.is_resource or self.engine not in ("halo1pc", "halo1pcdemo", - "halo1ce", "halo1yelo", - "halo1vap"): + if self.is_resource or not self.resource_maps_folder: return {} - map_paths = {"bitmaps": None, "sounds": None, "loc": None} - if self.engine not in ("halo1ce", "halo1yelo", "halo1vap"): - map_paths.pop('loc') - - data_files = False - if hasattr(self.map_header, "yelo_header"): - data_files = self.map_header.yelo_header.flags.uses_mod_data_files - - if not is_path_empty(maps_dir): - maps_dir = Path(maps_dir) - elif data_files: - maps_dir = self.filepath.parent.joinpath("data_files") - else: - maps_dir = self.filepath.parent + map_paths = { + name: None for name in ( + *(["bitmaps"] if self.uses_bitmaps_map else []), + *(["sounds"] if self.uses_sounds_map else []), + *(["loc"] if self.uses_loc_map else []), + ) + } - map_name_str = "%s.map" - if data_files: - map_name_str = "~" + map_name_str + name_str = self.resource_map_prefix + "%s.map" + maps_dir = ( + Path(maps_dir) if not is_path_empty(maps_dir) else + self.resource_maps_folder + ) # detect the map paths for the resource maps - for map_name in sorted(map_paths.keys()): - map_path = maps_dir.joinpath(map_name_str % map_name) - if self.maps.get(map_name) is not None: - map_paths[map_name] = self.maps[map_name].filepath - elif map_path.is_file(): - map_paths[map_name] = map_path + if maps_dir: + for map_name in sorted(map_paths.keys()): + map_path = maps_dir.joinpath(name_str % map_name) + if self.maps.get(map_name) is not None: + map_paths[map_name] = self.maps[map_name].filepath + elif map_path.is_file(): + map_paths[map_name] = map_path return map_paths @@ -1411,6 +1505,16 @@ def generate_map_info_string(self): string = HaloMap.generate_map_info_string(self) index, header = self.tag_index, self.map_header + if self.engine == "halo1mcc": + string += """\n Calculated information: + use bitmaps map == %s + use sounds map == %s + no remastered sync == %s""" % ( + bool(header.mcc_flags.use_bitmaps_map), + bool(header.mcc_flags.use_sounds_map), + bool(header.mcc_flags.disable_remastered_sync), + ) + string += """ Calculated information: @@ -1421,28 +1525,33 @@ def generate_map_info_string(self): tag count == %s scenario tag id == %s index array pointer == %s non-magic == %s - model data pointer == %s meta data length == %s vertex parts count == %s index parts count == %s""" % ( self.index_magic, self.map_magic, index.tag_count, index.scenario_tag_id & 0xFFff, index.tag_index_offset, index.tag_index_offset - self.map_magic, - index.model_data_offset, header.tag_data_size, + header.tag_data_size, index.vertex_parts_count, index.index_parts_count) - if index.SIZE == 36: + if hasattr(index, "model_data_size"): string += """ - index parts pointer == %s non-magic == %s""" % ( - index.index_parts_offset, index.index_parts_offset - self.map_magic) - else: - string += """ - vertex data size == %s + vertex data pointer == %s + index data pointer == %s index data size == %s model data size == %s""" % ( - index.vertex_data_size, - index.model_data_size - index.vertex_data_size, - index.model_data_size) + index.model_data_offset, + index.index_parts_offset, + index.model_data_size - index.index_parts_offset, + index.model_data_size + ) + else: + string += """ + vertex refs pointer == %s non-magic == %s + index refs pointer == %s non-magic == %s""" % ( + index.model_data_offset, index.model_data_offset - self.map_magic, + index.index_parts_offset, index.index_parts_offset - self.map_magic, + ) string += "\n\nSbsp magic and headers:\n" for tag_id in self.bsp_magics: @@ -1450,83 +1559,42 @@ def generate_map_info_string(self): if header is None: continue magic = self.bsp_magics[tag_id] + offset = self.bsp_header_offsets[tag_id] string += """ %s.structure_scenario_bsp bsp base pointer == %s bsp magic == %s bsp size == %s bsp metadata pointer == %s non-magic == %s\n""" % ( - index.tag_index[tag_id].path, self.bsp_header_offsets[tag_id], + index.tag_index[tag_id].path, offset, magic, self.bsp_sizes[tag_id], header.meta_pointer, - header.meta_pointer - magic) + header.meta_pointer + offset - magic + ) + if self.engine in ("halo1mcc", "halo1anni"): + string += """\ + render verts size == %s + render verts pointer == %s\n""" % ( + header.uncompressed_render_vertices_size, + header.uncompressed_render_vertices_pointer, + ) + else: + string += """\ + uncomp mats count == %s + uncomp mats pointer == %s non-magic == %s + comp mats count == %s + comp mats pointer == %s non-magic == %s\n""" % ( + header.uncompressed_lightmap_materials_count, + header.uncompressed_lightmap_materials_pointer, + header.uncompressed_lightmap_materials_pointer + offset - magic, + header.compressed_lightmap_materials_count, + header.compressed_lightmap_materials_pointer, + header.compressed_lightmap_materials_pointer + offset - magic, + ) - if self.engine == "halo1yelo": - string += self.generate_yelo_info_string() - elif self.engine == "halo1vap": + if self.engine == "halo1vap": string += self.generate_vap_info_string() return string - def generate_yelo_info_string(self): - yelo = self.map_header.yelo_header - flags = yelo.flags - info = yelo.build_info - version = yelo.tag_versioning - cheape = yelo.cheape_definitions - rsrc = yelo.resources - min_os = info.minimum_os_build - - return """ -Yelo information: - Mod name == %s - Memory upgrade amount == %sx - - Flags: - uses memory upgrades == %s - uses mod data files == %s - is protected == %s - uses game state upgrades == %s - has compression parameters == %s - - Build info: - build string == %s - timestamp == %s - stage == %s - revision == %s - - Cheape: - build string == %s - version == %s.%s.%s - size == %s - offset == %s - decompressed size == %s - - Versioning: - minimum open sauce == %s.%s.%s - project yellow == %s - project yellow globals == %s - - Resources: - compression parameters header offset == %s - tag symbol storage header offset == %s - string id storage header offset == %s - tag string to id storage header offset == %s\n""" % ( - yelo.mod_name, yelo.memory_upgrade_multiplier, - bool(flags.uses_memory_upgrades), - bool(flags.uses_mod_data_files), - bool(flags.is_protected), - bool(flags.uses_game_state_upgrades), - bool(flags.has_compression_params), - info.build_string, info.timestamp, info.stage.enum_name, - info.revision, cheape.build_string, - info.cheape.maj, info.cheape.min, info.cheape.build, - cheape.size, cheape.offset, cheape.decompressed_size, - min_os.maj, min_os.min, min_os.build, - version.project_yellow, version.project_yellow_globals, - rsrc.compression_params_header_offset, - rsrc.tag_symbol_storage_header_offset, - rsrc.string_id_storage_header_offset, - rsrc.tag_string_to_id_storage_header_offset) - def generate_vap_info_string(self): vap = self.map_header.vap_header diff --git a/reclaimer/meta/wrappers/halo1_mcc_map.py b/reclaimer/meta/wrappers/halo1_mcc_map.py new file mode 100644 index 00000000..66e6ed35 --- /dev/null +++ b/reclaimer/meta/wrappers/halo1_mcc_map.py @@ -0,0 +1,122 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# +from pathlib import Path +from reclaimer.meta.wrappers.halo1_map import Halo1Map +from reclaimer.mcc_hek.handler import MCCHaloHandler +from reclaimer.mcc_hek.defs.sbsp import sbsp_meta_header_def +from reclaimer.meta.wrappers.halo1_rsrc_map import uses_external_sounds + +from supyr_struct.util import is_path_empty + +class Halo1MccMap(Halo1Map): + '''Masterchief Collection Halo 1 map''' + + # Module path printed when loading the tag defs + tag_defs_module = "reclaimer.mcc_hek.defs" + # Handler that controls how to load tags, eg tag definitions + handler_class = MCCHaloHandler + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None + + sbsp_meta_header_def = sbsp_meta_header_def + + indexable_tag_classes = frozenset(("bitm", "snd!")) + + @property + def uses_loc_map(self): + return False + @property + def uses_sounds_map(self): + try: + return self.map_header.mcc_flags.use_sounds_map + except AttributeError: + return False + @property + def uses_fmod_sound_bank(self): + return not self.uses_sounds_map + + def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): + # for sounds, ensure we can extract ALL their sample data from either + # resource map or primary map before potentially overwriting local + # tag files with them. remastered sounds are stored in fmod, and + # we can't extract those tags without missing the sample data + sounds = self.maps.get("sounds") + sounds_data = getattr(sounds, "map_data", None) + if tag_cls == "snd!" and not(self.uses_sounds_map and sounds_data): + if uses_external_sounds(meta): + # no sounds.map to read sounds from, and sound + # data is specified as external. can't extract + raise ValueError("Sound sample data missing.") + + meta = super().meta_to_tag_data(meta, tag_cls, tag_index_ref, **kwargs) + if tag_cls == "sbsp": + for lm in meta.lightmaps.STEPTREE: + for mat in lm.materials.STEPTREE: + mat.lightmap_vertices_offset = 0 + mat.vertices_offset = 0 + + return meta + + def inject_rawdata(self, meta, tag_cls, tag_index_ref): + if tag_cls == "snd!" and self.uses_fmod_sound_bank and uses_external_sounds(meta): + # no sounds.map to read sounds from, and sound + # data is specified as external. can't extract + return None + elif tag_cls == "sbsp": + # mcc render geometry isn't stored the same way as custom edition/xbox. + # it's stored relative to the pointers in the sbsp meta header, and the + # size of the verts isn't calculated into the size of the sbsp sector. + + tag_id = tag_index_ref.id & 0xFFFF + bsp_header = self.bsp_headers.get(tag_id, None) + if bsp_header is None: + raise ValueError("No bsp header found for tag %s of type %s" % ( + tag_id, tag_cls, + )) + + uc_sector_start = bsp_header.uncompressed_render_vertices_pointer + uc_sector_size = bsp_header.uncompressed_render_vertices_size + c_sector_start = bsp_header.compressed_render_vertices_pointer + c_sector_size = bsp_header.compressed_render_vertices_size + uc_sector_end = uc_sector_start + uc_sector_size + c_sector_end = c_sector_start + c_sector_size + map_data = self.map_data + + for lm in meta.lightmaps.STEPTREE: + for mat in lm.materials.STEPTREE: + if mat.vertex_type.enum_name == "compressed": + data_block = mat.compressed_vertices + start, end = c_sector_start, c_sector_end + vert_size, lm_vert_size = 32, 8 + else: + data_block = mat.uncompressed_vertices + start, end = uc_sector_start, uc_sector_end + vert_size, lm_vert_size = 56, 20 + + verts_offset = start + mat.vertices_offset + lm_verts_offset = start + mat.lightmap_vertices_offset + verts_size = vert_size * mat.vertices_count + lm_verts_size = lm_vert_size * mat.lightmap_vertices_count + vert_data = b'' + if verts_size and verts_size + verts_offset > end: + print("Warning: Render vertices pointed to outside sector.") + elif verts_size: + map_data.seek(verts_offset) + vert_data += map_data.read(verts_size) + + if lm_verts_size and lm_verts_size + lm_verts_offset > end: + print("Warning: Lightmap vertices pointed to outside sector.") + elif lm_verts_size: + map_data.seek(lm_verts_offset) + vert_data += map_data.read(lm_verts_size) + + data_block.data = bytearray(vert_data) + else: + return super().inject_rawdata(meta, tag_cls, tag_index_ref) \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo1_rsrc_map.py b/reclaimer/meta/wrappers/halo1_rsrc_map.py index c5f24bb1..22965a09 100644 --- a/reclaimer/meta/wrappers/halo1_rsrc_map.py +++ b/reclaimer/meta/wrappers/halo1_rsrc_map.py @@ -6,11 +6,17 @@ # Reclaimer is free software under the GNU General Public License v3.0. # See LICENSE for more information. # - +from collections import namedtuple +from copy import deepcopy from struct import unpack from traceback import format_exc +from reclaimer.constants import GEN_1_HALO_CUSTOM_ENGINES,\ + GEN_1_HALO_PC_ENGINES, GEN_1_HALO_GBX_ENGINES from reclaimer import data_extraction +from reclaimer.mcc_hek.defs.bitm import bitm_def as pixel_root_subdef +from reclaimer.mcc_hek.defs.objs.bitm import MccBitmTag, HALO_P8_PALETTE +from reclaimer.stubbs.defs.objs.bitm import StubbsBitmTag, STUBBS_P8_PALETTE from reclaimer.util import get_is_xbox_map from reclaimer.meta.halo_map import map_header_def, tag_index_pc_def from reclaimer.meta.halo1_rsrc_map import lite_halo1_rsrc_map_def as halo1_rsrc_map_def @@ -18,10 +24,14 @@ from reclaimer.meta.wrappers.map_pointer_converter import MapPointerConverter from reclaimer.meta.wrappers.tag_index_manager import TagIndexManager from reclaimer.meta.wrappers.halo_map import HaloMap +from reclaimer.sounds import ogg as sounds_ogg, constants as sound_const from supyr_struct.buffer import BytearrayBuffer, get_rawdata from supyr_struct.field_types import FieldType +# reassign since we only want a reference to the sub-definition +pixel_root_subdef = pixel_root_subdef.subdefs['pixel_root'] + # this is ultra hacky, but it seems to be the only # way to fix the tagid for the sounds resource map sound_rsrc_id_map = { @@ -58,10 +68,6 @@ def inject_sound_data(map_data, rsrc_data, rawdata_ref, map_magic): - if not rawdata_ref.size: - rawdata_ref.data = b'' - return - if rawdata_ref.flags.data_in_resource_map: data, ptr = rsrc_data, rawdata_ref.raw_pointer elif rawdata_ref.pointer == 0: @@ -69,8 +75,51 @@ def inject_sound_data(map_data, rsrc_data, rawdata_ref, map_magic): else: data, ptr = map_data, rawdata_ref.pointer + map_magic - data.seek(ptr) - rawdata_ref.data = data.read(rawdata_ref.size) + if data and rawdata_ref.size: + data.seek(ptr) + rawdata_ref.data = data.read(rawdata_ref.size) + else: + # hack to ensure the size is preserved when + # we replace the rawdata with empty bytes + size = rawdata_ref.size + rawdata_ref.data = b'' + rawdata_ref.size = size + + +def uses_external_sounds(sound_meta): + for pitches in sound_meta.pitch_ranges.STEPTREE: + for perm in pitches.permutations.STEPTREE: + for b in (perm.samples, perm.mouth_data, perm.subtitle_data): + if b.flags.data_in_resource_map: + return True + return False + + +class MetaBitmTag(): + ''' + This class exists to facilitate processing bitmap tags extracted + from maps without fully converting them to tag objects first. + ''' + _fake_data_block = namedtuple('FakeDataBlock', + ("blam_header", "tagdata") + ) + def __init__(self, tagdata=None): + self.data = self._fake_data_block(None, tagdata) + + # stubed since there's nothing to calculate here + def calc_internal_data(self): pass + + @property + def pixel_root_definition(self): return pixel_root_subdef + + +class MetaHaloBitmTag(MetaBitmTag, MccBitmTag): + @property + def p8_palette(self): return HALO_P8_PALETTE + +class MetaStubbsBitmTag(MetaBitmTag, StubbsBitmTag): + @property + def p8_palette(self): return STUBBS_P8_PALETTE class Halo1RsrcMap(HaloMap): @@ -104,19 +153,15 @@ def load_map(self, map_path, **kwargs): pth = self.orig_tag_index[0].tag.path if self.orig_tag_index else "" self.filepath = map_path - ce_engine = "" - for halo_map in self.maps.values(): - ce_engine = getattr(halo_map, "engine") - if ce_engine: - break - rsrc_tag_count = len(rsrc_map.data.tags) if resource_type == 3 or (pth.endswith('__pixels') or pth.endswith('__permutations')): - if ce_engine: - self.engine = ce_engine - else: - self.engine = "halo1ce" + engine = "" + for halo_map in self.maps.values(): + engine = engine or getattr(halo_map, "engine") + if engine: break + + self.engine = engine or "halo1ce" elif ((resource_type == 1 and rsrc_tag_count == 1107) or (resource_type == 2 and rsrc_tag_count == 7192)): self.engine = "halo1pcdemo" @@ -198,10 +243,16 @@ def get_dependencies(self, meta, tag_id, tag_cls): return () tag_id = meta.promotion_sound.id & 0xFFff - if tag_id not in range(len(self.tag_index.tag_index)): + tag_index_array = self.tag_index.tag_index + if tag_id not in range(len(tag_index_array)): return () - return [self.tag_index.tag_index[tag_id]] + ref = deepcopy(meta.promotion_sound) + tag_index_ref = tag_index_array[tag_id] + ref.tag_class.data = tag_index_ref.class_1.data + ref.id = tag_index_ref.id + ref.filepath = tag_index_ref.path + return [ref] def is_indexed(self, tag_id): return True @@ -223,8 +274,11 @@ def get_meta(self, tag_id, reextract=False, **kw): kwargs = dict(parsing_resource=True) desc = self.get_meta_descriptor(tag_cls) - if desc is None or self.engine not in ("halo1ce", "halo1yelo", - "halo1vap"): + if (desc is None or self.engine == "halo1mcc" or + self.engine not in GEN_1_HALO_CUSTOM_ENGINES): + # NOTE: mcc resource maps DON'T contain metadata, they only + # contain bitmap pixel data and sound sample data. + # as such, they're EXACTLY like halo1pc resource maps return elif tag_cls != 'snd!': # the pitch ranges pointer in resource sound tags is invalid, so @@ -244,7 +298,7 @@ def get_meta(self, tag_id, reextract=False, **kw): desc, parent=block, attr_index=0, rawdata=self.map_data, tag_index_manager=self.snd_rsrc_tag_index_manager, tag_cls=tag_cls, root_offset=tag_index_ref.meta_offset, - safe_mode=not kw.get("disable_safe_mode"), + safe_mode=(self.safe_mode and not kw.get("disable_safe_mode")), indexed=True, **kwargs) FieldType.force_normal() @@ -258,37 +312,90 @@ def get_meta(self, tag_id, reextract=False, **kw): return block[0] def meta_to_tag_data(self, meta, tag_cls, tag_index_ref, **kwargs): - magic = self.map_magic - engine = self.engine - map_data = self.map_data - tag_index = self.tag_index - is_xbox = get_is_xbox_map(engine) + magic = self.map_magic + engine = self.engine + map_data = self.map_data + tag_index = self.tag_index + byteswap = kwargs.get("byteswap", True) if tag_cls == "bitm": + bitm_tag_cls = MetaStubbsBitmTag if "stubbs" in engine else MetaHaloBitmTag + bitm_tag = bitm_tag_cls(meta) + bitmaps = meta.bitmaps.STEPTREE + # set the size of the compressed plate data to nothing meta.compressed_color_plate_data.STEPTREE = BytearrayBuffer() - new_pixels_offset = 0 - - # uncheck the prefer_low_detail flag and - # set up the pixels_offset correctly. - for bitmap in meta.bitmaps.STEPTREE: - bitmap.flags.prefer_low_detail = is_xbox - bitmap.pixels_offset = new_pixels_offset - new_pixels_offset += bitmap.pixels_meta_size - - # clear some meta-only fields - bitmap.pixels_meta_size = 0 - bitmap.bitmap_id_unknown1 = bitmap.bitmap_id_unknown2 = 0 - bitmap.bitmap_data_pointer = bitmap.base_address = 0 + # set up the pixels_offsets + pixels_offset = 0 + for bitmap in bitmaps: + bitmap.pixels_offset = pixels_offset + pixels_offset += bitmap.pixels_meta_size + + # undo xbox-specific stuff(reorder bitmaps and unswizzle) + if get_is_xbox_map(engine): + # rearrange the bitmap pixels so they're in standard format + try: + bitm_tag.parse_bitmap_blocks() + bitm_tag.sanitize_bitmaps() + bitm_tag.set_swizzled(False) + bitm_tag.add_bitmap_padding(False) + + # serialize the pixel_data and replace the parsed block with it + meta.processed_pixel_data.data = meta.processed_pixel_data.data.serialize() + except Exception: + print(format_exc()) + print("Failed to convert xbox bitmap data to pc.") + + # clear meta-only fields + for bitmap in bitmaps: + bitmap.flags.data &= 0x3F + bitmap.base_address = 0 + bitmap.pixels_meta_size = bitmap.bitmap_data_pointer = 0 + bitmap.bitmap_id_unknown1 = bitmap.bitmap_id_unknown2 = 0 elif tag_cls == "snd!": meta.maximum_bend_per_second = meta.maximum_bend_per_second ** 30 + meta.unknown1 = 0xFFFFFFFF + meta.unknown2 = 0xFFFFFFFF + bytes_per_sample = sound_const.channel_counts.get( + meta.encoding.data, 1 + ) * 2 for pitch_range in meta.pitch_ranges.STEPTREE: - for permutation in pitch_range.permutations.STEPTREE: - if permutation.compression.enum_name == "none": - # byteswap pcm audio - byteswap_pcm16_samples(permutation.samples) + # null some meta-only fields + pitch_range.playback_rate = 0.0 + pitch_range.unknown1 = -1 + pitch_range.unknown2 = -1 + + for perm in pitch_range.permutations.STEPTREE: + if perm.compression.enum_name == "none": + buffer_size = len(perm.samples) + if byteswap: + # byteswap pcm audio + byteswap_pcm16_samples(perm.samples) + elif perm.compression.enum_name == "ogg": + # oggvorbis NEEDS this set for proper playback ingame + buffer_size = ( + sounds_ogg.get_ogg_pcm_sample_count(perm.samples.data) + if sound_const.OGGVORBIS_AVAILABLE else + # oh well. default to whatever it's set to + (perm.buffer_size // bytes_per_sample) + ) * bytes_per_sample + else: + buffer_size = 0 + + # fix buffer_size possibly being incorrect + perm.buffer_size = buffer_size + + # null some meta-only fields + perm.sample_data_pointer = perm.parent_tag_id = perm.unknown = 0 + if hasattr(perm, "runtime_flags"): # mcc + perm.runtime_flags = 0 + else: # non-mcc + perm.parent_tag_id2 = 0 + + for b in (perm.samples, perm.mouth_data, perm.subtitle_data): + b.flags.data_in_resource_map = False return meta @@ -300,18 +407,13 @@ def inject_rawdata(self, meta, tag_cls, tag_index_ref): magic = self.map_magic engine = self.engine - map_data = self.map_data - - try: bitmap_data = bitmaps.map_data - except Exception: bitmap_data = None - try: sound_data = sounds.map_data - except Exception: sound_data = None - try: loc_data = loc.map_data - except Exception: loc_data = None + map_data = self.map_data + bitmap_data = getattr(bitmaps, "map_data", None) + sound_data = getattr(sounds, "map_data", None) + loc_data = getattr(loc, "map_data", None) is_not_indexed = not self.is_indexed(tag_index_ref.id & 0xFFff) - might_be_in_rsrc = engine in ("halo1pc", "halo1pcdemo", - "halo1ce", "halo1yelo", "halo1vap") + might_be_in_rsrc = engine in GEN_1_HALO_GBX_ENGINES might_be_in_rsrc &= not self.is_resource # get some rawdata that would be pretty annoying to do in the parser @@ -363,30 +465,34 @@ def inject_rawdata(self, meta, tag_cls, tag_index_ref): b = meta.string loc_data.seek(b.pointer + meta_offset) meta.string.data = loc_data.read(b.size).decode('utf-16-le') + elif tag_cls == "snd!": # might need to get samples and permutations from the resource map - is_pc = engine in ("halo1pc", "halo1pcdemo") - is_ce = engine in ("halo1ce", "halo1yelo", "halo1vap") - if not(is_pc or is_ce): - return meta - elif sound_data is None: - return + is_mcc = engine == "halo1mcc" + is_pc = engine in GEN_1_HALO_PC_ENGINES + is_ce = engine in GEN_1_HALO_CUSTOM_ENGINES and not is_mcc # ce tagpaths are in the format: path__permutations # ex: sound\sfx\impulse\coolant\enter_water__permutations # # pc tagpaths are in the format: path__pitchrange__permutation # ex: sound\sfx\impulse\coolant\enter_water__0__0 - other_data = map_data - sound_magic = 0 - magic - # DO NOT optimize this section. The logic is like this on purpose - if is_pc: - pass - elif self.is_resource: - other_data = sound_data - sound_magic = tag_index_ref.meta_offset + meta.get_size() - elif sounds is None: + + if not(is_pc or is_ce or is_mcc): + # not pc, ce, or mcc, so sound data is read on initial tag parse return + elif self.is_resource and is_ce: + # ce sounds.map contain tagdata, not just sample data. + # HOWEVER, the pointers in the tag data are relative to + # the END of the tag(idky), so we set the magic to it. + other_data = sound_data # reading for resource, so sound map IS map data + sound_magic = tag_index_ref.meta_offset + meta.get_size() + else: + # either samples are in resource map and are pointed to with + # the raw pointer(relative to file start), or are in the main + # map and are pointed to with the magic-relative pointer + other_data = map_data + sound_magic = 0 - magic for pitches in meta.pitch_ranges.STEPTREE: for perm in pitches.permutations.STEPTREE: diff --git a/reclaimer/meta/wrappers/halo1_xbox_map.py b/reclaimer/meta/wrappers/halo1_xbox_map.py new file mode 100644 index 00000000..c6875098 --- /dev/null +++ b/reclaimer/meta/wrappers/halo1_xbox_map.py @@ -0,0 +1,33 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# +from reclaimer.meta.wrappers.halo1_map import Halo1Map +from reclaimer.hek.handler import HaloHandler + +class Halo1XboxMap(Halo1Map): + '''Halo 1 Xbox map''' + + # Module path printed when loading the tag defs + tag_defs_module = "reclaimer.hek.defs" + # Handler that controls how to load tags, eg tag definitions + handler_class = HaloHandler + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None + + def is_indexed(self, tag_id): + return False + + @property + def resource_maps_folder(self): return None + @property + def uses_bitmaps_map(self): return False + @property + def uses_loc_map(self): return False + @property + def uses_sounds_map(self): return False \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo1_yelo.py b/reclaimer/meta/wrappers/halo1_yelo.py index f1eb6e2d..8ba796f2 100644 --- a/reclaimer/meta/wrappers/halo1_yelo.py +++ b/reclaimer/meta/wrappers/halo1_yelo.py @@ -6,16 +6,167 @@ # Reclaimer is free software under the GNU General Public License v3.0. # See LICENSE for more information. # -from reclaimer.meta.wrappers.halo1_map import Halo1Map -from reclaimer.os_v4_hek.handler import OsV4HaloHandler +from reclaimer.meta.wrappers.halo1_map import Halo1Map, int_to_fourcc +from reclaimer.os_hek.defs.gelc import gelc_def +from reclaimer.os_v4_hek.defs.coll import fast_coll_def +from reclaimer.os_v4_hek.defs.sbsp import fast_sbsp_def +from reclaimer.os_v4_hek.handler import OsV4HaloHandler +from supyr_struct.defs.frozen_dict import FrozenDict + class Halo1YeloMap(Halo1Map): '''Generation 1 Yelo map''' + resource_map_prefix = "~" # Module path printed when loading the tag defs tag_defs_module = "reclaimer.os_v4_hek.defs" # Handler that controls how to load tags, eg tag definitions handler_class = OsV4HaloHandler + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None + + @property + def is_fully_yelo(self): + # since it's possible to compile open sauce maps without the hard + # requirement that they be yelo maps, this wrapper is able to be + # used with the engine still set to halo1ce. to determine if the + # map is truely a .yelo map, we need to check the engine. + yelo_header = self.map_header.yelo_header + return "" not in ( + yelo_header.yelo.enum_name, + yelo_header.version_type.enum_name + ) + + @property + def uses_mod_data_files(self): + if not self.is_fully_yelo: + return False + + try: + return self.map_header.yelo_header.flags.uses_mod_data_files + except AttributeError: + return False + @property + def resource_map_prefix(self): + return "~" if self.uses_mod_data_files else "" + @property + def resource_maps_folder(self): + return self.filepath.parent.joinpath( + "data_files" if self.uses_mod_data_files else "" + ) + @property + def decomp_file_ext(self): + return ".yelo" if self.is_fully_yelo else self._decomp_file_ext + + @property + def project_yellow_tag_id(self): + if not(self.is_fully_yelo and self.tag_index): + return None + + for b in self.tag_index.tag_index: + if int_to_fourcc(b.class_1.data) == "yelo": + return b.id & 0xFFff + + @property + def globals_tag_id(self): + matg_tag_id = None + if self.is_fully_yelo: + yelo_meta = self.get_meta(self.project_yellow_tag_id) + if yelo_meta: + matg_tag_id = yelo_meta.globals_override.id & 0xFFff + + for b in self.tag_index.tag_index: + if matg_tag_id in range(len(self.tag_index.tag_index)): + break + + if int_to_fourcc(b.class_1.data) == "matg": + matg_tag_id = b.id & 0xFFff + + if matg_tag_id in range(len(self.tag_index.tag_index)): + return self.tag_index.tag_index[matg_tag_id].id & 0xFFff + + def setup_defs(self): + this_class = type(self) + if this_class.defs is None: + this_class.defs = defs = {} + print(" Loading definitions in '%s'" % self.tag_defs_module) + this_class.handler = self.handler_class( + build_reflexive_cache=False, build_raw_data_cache=False, + debug=2) + + this_class.defs = dict(this_class.handler.defs) + this_class.defs["coll"] = fast_coll_def + this_class.defs["sbsp"] = fast_sbsp_def + this_class.defs["gelc"] = gelc_def + this_class.defs = FrozenDict(this_class.defs) + + # make a shallow copy for this instance to manipulate + self.defs = dict(self.defs) + + def generate_map_info_string(self): + string = super().generate_map_info_string() + if self.is_fully_yelo: + string += self.generate_yelo_info_string() + return string + + def generate_yelo_info_string(self): + yelo = self.map_header.yelo_header + flags = yelo.flags + info = yelo.build_info + version = yelo.tag_versioning + cheape = yelo.cheape_definitions + rsrc = yelo.resources + min_os = info.minimum_os_build + + return """ +Yelo information: + Mod name == %s + Memory upgrade amount == %sx + + Flags: + uses memory upgrades == %s + uses mod data files == %s + is protected == %s + uses game state upgrades == %s + has compression parameters == %s + + Build info: + build string == %s + timestamp == %s + stage == %s + revision == %s + + Cheape: + build string == %s + version == %s.%s.%s + size == %s + offset == %s + decompressed size == %s + + Versioning: + minimum open sauce == %s.%s.%s + project yellow == %s + project yellow globals == %s - def __init__(self, maps=None): - Halo1Map.__init__(self, maps) + Resources: + compression parameters header offset == %s + tag symbol storage header offset == %s + string id storage header offset == %s + tag string to id storage header offset == %s\n""" % ( + yelo.mod_name, yelo.memory_upgrade_multiplier, + bool(flags.uses_memory_upgrades), + bool(flags.uses_mod_data_files), + bool(flags.is_protected), + bool(flags.uses_game_state_upgrades), + bool(flags.has_compression_params), + info.build_string, info.timestamp, info.stage.enum_name, + info.revision, cheape.build_string, + info.cheape.maj, info.cheape.min, info.cheape.build, + cheape.size, cheape.offset, cheape.decompressed_size, + min_os.maj, min_os.min, min_os.build, + version.project_yellow, version.project_yellow_globals, + rsrc.compression_params_header_offset, + rsrc.tag_symbol_storage_header_offset, + rsrc.string_id_storage_header_offset, + rsrc.tag_string_to_id_storage_header_offset) \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo3_beta_map.py b/reclaimer/meta/wrappers/halo3_beta_map.py index 8667686a..71a837fc 100644 --- a/reclaimer/meta/wrappers/halo3_beta_map.py +++ b/reclaimer/meta/wrappers/halo3_beta_map.py @@ -11,4 +11,7 @@ class Halo3BetaMap(Halo3Map): tag_defs_module = "" - tag_classes_to_load = tuple() + tag_classes_to_load = () + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo3_map.py b/reclaimer/meta/wrappers/halo3_map.py index 420849da..e2b37a1d 100644 --- a/reclaimer/meta/wrappers/halo3_map.py +++ b/reclaimer/meta/wrappers/halo3_map.py @@ -66,8 +66,6 @@ def get_bitmap_pixel_data(halo_map, bitm_meta, bitmap_index): def inject_bitmap_data(halo_map, bitm_meta): processed_pixel_data = bitm_meta.processed_pixel_data bitmaps = bitm_meta.bitmaps.STEPTREE - n_assets = bitm_meta.zone_assets_normal.STEPTREE - i_assets = bitm_meta.zone_assets_interleaved.STEPTREE processed_pixel_data.data = bytearray() for i in range(len(bitmaps)): @@ -184,7 +182,7 @@ def get_root_tag(self, tag_id_or_cls): if isinstance(tag_id_or_cls, int): tag_id_or_cls &= 0xFFff if tag_id_or_cls not in self.root_tags: - self.load_root_tags(tag_id_or_cls) + self.load_root_tags([tag_id_or_cls]) return self.root_tags.get(tag_id_or_cls) @@ -198,7 +196,7 @@ def load_root_tags(self, tag_ids_to_load=()): ("scenario", "globals")) for tag_id in tag_ids_to_load: - tag_cls = tag_classes_to_load_by_ids[tag_id] + tag_cls = tag_classes_to_load_by_ids.get(tag_id) meta = self.get_meta(tag_id) if meta: self.root_tags[tag_id & 0xFFff] = meta diff --git a/reclaimer/meta/wrappers/halo3_odst_map.py b/reclaimer/meta/wrappers/halo3_odst_map.py index 7f23f18f..baa43e9c 100644 --- a/reclaimer/meta/wrappers/halo3_odst_map.py +++ b/reclaimer/meta/wrappers/halo3_odst_map.py @@ -8,7 +8,28 @@ # from .halo3_map import Halo3Map +from reclaimer.h3.defs.bitm import bitm_def +from reclaimer.h3.defs.play import play_def +from reclaimer.h3.defs.zone import zone_def_odst_partial +from supyr_struct.defs.frozen_dict import FrozenDict class Halo3OdstMap(Halo3Map): tag_defs_module = "" - tag_classes_to_load = tuple() + tag_classes_to_load = ( + "play", "bitm", "zone" + ) + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None + + def setup_defs(self): + this_class = type(self) + if this_class.defs is None: + this_class.defs = FrozenDict({ + "zone": zone_def_odst_partial, + "bitm": bitm_def, + "play": play_def, + }) + + # make a shallow copy for this instance to manipulate + self.defs = dict(self.defs) \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo4_beta_map.py b/reclaimer/meta/wrappers/halo4_beta_map.py index 50fa37c5..200d2aa9 100644 --- a/reclaimer/meta/wrappers/halo4_beta_map.py +++ b/reclaimer/meta/wrappers/halo4_beta_map.py @@ -11,4 +11,7 @@ class Halo4BetaMap(Halo4Map): tag_defs_module = "" - tag_classes_to_load = tuple() + tag_classes_to_load = () + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo4_map.py b/reclaimer/meta/wrappers/halo4_map.py index dadf043c..de723470 100644 --- a/reclaimer/meta/wrappers/halo4_map.py +++ b/reclaimer/meta/wrappers/halo4_map.py @@ -11,4 +11,7 @@ class Halo4Map(Halo3Map): tag_defs_module = "" - tag_classes_to_load = tuple() + tag_classes_to_load = () + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo5_map.py b/reclaimer/meta/wrappers/halo5_map.py index 06f1c548..9d1415cc 100644 --- a/reclaimer/meta/wrappers/halo5_map.py +++ b/reclaimer/meta/wrappers/halo5_map.py @@ -12,3 +12,6 @@ class Halo5Map(Halo3Map): tag_defs_module = "" tag_classes_to_load = tuple() + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo_map.py b/reclaimer/meta/wrappers/halo_map.py index 7d0def3a..154a9641 100644 --- a/reclaimer/meta/wrappers/halo_map.py +++ b/reclaimer/meta/wrappers/halo_map.py @@ -16,9 +16,10 @@ from traceback import format_exc from string import ascii_letters +from reclaimer.constants import GEN_1_HALO_CUSTOM_ENGINES from reclaimer.meta.halo_map import get_map_version, get_map_header,\ get_tag_index, get_index_magic, get_map_magic, get_is_compressed_map,\ - decompress_map + decompress_map, get_engine_name from reclaimer.meta.wrappers.map_pointer_converter import MapPointerConverter from reclaimer.util import is_protected_tag, int_to_fourcc, path_normalize @@ -29,7 +30,12 @@ VALID_MODULE_NAME_CHARS = ascii_letters + '_' + '0123456789' -backslash_fix = re.compile(r"\\{2,}") +multi_backslash_sub = re.compile(r"\\{2,}|/+") +lead_trail_slash_space_sub = re.compile(r"^[\s\\]+|[\s\\]+$") +def sanitize_tag_path(tag_path): + return lead_trail_slash_space_sub.sub( + '', multi_backslash_sub.sub(r'\\', tag_path) + ).lower() class HaloMap: @@ -66,6 +72,7 @@ class HaloMap: engine = "" is_resource = False is_compressed = False + safe_mode = True handler = None @@ -212,7 +219,7 @@ def get_dependencies(self, meta, tag_id, tag_cls): return () def is_indexed(self, tag_id): - return bool(self.tag_index.tag_index[tag_id].indexed) + return False def cache_original_tag_paths(self): tags = () if self.tag_index is None else self.tag_index.tag_index @@ -224,23 +231,39 @@ def basic_deprotection(self): if self.tag_index is None: return - i = 0 - found_counts = {} - for b in self.tag_index.tag_index: - tag_path = backslash_fix.sub( - r'\\', b.path.replace("/", "\\")).strip().strip("\\").lower() - - name_id = (tag_path, b.class_1.enum_name) - if is_protected_tag(tag_path): + name_id_counts = {} + # count how many times each tag name and class combination is used + for i, b in enumerate(self.tag_index.tag_index): + name_id = (sanitize_tag_path(b.path), b.class_1.enum_name) + name_id_counts[name_id] = name_id_counts.get(name_id, 0) + 1 + + # figure out if any reused tag names stand out as protected + # because of being reused in with the same tag class + protected_names = set(( + name_id[0] + for name_id, count in name_id_counts.items() + if is_protected_tag(name_id[0]) or count > 1 + )) + + # rename tags deemed to be protected + for i, b in enumerate(self.tag_index.tag_index): + tag_path = sanitize_tag_path(b.path) + if not self.is_indexed(i) and tag_path in protected_names: tag_path = "protected_%s" % i - elif name_id in found_counts: - tag_path = "%s_%s" % (tag_path, found_counts[name_id]) - found_counts[name_id] += 1 - else: - found_counts[name_id] = 1 b.path = tag_path - i += 1 + + name_id_counts = {} + # this bit is cause apparently spv3/open sauce v4/neil fucked up + # resource maps, and made it so that can contain duplicate tags. + for i, b in enumerate(self.tag_index.tag_index): + tag_path = sanitize_tag_path(b.path) + name_id = (tag_path, b.class_1.enum_name) + count = name_id_counts.setdefault(name_id, 1) + if self.is_indexed(i) and count > 1: + b.path = "%s_%s" % (tag_path, count) + + name_id_counts[name_id] = count + 1 def get_meta_descriptor(self, tag_cls): tagdef = self.defs.get(tag_cls) @@ -319,7 +342,12 @@ def load_resource_maps(self, maps_dir="", map_paths=(), **kw): print("Loading %s..." % map_name) new_map.load_map(map_path, **kw) - if new_map.engine != self.engine: + if (self.engine in GEN_1_HALO_CUSTOM_ENGINES and + new_map.engine in ("halo1pc", "halo1ce")): + # cant tell mcc resource maps apart from ce and pc resource maps. + # the same can be said of telling apart yelo and ce resource maps. + new_map.engine = self.engine + elif new_map.engine != self.engine: if do_printout: print("Incorrect engine for this map.") self.maps.pop(new_map.map_name, None) @@ -371,6 +399,9 @@ def load_map(self, map_path, **kwargs): tag_index = self.orig_tag_index = get_tag_index( self.map_data, map_header) + # calculate this more accurately now that the map data is available + self.engine = get_engine_name(map_header, self.map_data) + if tag_index is None: print(" Could not read tag index.") return diff --git a/reclaimer/meta/wrappers/halo_reach_beta_map.py b/reclaimer/meta/wrappers/halo_reach_beta_map.py index a99c0ca3..d68deefb 100644 --- a/reclaimer/meta/wrappers/halo_reach_beta_map.py +++ b/reclaimer/meta/wrappers/halo_reach_beta_map.py @@ -11,4 +11,7 @@ class HaloReachBetaMap(HaloReachMap): tag_defs_module = "" - tag_classes_to_load = tuple() + tag_classes_to_load = () + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None \ No newline at end of file diff --git a/reclaimer/meta/wrappers/halo_reach_map.py b/reclaimer/meta/wrappers/halo_reach_map.py index 5159e618..cae81c23 100644 --- a/reclaimer/meta/wrappers/halo_reach_map.py +++ b/reclaimer/meta/wrappers/halo_reach_map.py @@ -11,4 +11,7 @@ class HaloReachMap(Halo3Map): tag_defs_module = "" - tag_classes_to_load = tuple() + tag_classes_to_load = () + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None \ No newline at end of file diff --git a/reclaimer/meta/wrappers/shadowrun_map.py b/reclaimer/meta/wrappers/shadowrun_map.py index 43f27f8d..9dc31608 100644 --- a/reclaimer/meta/wrappers/shadowrun_map.py +++ b/reclaimer/meta/wrappers/shadowrun_map.py @@ -7,13 +7,15 @@ # See LICENSE for more information. # -from reclaimer.meta.wrappers.halo1_map import Halo1Map +from reclaimer.meta.wrappers.halo1_xbox_map import Halo1XboxMap from reclaimer.shadowrun_prototype.handler import ShadowrunPrototypeHandler from reclaimer.shadowrun_prototype.constants import sr_tag_class_fcc_to_ext from supyr_struct.defs.frozen_dict import FrozenDict -class ShadowrunMap(Halo1Map): +class ShadowrunMap(Halo1XboxMap): + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. defs = None handler_class = ShadowrunPrototypeHandler diff --git a/reclaimer/meta/wrappers/stubbs_map.py b/reclaimer/meta/wrappers/stubbs_map.py index 2f44b27b..9f03dca1 100644 --- a/reclaimer/meta/wrappers/stubbs_map.py +++ b/reclaimer/meta/wrappers/stubbs_map.py @@ -7,26 +7,42 @@ # See LICENSE for more information. # -from reclaimer.meta.wrappers.halo1_map import Halo1Map +from reclaimer.meta.wrappers.halo1_xbox_map import Halo1XboxMap from reclaimer.stubbs.constants import stubbs_tag_class_fcc_to_ext from reclaimer.stubbs.handler import StubbsHandler from supyr_struct.defs.frozen_dict import FrozenDict -class StubbsMap(Halo1Map): - defs = None +class StubbsMap(Halo1XboxMap): + xbox_defs = None + pc_defs = None handler_class = StubbsHandler tag_defs_module = StubbsHandler.default_defs_path tag_classes_to_load = tuple(sorted(stubbs_tag_class_fcc_to_ext.keys())) + + @property + def defs(self): + this_class = type(self) + return this_class.pc_defs if self.engine == "stubbspc" else this_class.xbox_defs + @defs.setter + def defs(self, val): + this_class = type(self) + if self.engine == "stubbspc": + this_class.pc_defs = val + else: + this_class.xbox_defs = val + + def is_indexed(self, tag_id): + return False def setup_defs(self): this_class = type(self) - if not this_class.defs: + if not(this_class.xbox_defs and this_class.pc_defs): print(" Loading definitions in %s" % self.handler_class.default_defs_path) - this_class.defs = defs = {} + this_class.xbox_defs = this_class.pc_defs = {} # these imports were moved here because their defs would otherwise # be built when this module was imported, which is not good practice @@ -39,16 +55,16 @@ def setup_defs(self): this_class.handler = self.handler_class( build_reflexive_cache=False, build_raw_data_cache=False, debug=2) - this_class.defs = dict(this_class.handler.defs) - - if self.engine == "stubbspc": - this_class.defs["mode"] = pc_mode_def - else: - this_class.defs["mode"] = mode_def - this_class.defs["antr"] = antr_def - this_class.defs["coll"] = coll_def - this_class.defs["sbsp"] = sbsp_def - this_class.defs = FrozenDict(this_class.defs) + this_class.xbox_defs = dict(this_class.handler.defs) + this_class.xbox_defs.update( + antr=antr_def, coll=coll_def, sbsp=sbsp_def, mode=mode_def + ) + this_class.xbox_defs = FrozenDict(this_class.xbox_defs) + this_class.pc_defs = dict(self.xbox_defs) + this_class.pc_defs.update(mode=pc_mode_def) + + this_class.xbox_defs = FrozenDict(this_class.xbox_defs) + this_class.pc_defs = FrozenDict(this_class.pc_defs) # make a shallow copy for this instance to manipulate self.defs = dict(self.defs) diff --git a/reclaimer/meta/wrappers/stubbs_map_64bit.py b/reclaimer/meta/wrappers/stubbs_map_64bit.py new file mode 100644 index 00000000..e9201a85 --- /dev/null +++ b/reclaimer/meta/wrappers/stubbs_map_64bit.py @@ -0,0 +1,27 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.meta.wrappers.stubbs_map import * + +# so, turns out the steam re-release of stubbs uses 64bit pointers in most +# areas of the map(aside from the header it seems). We're going to "support" +# this enough to load and display the map header and index, but nothing else. +# extracting from 64bit stubbs can come later, but for now lets at least not +# have Reclaimer or Refinery crash when trying to load these maps. +class StubbsMap64Bit(StubbsMap): + handler_class = StubbsHandler + + tag_defs_module = StubbsHandler.default_defs_path + tag_classes_to_load = () + # NOTE: setting defs to None so setup_defs doesn't think the + # defs are setup cause of class property inheritance. + defs = None + + def setup_defs(self): + self.defs = {} diff --git a/reclaimer/misc/defs/composer_playlist.py b/reclaimer/misc/defs/composer_playlist.py index 49dfe7c8..c6ffdb82 100644 --- a/reclaimer/misc/defs/composer_playlist.py +++ b/reclaimer/misc/defs/composer_playlist.py @@ -11,7 +11,7 @@ from supyr_struct.field_types import * from supyr_struct.defs.tag_def import TagDef -from binilla.constants import VISIBILITY_METADATA +from reclaimer.constants import VISIBILITY_METADATA def get(): return composer_playlist_def diff --git a/reclaimer/misc/defs/fmod.py b/reclaimer/misc/defs/fmod.py new file mode 100644 index 00000000..1963418d --- /dev/null +++ b/reclaimer/misc/defs/fmod.py @@ -0,0 +1,223 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.common_descs import * +from .objs.fmod import FModSoundBankTag,\ + FMOD_BANK_HEADER_SIZE, FMOD_SAMPLE_CHUNK_SIZE +from supyr_struct.defs.tag_def import TagDef + + +def get(): + return fmod_bank_def, fmod_list_def + + +def _get_fmod_bank(block): + while block and getattr(block, "NAME", "") != "fmod_bank": + block = block.parent + return block + + +def has_next_chunk(parent=None, **kwargs): + if parent is None: + return False + return (parent[-1] if parent else parent.parent).header.has_next_chunk + + +def sample_data_size( + parent=None, node=None, attr_index=None, + new_val=None, root_offset=0, offset=0, rawdata=None, + **kwargs + ): + if new_val is not None: + # can't set size here + return + elif parent is not None and attr_index is not None: + node = parent[attr_index] + + if node is not None: + return len(node) + + sample_array = getattr(parent, "parent", None) + if not sample_array or not rawdata: + # NOTE: checking if rawdata is passed to indicate that + # we're actually trying to parse from something. + # if it's not, we've appended an empty block. + return 0 + + next_sample_index = sample_array.index_by_id(parent) + 1 + if next_sample_index >= len(sample_array): + start = parent.header.data_qword_offset * FMOD_SAMPLE_CHUNK_SIZE + end = sample_array.parent.header.sample_data_size + else: + start = sample_data_pointer(parent=parent) + end = sample_data_pointer(parent=sample_array[next_sample_index]) + + return max(0, end - start) + + +def sample_data_pointer(parent=None, root_offset=0, offset=0, **kwargs): + sample_header = parent.header + bank_header = _get_fmod_bank(sample_header).header + return ( + root_offset + offset + FMOD_BANK_HEADER_SIZE + + bank_header.sample_headers_size + + bank_header.sample_names_size + + sample_header.data_qword_offset * FMOD_SAMPLE_CHUNK_SIZE + ) + + +def sample_name_offsets_pointer(parent=None, root_offset=0, offset=0, **kwargs): + return ( + root_offset + offset + FMOD_BANK_HEADER_SIZE + + _get_fmod_bank(parent).header.sample_headers_size + ) + + +def sample_name_pointer(parent=None, root_offset=0, offset=0, **kwargs): + return sample_name_offsets_pointer( + parent, root_offset, offset + ) + parent.offset + + +sound_list_header = Container("sound_list_header", + UInt32('string_len'), + StrUtf8('sound_name', + SIZE='.string_len', + WIDGET_WIDTH=100 + ), + UInt32('sample_index'), + UInt32('sample_count'), + ) + +fmod_list_header = Struct("header", + UInt32("version", DEFAULT=1), + UInt32("sound_count"), + SIZE=8 + ) + +fmod_list_def = TagDef('fmod_list', + fmod_list_header, + Array("sample_headers", + SUB_STRUCT=sound_list_header, SIZE=".header.sound_count", + DYN_NAME_PATH=".sound_name", WIDGET=DynamicArrayFrame + ), + ext='.bin', endian='<' + ) + +chunk_header = BitStruct("header", + Bit("has_next_chunk"), + UBitInt("size", SIZE=24), + UBitEnum("type", + "invalid", + "channels", + "frequency", + "loop", + "unknown4", + "unknown5", + "xma_seek", + "dsp_coeff", + "unknown8", + "unknown9", + "xwma_data", + "vorbis_data", + SIZE=4, + ), + Pad(3), + SIZE=4 + ) + +chunk = Container("chunk", + chunk_header, + BytesRaw("data", SIZE=".header.size"), + ) + +sample_header = BitStruct("header", + Bit("has_next_chunk"), + UBitEnum("frequency", + "invalid", + "hz_8000", + "hz_11000", + "hz_11025", + "hz_16000", + "hz_22050", + "hz_24000", + "hz_32000", + "hz_44100", + "hz_48000", + SIZE=4 + ), + Bit("channel_count"), # add 1 cause at least 1 is assumed + UBitInt("data_qword_offset", SIZE=28), + UBitInt("sample_count", SIZE=30), + SIZE=8, + ) + +sample = Container("sample", + sample_header, + WhileArray("chunks", + CASE=has_next_chunk, SUB_STRUCT=chunk + ), + STEPTREE=BytesRaw("sample_data", + SIZE=sample_data_size, POINTER=sample_data_pointer + ) + ) + +sample_name = QStruct("sample_name", + UInt32("offset"), + STEPTREE=CStrUtf8("name", + POINTER=sample_name_pointer, + WIDGET_WIDTH=100 + ), + SIZE=4, + ) + +fmod_bank_header = Struct("header", + UInt32('sig', DEFAULT=893539142), # 'FSB5' bytes as little-endian uint32 + UInt32("version", DEFAULT=1), + UInt32("sample_count"), + UInt32("sample_headers_size"), + UInt32("sample_names_size"), + UInt32("sample_data_size"), + UEnum32("mode", + "none", + "pcm8", + "pcm16", + "pcm24", + "pcm32", + "pcm_float", + "gcadpcm", + "imaadpcm", + "vag", + "hevag", + "xma", + "mpeg", + "celt", + "at9", + "xwma", + "vorbis", + ), + Pad(8), + StrHex('hash', SIZE=8), + StrHex('guid', SIZE=16), + SIZE=FMOD_BANK_HEADER_SIZE + ) + +fmod_bank_def = TagDef('fmod_bank', + fmod_bank_header, + Array("samples", + SUB_STRUCT=sample, + SIZE=".header.sample_count" + ), + Array("names", + SUB_STRUCT=sample_name, SIZE=".header.sample_count", + POINTER=sample_name_offsets_pointer, + DYN_NAME_PATH=".name", WIDGET=DynamicArrayFrame, + ), + ext='.fsb', endian='<', tag_cls=FModSoundBankTag + ) diff --git a/reclaimer/misc/defs/objs/fmod.py b/reclaimer/misc/defs/objs/fmod.py new file mode 100644 index 00000000..182b29b7 --- /dev/null +++ b/reclaimer/misc/defs/objs/fmod.py @@ -0,0 +1,52 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from supyr_struct.tag import Tag + +FMOD_BANK_HEADER_SIZE = 60 +FMOD_SAMPLE_CHUNK_SIZE = 16 +FMOD_SAMPLE_DATA_ALIGN = 32 + +class FModSoundBankTag(Tag): + + def set_pointers(self, offset=0): + header = self.data.header + samples = self.data.samples + names = self.data.names + if len(samples) != len(names): + raise ValueError("Number of samples does not match number of names.") + + header.sample_count = len(samples) + sample_headers_size = sample_names_size = sample_data_size = 0 + + for sample in samples: + sample_headers_size += sample.header.binsize + sample.chunks.binsize + sample.header.data_qword_offset = sample_data_size + sample_size = ( + len(sample.sample_data) + FMOD_SAMPLE_CHUNK_SIZE - 1 + ) // FMOD_SAMPLE_CHUNK_SIZE + + sample_data_size += sample_size + + sample_names_size += 4*header.sample_count # string offset sizes + for name in names: + name.offset = sample_names_size + sample_names_size += len(name.name) + 1 # add 1 for null terminator + + sample_data_off = FMOD_BANK_HEADER_SIZE + sample_headers_size + sample_names_size + sample_name_padding = ( + (FMOD_SAMPLE_DATA_ALIGN - sample_data_off % FMOD_SAMPLE_DATA_ALIGN) + ) % FMOD_SAMPLE_DATA_ALIGN + header.sample_headers_size = sample_headers_size + header.sample_names_size = sample_names_size + sample_name_padding + header.sample_data_size = sample_data_size*FMOD_SAMPLE_CHUNK_SIZE + + def serialize(self, *args, **kwargs): + self.set_pointers() + super().serialize(*args, **kwargs) \ No newline at end of file diff --git a/reclaimer/misc/defs/recorded_animations.py b/reclaimer/misc/defs/recorded_animations.py new file mode 100644 index 00000000..748ad855 --- /dev/null +++ b/reclaimer/misc/defs/recorded_animations.py @@ -0,0 +1,325 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from math import pi +from supyr_struct.field_types import * +from supyr_struct.defs.tag_def import TagDef +from reclaimer.common_descs import * + +import traceback + +# Faster than if/elif chain +FUNCTION_MAP = { + "end_anim" : None, + "set_anim_state" : "anim_state", + "set_aim_speed" : "aim_speed", + "set_control_flags" : "control_flags", + "set_weapon_index" : "weapon_index", + "set_throttle" : "throttle", + "set_face_char" : "angle_delta_byte", + "set_aim_char" : "angle_delta_byte", + "set_face_aim_char" : "angle_delta_byte", + "set_look_char" : "angle_delta_byte", + "set_face_look_char" : "angle_delta_byte", + "set_aim_look_char" : "angle_delta_byte", + "set_face_aim_look_char" : "angle_delta_byte", + "set_face_short" : "angle_delta_short", + "set_aim_short" : "angle_delta_short", + "set_face_aim_short" : "angle_delta_short", + "set_look_short" : "angle_delta_short", + "set_face_look_short" : "angle_delta_short", + "set_aim_look_short" : "angle_delta_short", + "set_face_aim_look_short" : "angle_delta_short", + } +ANGLE_DELTA_SCALE = pi/1000 + + +def build_r_a_stream_block(version, rawdata, simple=False): + v0_def = r_a_stream_v0_simple_def if simple else r_a_stream_v0_def + v3_def = r_a_stream_v3_simple_def if simple else r_a_stream_v3_def + v4_def = r_a_stream_v4_simple_def if simple else r_a_stream_v4_def + return ( + v0_def.build(rawdata=rawdata) if version == 0 else + v3_def.build(rawdata=rawdata) if version == 3 else + v4_def.build(rawdata=rawdata) if version == 4 else + None + ) + + +def has_next_event(parent=None, rawdata=None, **kwargs): + try: + # Proper case + if len(parent) and parent[-1].event_function.function.enum_name == "end_anim": + return False + # End of stream reached before end packet read from stream + elif len(rawdata.peek(1)) < 1: + print("Warning: End of data reached before an end packet was read. Tag may be corrupt!") + return False + return True + except AttributeError: + return False + + +def time_delta_case(parent=None, **kwargs): + return parent.event_function.size.enum_name if parent else None + + +def parameter_case(parent=None, **kwargs): + func_name = parent.event_function.function.enum_name if parent else None + return FUNCTION_MAP.get(func_name, None) + + +def get_event_summary_string(parent=None, **kw): + if parent is None: + return None + + try: + param_strs = [] + + time_type = parent.event_function.time_delay_type.enum_name + func_type = parent.event_function.function.enum_name + param_strs.append( + "[%.3fs] " % (( + 1 if time_type == "tick" else + 0 if time_type == "instant" else + parent.time.delay + ) / 30) + ) + + if func_type != "end_anim": + param_strs.append("%s {" % func_type.\ + replace("char", "").replace("short", "").strip("_") + ) + else: + param_strs.append("end_anim") + + params_desc = "" + params = parent.parameters + if func_type == "set_anim_state": + params_desc = "%s}" % params.animation_state.enum_name + elif func_type == "set_aim_speed": + params_desc = "%s}" % params.aim_speed.enum_name + elif func_type == "set_control_flags": + flags = params.control_flags + params_desc = "%s}" % "|".join( + name.split("_")[0] for name in flags.NAME_MAP + if flags[name] + ) or "None" + elif func_type == "set_weapon_index": + params_desc = "%s}" % params.weapon_index + elif func_type != "end_anim": + scale = 1 if func_type == "set_throttle" else ANGLE_DELTA_SCALE + + params_desc = ", ".join( + "%.3f" % ( + params[name] * scale + ) + for name in params.NAME_MAP + ) + "}" + + if params_desc: + param_strs.append(params_desc) + + parent.summary = "".join(param_strs) + except Exception: + print(traceback.format_exc()) + + +animation_state = UEnum8("animation_state", + "sleep", + "alert1", + "alert2", + "stand1", + "stand2", + "flee", + "flaming", + ) + +aim_speed = UEnum8("aim_speed", + "alert", + "relaxed", + ) + +control_flags = Bool16("control_flags", + "crouch", + "jump", + "user1", + "user2", + "flashlight", + "lock_facing", + "action", + "melee", + "unlock_facing", + "walk", + "reload", + "primary_trigger", + "secondary_trigger", + "grenade", + "exchange", + ) + +angle_delta_byte = QStruct("", + SInt8("x"), SInt8("y"), ORIENT="h" + ) + +angle_delta_short = QStruct("", + SInt16("x"), SInt16("y"), ORIENT="h" + ) + +event_function = BitStruct("event_function", + UBitEnum("time_delay_type", + "instant", + "tick", + "byte", # Suggested edit: The code asserts 1 < time_delta < UNSIGNED_CHAR_MAX + "short", # Suggested edit: The code asserts UNSIGNED_CHAR_MAX < time_delta + SIZE=2 + ), + UBitEnum("function", # Suggested edit: The code asserts this to being n < NUMBEROF(apply_funcs) [0x5b] + ("end_anim", 1), # 1 + "set_anim_state", # 2 + "set_aim_speed", # 3 + "set_control_flags", # 4 + "set_weapon_index", # 5 + "set_throttle", # 6 + # start of char difference # 7 For future mgmt: This isn't skipped, we just precalculated the flags below + ("set_face_char", 8), # 8 + "set_aim_char", # 9 + "set_face_aim_char", # 10 + "set_look_char", # 11 + "set_face_look_char", # 12 + "set_aim_look_char", # 13 + "set_face_aim_look_char", # 14 + # start of short difference # 15 For future mgmt: This isn't skipped, we just precalculated the flags below + ("set_face_short", 16), # 16 + "set_aim_short", # 17 + "set_face_aim_short", # 18 + "set_look_short", # 19 + "set_face_look_short", # 20 + "set_aim_look_short", # 21 + "set_face_aim_look_short", # 22 + SIZE=6 + ), + SIZE=1, HIDE_TITLE=True + ) + +event_fields = ( + event_function, + Switch("time", + CASES={ + "byte" : QStruct("time", UInt8("delay")), # Renamed to "delay" as I feel it better represents what's going on here + "short" : QStruct("time", UInt16("delay")), + }, + CASE=".event_function.time_delay_type.enum_name" + ), + Switch("parameters", + CASES={ + "aim_speed" : Struct("aim_speed", aim_speed), + "control_flags" : Struct("control_flags", control_flags), + "anim_state" : Struct("anim_state", animation_state), + "weapon_index" : QStruct("weapon_index", SInt16("weapon_index")), + "throttle" : QStruct("throttle", INCLUDE=xy_float), + "angle_delta_byte" : QStruct("angle_delta", INCLUDE=angle_delta_byte), + "angle_delta_short" : QStruct("angle_delta", INCLUDE=angle_delta_short), + }, + CASE=parameter_case, + ) + ) + +event = Container("event", + *event_fields, + # NOTE: summary is after event so the event data is already parsed + Computed("summary", + COMPUTE_READ=get_event_summary_string, WIDGET_WIDTH=60 + ) + ) + +event_simple = Container("event", + *event_fields, + ) + +events_simple = WhileArray("events", + SUB_STRUCT=event_simple, CASE=has_next_event + ) + +events = WhileArray("events", + SUB_STRUCT=event, CASE=has_next_event, + DYN_NAME_PATH=".summary", WIDGET=DynamicArrayFrame + ) + +# Suggested edit: the "stream_headers" versions are actually "control" versions internally +r_a_stream_header_fields_0 = ( + animation_state, + aim_speed, + control_flags, + # This is weird and I hate it. Still trying to figure out what is going on here + SInt8("unknown_01", DEFAULT=-1), + SInt8("weapon_index", DEFAULT=-1, MIN=-1, MAX=3), # clamped[-1,4) + + UInt8("unknown_02"), + UInt8("unknown_03"), + + QStruct("velocity", INCLUDE=xy_float), + QStruct("facing_vector", INCLUDE=xyz_float), #, EDITABLE=False), # For now I'm gonna keep editable + QStruct("aiming_vector", INCLUDE=xyz_float), #, EDITABLE=False), + QStruct("looking_vector", INCLUDE=xyz_float), #, EDITABLE=False), + ) + +r_a_stream_header_fields_1 = ( + QStruct("facing", INCLUDE=angle_delta_short), + QStruct("aiming", INCLUDE=angle_delta_short), + QStruct("looking", INCLUDE=angle_delta_short), + ) + +r_a_stream_header_v0 = Struct("r_a_stream_header_v0", + *r_a_stream_header_fields_0, + *r_a_stream_header_fields_1, + SIZE=64 + ) + +r_a_stream_header_v3 = Struct("r_a_stream_header_v3", + *r_a_stream_header_fields_0, + + SInt16("unknown_04"), + SInt16("unknown_05"), + SInt16("grenade_index", MIN=0, MAX=3), + + *r_a_stream_header_fields_1, + SIZE=70 + ) + +r_a_stream_header_v4 = Struct("r_a_stream_header_v4", + *r_a_stream_header_fields_0, + + SInt16("unknown_04"), + SInt16("unknown_05"), + SInt16("grenade_index", MIN=0, MAX=3), # Must be -1 < n < 4 + SInt16("zoom_level", MIN=0), # Must be positive + + *r_a_stream_header_fields_1, + SIZE=72 + ) + +r_a_stream_v0 = Container("r_a_stream", r_a_stream_header_v0, events) +r_a_stream_v3 = Container("r_a_stream", r_a_stream_header_v3, events) +r_a_stream_v4 = Container("r_a_stream", r_a_stream_header_v4, events) + +r_a_stream_v0_def = BlockDef(r_a_stream_v0) +r_a_stream_v3_def = BlockDef(r_a_stream_v3) +r_a_stream_v4_def = BlockDef(r_a_stream_v4) + +r_a_stream_v0_simple_def = BlockDef("r_a_stream", r_a_stream_header_v0, events_simple) +r_a_stream_v3_simple_def = BlockDef("r_a_stream", r_a_stream_header_v3, events_simple) +r_a_stream_v4_simple_def = BlockDef("r_a_stream", r_a_stream_header_v4, events_simple) + +r_a_stream_v0_tagdef = TagDef(r_a_stream_v0, endian="<") +r_a_stream_v3_tagdef = TagDef(r_a_stream_v3, endian="<") +r_a_stream_v4_tagdef = TagDef(r_a_stream_v4, endian="<") + +def get(): + return r_a_stream_v0_tagdef, r_a_stream_v3_tagdef, r_a_stream_v4_tagdef diff --git a/reclaimer/misc/handler.py b/reclaimer/misc/handler.py index 8671376a..4226b891 100644 --- a/reclaimer/misc/handler.py +++ b/reclaimer/misc/handler.py @@ -10,6 +10,8 @@ from pathlib import Path import os +# NOTE: this is a pretty tough dependency to move to make +# reclaimer able to operate without binilla installed. from binilla.handler import Handler from reclaimer.misc.defs import __all__ as all_def_names diff --git a/reclaimer/model/constants.py b/reclaimer/model/constants.py index f8787891..73b70fad 100644 --- a/reclaimer/model/constants.py +++ b/reclaimer/model/constants.py @@ -22,6 +22,8 @@ HALO_1_MAX_MARKERS = 256 +HALO_1_MAX_MARKERS_PER_PERM = 32 + # If a jms file is prefixed with this token it # cannot be randomly chosen as a permutation @@ -29,5 +31,18 @@ SCALE_INTERNAL_TO_JMS = 100.0 -JMS_VERSION_HALO_1 = "8200" -JMS_VERSION_HALO_2_8210 = "8210" +JMS_VER_HALO_1_OLDEST_KNOWN = 8197 +JMS_VER_HALO_1_TRI_REGIONS = 8198 +JMS_VER_HALO_1_3D_UVWS = 8199 +JMS_VER_HALO_1_MARKER_RADIUS = 8200 +JMS_VER_HALO_1_RETAIL = JMS_VER_HALO_1_MARKER_RADIUS +JMS_VER_HALO_2_RETAIL = 8210 + +JMS_VER_ALL = frozenset(( + JMS_VER_HALO_1_OLDEST_KNOWN, + JMS_VER_HALO_1_TRI_REGIONS, + JMS_VER_HALO_1_3D_UVWS, + JMS_VER_HALO_1_MARKER_RADIUS, + JMS_VER_HALO_1_RETAIL, + JMS_VER_HALO_2_RETAIL, + )) \ No newline at end of file diff --git a/reclaimer/model/jms/file.py b/reclaimer/model/jms/file.py index 32f0ae15..655309e7 100644 --- a/reclaimer/model/jms/file.py +++ b/reclaimer/model/jms/file.py @@ -17,8 +17,9 @@ from pathlib import Path from reclaimer.model.constants import ( - JMS_VERSION_HALO_1, - JMS_VERSION_HALO_2_8210, + JMS_VER_HALO_1_OLDEST_KNOWN, + JMS_VER_HALO_1_RETAIL, + JMS_VER_HALO_2_RETAIL, JMS_PERM_CANNOT_BE_RANDOMLY_CHOSEN_TOKEN, ) from reclaimer.util import ( @@ -37,14 +38,24 @@ # https://regex101.com/r/ySgI1Z/1 JMS_V2_SPLIT_REGEX = re.compile(r'\n+(;.*)*\s*|\t+') + def read_jms(jms_string, stop_at="", perm_name=None): ''' Converts a jms string into a JmsModel instance. ''' - jms_data = tuple(JMS_V1_SPLIT_REGEX.split(jms_string)) + jms_string = jms_string.lstrip("\ufeff") + jms_data = tuple(JMS_V1_SPLIT_REGEX.split(jms_string)) + try: + version = int(jms_data[0].strip()) + except ValueError: + version = 0 - version = jms_data[0].strip() - if version == JMS_VERSION_HALO_1: + if (version < JMS_VER_HALO_1_OLDEST_KNOWN or + version > JMS_VER_HALO_2_RETAIL): + print("Unknown JMS version '%s'" % version) + return None + + if version <= JMS_VER_HALO_1_RETAIL: # Halo 1 return _read_jms_8200(jms_data, stop_at, perm_name) @@ -52,16 +63,12 @@ def read_jms(jms_string, stop_at="", perm_name=None): # is a semicolon, and the line must start with it. # filter out any lines that start with a semicolon. - jms_string = jms_string.lstrip("\ufeff") jms_data = tuple(JMS_V2_SPLIT_REGEX.split(jms_string)) version = jms_data[0].strip() - if version == JMS_VERSION_HALO_2_8210: + if version <= JMS_VER_HALO_2_RETAIL: # Halo 2 return _read_jms_8210(jms_data, stop_at) - else: - print("Unknown JMS version '%s'" % version) - return None def _read_jms_8200(jms_data, stop_at="", perm_name=None): @@ -74,7 +81,7 @@ def _read_jms_8200(jms_data, stop_at="", perm_name=None): dat_i = 0 try: - jms_model.version = str(parse_jm_int(jms_data[dat_i])) + jms_model.version = int(parse_jm_int(jms_data[dat_i])) dat_i += 1 except Exception: print(traceback.format_exc()) @@ -89,6 +96,10 @@ def _read_jms_8200(jms_data, stop_at="", perm_name=None): print("Could not read node list checksum.") return jms_model + if jms_model.node_list_checksum >= 0x80000000: + # jms gave us an unsigned checksum.... sign it + jms_model.node_list_checksum -= 0x100000000 + stop = (stop_at == "nodes") if not stop: # read the nodes @@ -135,15 +146,25 @@ def _read_jms_8200(jms_data, stop_at="", perm_name=None): jms_model.markers[:] = (None, ) * parse_jm_int(jms_data[dat_i]) dat_i += 1 for i in range(len(jms_model.markers)): + marker_name = jms_data[dat_i] + region = 0 + radius = 1 + if jms_model.has_marker_regions: + region = parse_jm_int(jms_data[dat_i+1]) + dat_i += 1 + + if jms_model.has_marker_radius: + radius = parse_jm_float(jms_data[dat_i+9]) + jms_model.markers[i] = JmsMarker( - jms_data[dat_i], jms_model.name, - parse_jm_int(jms_data[dat_i+1]), parse_jm_int(jms_data[dat_i+2]), - parse_jm_float(jms_data[dat_i+3]), parse_jm_float(jms_data[dat_i+4]), - parse_jm_float(jms_data[dat_i+5]), parse_jm_float(jms_data[dat_i+6]), - parse_jm_float(jms_data[dat_i+7]), parse_jm_float(jms_data[dat_i+8]), parse_jm_float(jms_data[dat_i+9]), - parse_jm_float(jms_data[dat_i+10]) + marker_name, jms_model.name, + region, parse_jm_int(jms_data[dat_i+1]), + parse_jm_float(jms_data[dat_i+2]), parse_jm_float(jms_data[dat_i+3]), + parse_jm_float(jms_data[dat_i+4]), parse_jm_float(jms_data[dat_i+5]), + parse_jm_float(jms_data[dat_i+6]), parse_jm_float(jms_data[dat_i+7]), parse_jm_float(jms_data[dat_i+8]), + radius ) - dat_i += 11 + dat_i += 9 + jms_model.has_marker_radius except Exception: print(traceback.format_exc()) print("Failed to read markers.") @@ -174,6 +195,15 @@ def _read_jms_8200(jms_data, stop_at="", perm_name=None): jms_model.verts[:] = (None, ) * parse_jm_int(jms_data[dat_i]) dat_i += 1 for i in range(len(jms_model.verts)): + region = 0 + w_coord = 0 + if jms_model.has_vert_regions: + region = parse_jm_int(jms_data[dat_i]) + dat_i += 1 + + if jms_model.has_3d_uvws: + w_coord = parse_jm_int(jms_data[dat_i+11]) + jms_model.verts[i] = JmsVertex( parse_jm_int(jms_data[dat_i]), parse_jm_float(jms_data[dat_i+1]), parse_jm_float(jms_data[dat_i+2]), parse_jm_float(jms_data[dat_i+3]), @@ -182,9 +212,10 @@ def _read_jms_8200(jms_data, stop_at="", perm_name=None): min(1.0, max(-1.0, parse_jm_float(jms_data[dat_i+5]))), min(1.0, max(-1.0, parse_jm_float(jms_data[dat_i+6]))), parse_jm_int(jms_data[dat_i+7]), parse_jm_float(jms_data[dat_i+8]), - parse_jm_float(jms_data[dat_i+9]), parse_jm_float(jms_data[dat_i+10]), parse_jm_float(jms_data[dat_i+11]) + parse_jm_float(jms_data[dat_i+9]), parse_jm_float(jms_data[dat_i+10]), w_coord, + region=region ) - dat_i += 12 + dat_i += 11 + jms_model.has_3d_uvws except Exception: print(traceback.format_exc()) print("Failed to read vertices.") @@ -199,17 +230,38 @@ def _read_jms_8200(jms_data, stop_at="", perm_name=None): jms_model.tris[:] = (None, ) * parse_jm_int(jms_data[dat_i]) dat_i += 1 for i in range(len(jms_model.tris)): + region = 0 + if jms_model.has_face_regions: + region = parse_jm_int(jms_data[dat_i]) + dat_i += 1 + jms_model.tris[i] = JmsTriangle( - parse_jm_int(jms_data[dat_i]), parse_jm_int(jms_data[dat_i+1]), - parse_jm_int(jms_data[dat_i+2]), parse_jm_int(jms_data[dat_i+3]), parse_jm_int(jms_data[dat_i+4]), + region, parse_jm_int(jms_data[dat_i]), + parse_jm_int(jms_data[dat_i+1]), parse_jm_int(jms_data[dat_i+2]), parse_jm_int(jms_data[dat_i+3]), ) - dat_i += 5 + dat_i += 4 except Exception: print(traceback.format_exc()) print("Failed to read triangles.") del jms_model.tris[i: ] stop = True + # copy the region from the verts into triangles, or + # from the triangles into the verts + try: + verts = jms_model.verts + if jms_model.has_vert_regions: + for tri in jms_model.tris: + tri.region = verts[tri.v0].region + else: + for tri in jms_model.tris: + verts[tri.v0].region = tri.region + verts[tri.v1].region = tri.region + verts[tri.v2].region = tri.region + except Exception: + print(traceback.format_exc()) + print("Failed to copy regions between vertices and triangles.") + # return (jms_models[name], ) return jms_model @@ -527,18 +579,26 @@ def write_jms(filepath, jms_model, use_blitzkrieg_rounding=False): f.write("%s\n" % len(materials)) for mat in materials: - f.write("%s\n%s\n" % (mat.name + mat.properties, mat.tiff_path)) + f.write("%s%s%d\n%s\n" % ( + mat.name, mat.properties, mat.permutation_index, + mat.tiff_path + )) f.write("%s\n" % len(jms_model.markers)) for marker in jms_model.markers: - f.write("%s\n%s\n%s\n%s\t%s\t%s\t%s\n%s\t%s\t%s\n%s\n" % ( - marker.name[: 31], marker.region, marker.parent, + f.write("%s\n" % marker.name[: 31]) + if jms_model.has_marker_regions: + f.write("%s\n" % marker.region) + + f.write("%s\n%s\t%s\t%s\t%s\n%s\t%s\t%s\n" % ( + marker.parent, to_str(marker.rot_i), to_str(marker.rot_j), to_str(marker.rot_k), to_str(marker.rot_w), - to_str(marker.pos_x), to_str(marker.pos_y), to_str(marker.pos_z), - to_str(marker.radius) - ) - ) + to_str(marker.pos_x), to_str(marker.pos_y), to_str(marker.pos_z) + )) + + if jms_model.has_marker_radius: + f.write("%s\n" % to_str(marker.radius)) f.write("%s\n" % len(regions)) for region in regions: @@ -546,6 +606,9 @@ def write_jms(filepath, jms_model, use_blitzkrieg_rounding=False): f.write("%s\n" % len(jms_model.verts)) for vert in jms_model.verts: + if jms_model.has_vert_regions: + f.write("%s\n" % vert.region) + f.write("%s\n%s\t%s\t%s\n%s\t%s\t%s\n%s\n%s\n%s\n%s\n%s\n" % ( vert.node_0, to_str(vert.pos_x), to_str(vert.pos_y), to_str(vert.pos_z), @@ -558,8 +621,10 @@ def write_jms(filepath, jms_model, use_blitzkrieg_rounding=False): f.write("%s\n" % len(jms_model.tris)) for tri in jms_model.tris: - f.write("%s\n%s\n%s\t%s\t%s\n" % ( - tri.region, tri.shader, - tri.v0, tri.v1, tri.v2 + if jms_model.has_marker_regions: + f.write("%s\n" % tri.region) + + f.write("%s\n%s\t%s\t%s\n" % ( + tri.shader, tri.v0, tri.v1, tri.v2 ) ) diff --git a/reclaimer/model/jms/material.py b/reclaimer/model/jms/material.py index b1df6d08..f06b07f0 100644 --- a/reclaimer/model/jms/material.py +++ b/reclaimer/model/jms/material.py @@ -14,10 +14,15 @@ class JmsMaterial: __slots__ = ( "name", "tiff_path", "shader_path", "shader_type", - "properties" + "properties", "permutation_index" ) def __init__(self, name="__unnamed", tiff_path="", shader_path="", shader_type="", properties=""): + permutation_index = "0" + while name[-1:] in "0123456789": + permutation_index += name[-1] + name = name[:-1] + for c in "!@#$%^&*-.": if c in name and c not in properties: properties += c @@ -28,6 +33,7 @@ def __init__(self, name="__unnamed", tiff_path="", self.shader_path = shader_path if shader_path else name self.shader_type = shader_type self.properties = properties + self.permutation_index = int(permutation_index) @property def ai_defeaning(self): return "&" in self.properties @@ -92,6 +98,6 @@ def render_only(self, new_val): self.properties = self.properties.replace("!", "") + ("!" if new_val else "") def __repr__(self): - return """JmsMaterial(name=%s, - tiff_path=%s -)""" % (self.name, self.tiff_path) + return """JmsMaterial(name=%s%s, + tiff_path=%s, properties=%s +)""" % (self.name, self.permutation_index, self.tiff_path, self.properties) diff --git a/reclaimer/model/jms/merged_model.py b/reclaimer/model/jms/merged_model.py index 76fa0c57..970a90c1 100644 --- a/reclaimer/model/jms/merged_model.py +++ b/reclaimer/model/jms/merged_model.py @@ -75,8 +75,10 @@ def merge_jms_model(self, other_model): for mat in other_model.materials: self.materials.append( JmsMaterial( - mat.name, mat.tiff_path, mat.shader_path, - mat.shader_type, mat.properties) + mat.name + str(mat.permutation_index), + mat.tiff_path, mat.shader_path, + mat.shader_type, mat.properties + ) ) self.regions = {} diff --git a/reclaimer/model/jms/model.py b/reclaimer/model/jms/model.py index 37db7182..2c3f2169 100644 --- a/reclaimer/model/jms/model.py +++ b/reclaimer/model/jms/model.py @@ -12,7 +12,9 @@ import math -from ..constants import JMS_PERM_CANNOT_BE_RANDOMLY_CHOSEN_TOKEN +from ..constants import JMS_PERM_CANNOT_BE_RANDOMLY_CHOSEN_TOKEN,\ + JMS_VER_HALO_1_TRI_REGIONS, JMS_VER_HALO_1_3D_UVWS,\ + JMS_VER_HALO_1_MARKER_RADIUS, JMS_VER_HALO_1_RETAIL class JmsModel: @@ -34,7 +36,7 @@ class JmsModel: def __init__(self, name="", node_list_checksum=0, nodes=None, materials=None, markers=None, regions=None, - verts=None, tris=None, version="8200"): + verts=None, tris=None, version=JMS_VER_HALO_1_RETAIL): name = name.strip(" ") perm_name = name @@ -61,6 +63,17 @@ def __init__(self, name="", node_list_checksum=0, nodes=None, self.markers = markers if markers else [] self.verts = verts if verts else [] self.tris = tris if tris else [] + + @property + def has_face_regions(self): return self.version >= JMS_VER_HALO_1_TRI_REGIONS + @property + def has_marker_radius(self): return self.version >= JMS_VER_HALO_1_MARKER_RADIUS + @property + def has_marker_regions(self): return self.has_face_regions + @property + def has_vert_regions(self): return not self.has_marker_regions + @property + def has_3d_uvws(self): return self.version >= JMS_VER_HALO_1_3D_UVWS def calculate_vertex_normals(self): verts = self.verts diff --git a/reclaimer/model/jms/node.py b/reclaimer/model/jms/node.py index 98f8be42..3137246c 100644 --- a/reclaimer/model/jms/node.py +++ b/reclaimer/model/jms/node.py @@ -9,7 +9,7 @@ __all__ = ( 'JmsNode', ) -from ..constants import ( JMS_VERSION_HALO_1, JMS_VERSION_HALO_2_8210, ) +from ..constants import ( JMS_VER_HALO_1_RETAIL, JMS_VER_HALO_2_RETAIL, ) class JmsNode: __slots__ = ( @@ -45,7 +45,7 @@ def __repr__(self): def __eq__(self, other): if not isinstance(other, JmsNode): return False - elif self.name != other.name: + elif self.name.lower() != other.name.lower(): return False elif self.first_child != other.first_child: return False @@ -62,11 +62,13 @@ def __eq__(self, other): return False return True - def is_node_hierarchy_equal(self, other): + def is_node_hierarchy_equal(self, other, name_only=False): if not isinstance(other, JmsNode): return False - elif self.name != other.name: + elif self.name.lower() != other.name.lower(): return False + elif name_only: + pass elif self.first_child != other.first_child: return False elif self.sibling_index != other.sibling_index: @@ -74,8 +76,8 @@ def is_node_hierarchy_equal(self, other): return True @classmethod - def setup_node_hierarchy(cls, nodes, jms_version=JMS_VERSION_HALO_1): - if jms_version == JMS_VERSION_HALO_1: + def setup_node_hierarchy(cls, nodes, jms_version=JMS_VER_HALO_1_RETAIL): + if jms_version == JMS_VER_HALO_1_RETAIL: # Halo 1 parented_nodes = set() # setup the parent node hierarchy @@ -93,6 +95,6 @@ def setup_node_hierarchy(cls, nodes, jms_version=JMS_VERSION_HALO_1): sib_node = nodes[sib_idx] sib_node.parent_index = parent_idx sib_idx = sib_node.sibling_index - elif jms_version == JMS_VERSION_HALO_2_8210: + elif jms_version == JMS_VER_HALO_2_RETAIL: # Halo 2 pass diff --git a/reclaimer/model/jms/vertex.py b/reclaimer/model/jms/vertex.py index 8764f1ad..efe40887 100644 --- a/reclaimer/model/jms/vertex.py +++ b/reclaimer/model/jms/vertex.py @@ -19,7 +19,8 @@ class JmsVertex: "tangent_i", "tangent_j", "tangent_k", "node_1", "node_1_weight", "tex_u", "tex_v", "tex_w", - "other_nodes", "other_weights", "other_uvws" + "other_nodes", "other_weights", "other_uvws", + "region" ) def __init__(self, node_0=0, pos_x=0.0, pos_y=0.0, pos_z=0.0, @@ -28,7 +29,9 @@ def __init__(self, node_0=0, tex_u=0, tex_v=0, tex_w=0, binorm_i=0.0, binorm_j=1.0, binorm_k=0.0, tangent_i=1.0, tangent_j=0.0, tangent_k=0.0, - other_nodes=(), other_weights=(), other_uvws=()): + other_nodes=(), other_weights=(), other_uvws=(), + region=0, + ): if node_1_weight <= 0: node_1 = -1 node_1_weight = 0 @@ -54,18 +57,21 @@ def __init__(self, node_0=0, self.other_nodes = other_nodes self.other_weights = other_weights self.other_uvws = other_uvws + self.region = region def __repr__(self): return """JmsVertex(node_0=%s, x=%s, y=%s, z=%s, i=%s, j=%s, k=%s, node_1=%s, node_1_weight=%s, - u=%s, v=%s, w=%s + u=%s, v=%s, w=%s, + region=%s )""" % (self.node_0, self.pos_x, self.pos_y, self.pos_z, self.norm_i, self.norm_j, self.norm_k, self.node_1, self.node_1_weight, - self.tex_u, self.tex_v, self.tex_w) + self.tex_u, self.tex_v, self.tex_w, + self.region) def __eq__(self, other): if not isinstance(other, JmsVertex): @@ -81,6 +87,8 @@ def __eq__(self, other): return False elif abs(self.node_1_weight - other.node_1_weight) > 0.0001: return False + elif self.region != other.region: + return False elif self.node_0 != other.node_0 or self.node_1 != other.node_1: return False diff --git a/reclaimer/model/model_compilation.py b/reclaimer/model/model_compilation.py index 487c7226..e2ee50d4 100644 --- a/reclaimer/model/model_compilation.py +++ b/reclaimer/model/model_compilation.py @@ -12,7 +12,7 @@ from reclaimer.model.constants import ( HALO_1_MAX_MATERIALS, HALO_1_MAX_REGIONS, HALO_1_MAX_GEOMETRIES_PER_MODEL, - SCALE_INTERNAL_TO_JMS, HALO_1_NAME_MAX_LEN + SCALE_INTERNAL_TO_JMS, HALO_1_NAME_MAX_LEN, HALO_1_MAX_MARKERS_PER_PERM ) from reclaimer.model.jms import GeometryMesh from reclaimer.model.stripify import Stripifier @@ -34,7 +34,6 @@ def compile_gbxmodel(mod2_tag, merged_jms, ignore_errors=False): tagdata.node_list_checksum = merged_jms.node_list_checksum - errors = [] if len(merged_jms.materials) > HALO_1_MAX_MATERIALS: errors.append("Too many materials. Max count is %s." % (HALO_1_MAX_MATERIALS)) @@ -67,30 +66,22 @@ def compile_gbxmodel(mod2_tag, merged_jms, ignore_errors=False): node.pos_x**2 + node.pos_y**2 + node.pos_z**2) / SCALE_INTERNAL_TO_JMS - # record shader ordering and permutation indices + # make shader references mod2_shaders = tagdata.shaders.STEPTREE - shdr_perm_indices_by_name = {} - for mod2_shader in mod2_shaders: - shdr_name = mod2_shader.shader.filepath.split("\\")[-1] - shdr_perm_indices = shdr_perm_indices_by_name.setdefault(shdr_name, []) - shdr_perm_indices.append(mod2_shader.permutation_index) - del mod2_shaders[:] - # make shader references for mat in merged_jms.materials: mod2_shaders.append() mod2_shader = mod2_shaders[-1] - mod2_shader.shader.filepath = mat.shader_path + + mod2_shader.shader.filepath = mat.shader_path + mod2_shader.permutation_index = mat.permutation_index + if mat.shader_type: mod2_shader.shader.tag_class.set_to(mat.shader_type) else: mod2_shader.shader.tag_class.set_to("shader") shdr_name = mod2_shader.shader.filepath.split("\\")[-1].lower() - shdr_perm_indices = shdr_perm_indices_by_name.get(shdr_name) - if shdr_perm_indices: - mod2_shader.permutation_index = shdr_perm_indices.pop(0) - # make regions mod2_regions = tagdata.regions.STEPTREE @@ -98,7 +89,6 @@ def compile_gbxmodel(mod2_tag, merged_jms, ignore_errors=False): global_markers = {} geom_meshes = [] - all_lod_nodes = {lod: set([0]) for lod in util.LOD_NAMES} for region_name in sorted(merged_jms.regions): region = merged_jms.regions[region_name] @@ -129,17 +119,6 @@ def compile_gbxmodel(mod2_tag, merged_jms, ignore_errors=False): lod_mesh = perm.lod_meshes[lod_name] geom_meshes.append(lod_mesh) - # figure out which nodes this mesh utilizes - this_meshes_nodes = set() - for mesh in lod_mesh.values(): - for vert in mesh.verts: - if vert.node_1_weight < 1: - this_meshes_nodes.add(vert.node_0) - if vert.node_1_weight > 0: - this_meshes_nodes.add(vert.node_1) - - all_lod_nodes[lod_name].update(this_meshes_nodes) - lods_to_set = list(range(i, 5)) if skipped_lods: lods_to_set.extend(skipped_lods) @@ -152,25 +131,24 @@ def compile_gbxmodel(mod2_tag, merged_jms, ignore_errors=False): perm_added = True - # What are we doing here? - if len(perm.markers) > 32: - for marker in perm.markers: - global_markers.setdefault( - marker.name[: HALO_1_NAME_MAX_LEN], []).append(marker) - else: - perm_added |= bool(perm.markers) - mod2_markers = mod2_perm.local_markers.STEPTREE - for marker in perm.markers: - mod2_markers.append() - mod2_marker = mod2_markers[-1] + if len(perm.markers) > HALO_1_MAX_MARKERS_PER_PERM and not ignore_errors: + return ("Cannot add more than %s markers to a permutation. " + "This model would contain %s markers." % ( + HALO_1_MAX_MARKERS_PER_PERM, len(perm.markers)), ) - mod2_marker.name = marker.name[: HALO_1_NAME_MAX_LEN] - mod2_marker.node_index = marker.parent - mod2_marker.translation[:] = marker.pos_x / SCALE_INTERNAL_TO_JMS,\ - marker.pos_y / SCALE_INTERNAL_TO_JMS,\ - marker.pos_z / SCALE_INTERNAL_TO_JMS - mod2_marker.rotation[:] = marker.rot_i, marker.rot_j,\ - marker.rot_k, marker.rot_w + perm_added |= bool(perm.markers) + mod2_markers = mod2_perm.local_markers.STEPTREE + for marker in perm.markers: + mod2_markers.append() + mod2_marker = mod2_markers[-1] + + mod2_marker.name = marker.name[: HALO_1_NAME_MAX_LEN] + mod2_marker.node_index = marker.parent + mod2_marker.translation[:] = marker.pos_x / SCALE_INTERNAL_TO_JMS,\ + marker.pos_y / SCALE_INTERNAL_TO_JMS,\ + marker.pos_z / SCALE_INTERNAL_TO_JMS + mod2_marker.rotation[:] = marker.rot_i, marker.rot_j,\ + marker.rot_k, marker.rot_w if not(perm_added or ignore_errors): @@ -217,19 +195,6 @@ def compile_gbxmodel(mod2_tag, merged_jms, ignore_errors=False): mod2_marker.rotation[:] = marker.rot_i, marker.rot_j,\ marker.rot_k, marker.rot_w - # set the node counts per lod - for lod in util.LOD_NAMES: - lod_nodes = all_lod_nodes[lod] - adding = True - node_ct = len(mod2_nodes) - - for i in range(node_ct - 1, -1, -1): - if i in lod_nodes: - break - node_ct -= 1 - - setattr(tagdata, "%s_lod_nodes" % lod, max(0, node_ct - 1)) - # calculate triangle strips stripped_geom_meshes = [] @@ -320,3 +285,6 @@ def compile_gbxmodel(mod2_tag, merged_jms, ignore_errors=False): mod2_tris[i] = tri >> 8 mod2_tris[i + 1] = tri & 0xFF i += 2 + + # calculate final bits of data before saving + mod2_tag.calc_internal_data() diff --git a/reclaimer/model/model_decompilation.py b/reclaimer/model/model_decompilation.py index 2e1284b9..a5d3804f 100644 --- a/reclaimer/model/model_decompilation.py +++ b/reclaimer/model/model_decompilation.py @@ -82,9 +82,11 @@ def extract_model(tagdata, tag_path="", **kw): )) for b in tagdata.shaders.STEPTREE: - materials.append(JmsMaterial( - b.shader.filepath.split("/")[-1].split("\\")[-1]) - ) + shader_name = b.shader.filepath.replace("/", "\\").split("\\")[-1] + if b.permutation_index != 0: + shader_name += str(b.permutation_index) + + materials.append(JmsMaterial(shader_name)) markers_by_perm = {} geoms_by_perm_lod_region = {} @@ -136,12 +138,8 @@ def extract_model(tagdata, tag_path="", **kw): region_geoms.append(geom_index) last_geom_index = geom_index - try: - use_local_nodes = tagdata.flags.parts_have_local_nodes - except Exception: - use_local_nodes = False - def_node_map = list(range(128)) - def_node_map.append(-1) + can_have_local_nodes = hasattr(tagdata.flags, "parts_have_local_nodes") + def_node_map = [*range(128), -1] # use big endian since it will have been byteswapped comp_vert_unpacker = PyStruct(">3f3I2h2bh").unpack_from @@ -181,60 +179,65 @@ def extract_model(tagdata, tag_path="", **kw): v_origin = len(verts) shader_index = part.shader_index - try: - node_map = list(part.local_nodes) - node_map.append(-1) - compressed = False - except (AttributeError, KeyError): - compressed = True - - if not use_local_nodes: - node_map = def_node_map - - try: - unparsed = isinstance( - part.triangles.STEPTREE.data, bytearray) - except Exception: - unparsed = False + node_map = ( + [*part.local_nodes, -1] + if can_have_local_nodes and part.flags.ZONER else + def_node_map + ) + + tris_steptree = part.triangles.STEPTREE + cverts_steptree = part.compressed_vertices.STEPTREE + ucverts_steptree = part.uncompressed_vertices.STEPTREE + unparsed = isinstance( + getattr(tris_steptree, "data", None), + (bytearray, bytes) + ) + compressed = ( + bool(unparsed and getattr(cverts_steptree, "data", None)) or + bool(not unparsed and cverts_steptree) + ) + uncompressed = ( + bool(unparsed and getattr(ucverts_steptree, "data", None)) or + bool(not unparsed and ucverts_steptree) + ) + # prefer uncompressed vertices if both exist + compressed = compressed and not uncompressed - # TODO: Make this work in meta(parse verts and tris) try: - if compressed and unparsed: - vert_data = part.compressed_vertices.STEPTREE.data - for off in range(0, len(vert_data), 32): - v = comp_vert_unpacker(vert_data, off) - verts.append(JmsVertex( - v[8]//3, - v[0] * 100, v[1] * 100, v[2] * 100, - *decompress_normal32(v[3]), - v[9]//3, 1.0 - (v[10]/32767), - u_scale * v[6]/32767, 1.0 - v_scale * v[7]/32767)) - elif compressed: - for v in part.compressed_vertices.STEPTREE: - verts.append(JmsVertex( + vert_data = cverts_steptree if compressed else ucverts_steptree + if unparsed: + # if verts are unparsed, parse them + unpack, v_size = ( + (comp_vert_unpacker, 32) if compressed else + (uncomp_vert_unpacker, 68) + ) + data = vert_data.data + vert_data = [ + unpack(data, i) for i in range(0, len(data), v_size) + ] + + if compressed: + verts.extend( + JmsVertex( v[8]//3, v[0] * 100, v[1] * 100, v[2] * 100, *decompress_normal32(v[3]), v[9]//3, 1.0 - (v[10]/32767), - u_scale * v[6]/32767, 1.0 - v_scale * v[7]/32767)) - elif not compressed and unparsed: - vert_data = part.uncompressed_vertices.STEPTREE.data - for off in range(0, len(vert_data), 68): - v = uncomp_vert_unpacker(vert_data, off) - verts.append(JmsVertex( - node_map[v[14]], - v[0] * 100, v[1] * 100, v[2] * 100, - v[3], v[4], v[5], - node_map[v[15]], max(0, min(1, v[17])), - u_scale * v[12], 1.0 - v_scale * v[13])) + u_scale * v[6]/32767, 1.0 - v_scale * v[7]/32767 + ) + for v in vert_data + ) else: - for v in part.uncompressed_vertices.STEPTREE: - verts.append(JmsVertex( + verts.extend( + JmsVertex( node_map[v[14]], v[0] * 100, v[1] * 100, v[2] * 100, v[3], v[4], v[5], node_map[v[15]], max(0, min(1, v[17])), - u_scale * v[12], 1.0 - v_scale * v[13])) + u_scale * v[12], 1.0 - v_scale * v[13] + ) + for v in vert_data + ) except Exception: print(format_exc()) print("If you see this, tell Moses to stop " @@ -242,7 +245,7 @@ def extract_model(tagdata, tag_path="", **kw): try: if unparsed: - tri_block = part.triangles.STEPTREE.data + tri_block = tris_steptree.data tri_list = [-1] * (len(tri_block) // 2) for i in range(len(tri_list)): # assuming big endian @@ -252,9 +255,8 @@ def extract_model(tagdata, tag_path="", **kw): if tri_list[i] > 32767: tri_list[i] = -1 else: - tri_block = part.triangles.STEPTREE tri_list = [] - for triangle in tri_block: + for triangle in tris_steptree: tri_list.extend(triangle) swap = True diff --git a/reclaimer/model/util.py b/reclaimer/model/util.py index 8fd057e4..0fe8d3e8 100644 --- a/reclaimer/model/util.py +++ b/reclaimer/model/util.py @@ -9,6 +9,7 @@ import os +from reclaimer.constants import LOD_NAMES, SINT16_MAX from reclaimer.model.jms import JmsVertex from reclaimer.hek.defs.scex import scex_def from reclaimer.hek.defs.schi import schi_def @@ -31,16 +32,15 @@ mod2_verts_def = BlockDef( - raw_reflexive("vertices", mod2_vert_struct, 65535), + raw_reflexive("vertices", mod2_vert_struct, SINT16_MAX), endian='>' ) mod2_tri_strip_def = BlockDef( - raw_reflexive("triangle", mod2_tri_struct, 65535), + raw_reflexive("triangle", mod2_tri_struct, SINT16_MAX), endian='>' ) -LOD_NAMES = ("superhigh", "high", "medium", "low", "superlow") MAX_STRIP_LEN = 32763 * 3 EMPTY_GEOM_VERTS = ( diff --git a/reclaimer/os_hek/defs/actv.py b/reclaimer/os_hek/defs/actv.py index 2c47d2b0..5ac9110a 100644 --- a/reclaimer/os_hek/defs/actv.py +++ b/reclaimer/os_hek/defs/actv.py @@ -7,21 +7,10 @@ # See LICENSE for more information. # -from supyr_struct.util import desc_variant - from ...hek.defs.actv import * -# Create opensauce variant of grenade descriptor. - -os_actv_grenades = desc_variant(actv_grenades, - ("grenade_type", SEnum16("grenade_type", *grenade_types_os)), -) - -# Create os variant of actv descriptor using the new grenade descriptor. - -actv_body = desc_variant(actv_body, - ("grenades", os_actv_grenades), -) +grenades = desc_variant(actv_grenades, SEnum16("grenade_type", *grenade_types_os)) +actv_body = desc_variant(actv_body, grenades) def get(): return actv_def diff --git a/reclaimer/os_hek/defs/antr.py b/reclaimer/os_hek/defs/antr.py index e964f5bd..322c33aa 100644 --- a/reclaimer/os_hek/defs/antr.py +++ b/reclaimer/os_hek/defs/antr.py @@ -8,12 +8,11 @@ # from ...hek.defs.antr import * -from supyr_struct.util import desc_variant -antr_body = desc_variant( - antr_body, - ("animations", reflexive( - "animations", animation_desc, 2048, DYN_NAME_PATH=".name")) +antr_body = desc_variant(antr_body, + # original maximum according to comment? + # https://github.com/HaloMods/OpenSauce/blob/master/OpenSauce/Halo1/Halo1_CheApe/Halo1_CheApe_Readme.txt#L40 + reflexive("animations", animation_desc, 500, DYN_NAME_PATH=".name") ) def get(): diff --git a/reclaimer/os_hek/defs/bipd.py b/reclaimer/os_hek/defs/bipd.py index 816a1fc3..7ca9ff98 100644 --- a/reclaimer/os_hek/defs/bipd.py +++ b/reclaimer/os_hek/defs/bipd.py @@ -8,17 +8,11 @@ # from ...hek.defs.bipd import * - -#import and use the open saucified obje and unit attrs from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=0) - -bipd_body = Struct("tagdata", +obje_attrs = obje_attrs_variant(obje_attrs, "bipd") +bipd_body = Struct("tagdata", obje_attrs, unit_attrs, bipd_attrs, diff --git a/reclaimer/os_hek/defs/ctrl.py b/reclaimer/os_hek/defs/ctrl.py index 2766cec6..ee9135ee 100644 --- a/reclaimer/os_hek/defs/ctrl.py +++ b/reclaimer/os_hek/defs/ctrl.py @@ -8,17 +8,10 @@ # from ...hek.defs.ctrl import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=8) - -ctrl_body = dict(ctrl_body) -ctrl_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ctrl") +ctrl_body = desc_variant(ctrl_body, obje_attrs) def get(): return ctrl_def diff --git a/reclaimer/os_hek/defs/eqip.py b/reclaimer/os_hek/defs/eqip.py index 0db84889..74a5fd59 100644 --- a/reclaimer/os_hek/defs/eqip.py +++ b/reclaimer/os_hek/defs/eqip.py @@ -8,21 +8,11 @@ # from ...hek.defs.eqip import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=3) - -eqip_attrs = dict(eqip_attrs) -eqip_attrs[1] = SEnum16("grenade_type", *grenade_types_os) - -eqip_body = dict(eqip_body) -eqip_body[0] = obje_attrs -eqip_body[2] = eqip_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "eqip") +eqip_attrs = desc_variant(eqip_attrs, SEnum16("grenade_type", *grenade_types_os)) +eqip_body = desc_variant(eqip_body, obje_attrs, eqip_attrs) def get(): return eqip_def diff --git a/reclaimer/os_hek/defs/garb.py b/reclaimer/os_hek/defs/garb.py index 6c6a1bd4..37861ec4 100644 --- a/reclaimer/os_hek/defs/garb.py +++ b/reclaimer/os_hek/defs/garb.py @@ -8,17 +8,10 @@ # from ...hek.defs.garb import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=4) - -garb_body = dict(garb_body) -garb_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "garb") +garb_body = desc_variant(garb_body, obje_attrs) def get(): diff --git a/reclaimer/os_hek/defs/lifi.py b/reclaimer/os_hek/defs/lifi.py index 505b32d4..bcc0afdb 100644 --- a/reclaimer/os_hek/defs/lifi.py +++ b/reclaimer/os_hek/defs/lifi.py @@ -8,17 +8,10 @@ # from ...hek.defs.lifi import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=9) - -lifi_body = dict(lifi_body) -lifi_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "lifi") +lifi_body = desc_variant(lifi_body, obje_attrs) def get(): return lifi_def diff --git a/reclaimer/os_hek/defs/mach.py b/reclaimer/os_hek/defs/mach.py index 493331fc..42ba018a 100644 --- a/reclaimer/os_hek/defs/mach.py +++ b/reclaimer/os_hek/defs/mach.py @@ -8,17 +8,10 @@ # from ...hek.defs.mach import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=7) - -mach_body = dict(mach_body) -mach_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "mach") +mach_body = desc_variant(mach_body, obje_attrs) def get(): return mach_def diff --git a/reclaimer/os_hek/defs/magy.py b/reclaimer/os_hek/defs/magy.py index 9e94c898..1db831d3 100644 --- a/reclaimer/os_hek/defs/magy.py +++ b/reclaimer/os_hek/defs/magy.py @@ -8,31 +8,11 @@ # from .objs.magy import MagyTag -from ...hek.defs.antr import * +from .antr import * -magy_body = Struct("tagdata", - reflexive("objects", object_desc, 4), - reflexive("units", unit_desc, 32, DYN_NAME_PATH=".label"), - reflexive("weapons", weapon_desc, 1), - reflexive("vehicles", vehicle_desc, 1), - reflexive("devices", device_desc, 1), - reflexive("unit_damages", anim_enum_desc, 176, - *unit_damage_animation_names - ), - reflexive("fp_animations", fp_animation_desc, 1), - #i have no idea why they decided to cap it at 257 instead of 256.... - reflexive("sound_references", sound_reference_desc, 257, - DYN_NAME_PATH=".sound.filepath"), - Float("limp_body_node_radius"), - Bool16("flags", - "compress_all_animations", - "force_idle_compression", - ), - Pad(2), - reflexive("nodes", nodes_desc, 64, DYN_NAME_PATH=".name"), - reflexive("animations", animation_desc, 2048, DYN_NAME_PATH=".name"), - dependency_os("stock_animation", valid_model_animations_yelo), - SIZE=300, +magy_body = desc_variant(antr_body, + ("pad_13", dependency_os("stock_animation", valid_model_animations_yelo)), + SIZE=300, verify=False ) diff --git a/reclaimer/os_hek/defs/matg.py b/reclaimer/os_hek/defs/matg.py index c403f47a..1f2c31f1 100644 --- a/reclaimer/os_hek/defs/matg.py +++ b/reclaimer/os_hek/defs/matg.py @@ -13,8 +13,9 @@ def get(): return matg_def # replace the grenades reflexive with an open sauce one -matg_body = dict(matg_body) -matg_body[5] = reflexive("grenades", grenade, 4, *grenade_types_os) +matg_body = desc_variant(matg_body, + reflexive("grenades", grenade, 4, *grenade_types_os) + ) matg_def = TagDef("matg", blam_header_os('matg', 3), diff --git a/reclaimer/os_hek/defs/obje.py b/reclaimer/os_hek/defs/obje.py index 3ca5f1fb..7792fc71 100644 --- a/reclaimer/os_hek/defs/obje.py +++ b/reclaimer/os_hek/defs/obje.py @@ -9,17 +9,14 @@ from ...hek.defs.obje import * +obje_attrs = desc_variant(obje_attrs, + dependency('animation_graph', valid_model_animations_yelo) + ) +obje_attrs = obje_attrs_variant(obje_attrs) + def get(): return obje_def -# replace the model animations dependency with an open sauce one -obje_attrs = dict(obje_attrs) -obje_attrs[8] = dependency('animation_graph', valid_model_animations_yelo) - -obje_body = Struct('tagdata', - obje_attrs - ) - obje_def = TagDef("obje", blam_header('obje'), obje_body, diff --git a/reclaimer/os_hek/defs/objs/scnr.py b/reclaimer/os_hek/defs/objs/scnr.py new file mode 100644 index 00000000..f9adf45c --- /dev/null +++ b/reclaimer/os_hek/defs/objs/scnr.py @@ -0,0 +1,13 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.hek.defs.objs.scnr import ScnrTag + +class OsScnrTag(ScnrTag): + engine = "halo1yelo" diff --git a/reclaimer/os_hek/defs/plac.py b/reclaimer/os_hek/defs/plac.py index 655d8c49..95382ef4 100644 --- a/reclaimer/os_hek/defs/plac.py +++ b/reclaimer/os_hek/defs/plac.py @@ -8,17 +8,10 @@ # from ...hek.defs.plac import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=10) - -plac_body = dict(plac_body) -plac_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "plac") +plac_body = desc_variant(plac_body, obje_attrs) def get(): return plac_def diff --git a/reclaimer/os_hek/defs/proj.py b/reclaimer/os_hek/defs/proj.py index 3b01be73..cb05774a 100644 --- a/reclaimer/os_hek/defs/proj.py +++ b/reclaimer/os_hek/defs/proj.py @@ -8,17 +8,10 @@ # from ...hek.defs.proj import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=5) - -proj_body = dict(proj_body) -proj_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "proj") +proj_body = desc_variant(proj_body, obje_attrs) def get(): return proj_def diff --git a/reclaimer/os_hek/defs/scen.py b/reclaimer/os_hek/defs/scen.py index b56c75d3..d18b4285 100644 --- a/reclaimer/os_hek/defs/scen.py +++ b/reclaimer/os_hek/defs/scen.py @@ -8,17 +8,10 @@ # from ...hek.defs.scen import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=6) - -scen_body = dict(scen_body) -scen_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "scen") +scen_body = desc_variant(scen_body, obje_attrs) def get(): return scen_def diff --git a/reclaimer/os_hek/defs/scnr.py b/reclaimer/os_hek/defs/scnr.py index bc79a10c..752fc104 100644 --- a/reclaimer/os_hek/defs/scnr.py +++ b/reclaimer/os_hek/defs/scnr.py @@ -8,68 +8,36 @@ # from ...hek.defs.scnr import * -from supyr_struct.util import desc_variant +from .objs.scnr import OsScnrTag -player_starting_profile = Struct("player_starting_profile", - ascii_str32("name"), - float_zero_to_one("starting_health_modifier"), - float_zero_to_one("starting_shield_modifier"), - dependency("primary_weapon", "weap"), - SInt16("primary_rounds_loaded"), - SInt16("primary_rounds_total"), - dependency("secondary_weapon", "weap"), - SInt16("secondary_rounds_loaded"), - SInt16("secondary_rounds_total"), - SInt8("starting_frag_grenade_count", MIN=0), - SInt8("starting_plasma_grenade_count", MIN=0), - SInt8("starting_custom_2_grenade_count", MIN=0), - SInt8("starting_custom_3_grenade_count", MIN=0), - SIZE=104 +player_starting_profile = desc_variant(player_starting_profile, + ("pad_11", SInt8("starting_custom_2_grenade_count", MIN=0)), + ("pad_12", SInt8("starting_custom_3_grenade_count", MIN=0)), ) -ai_anim_reference = Struct("ai_animation_reference", - ascii_str32("animation_name"), - dependency_os("animation_graph", ("antr", "magy")), - SIZE=60 +ai_anim_reference = desc_variant(ai_anim_reference, + dependency_os("animation_graph", ("antr", "magy")) ) -reference = Struct("tag_reference", - Pad(24), - dependency_os("reference"), - SIZE=40 - ) +reference = desc_variant(reference, dependency_os("reference")) -# copy the scnr_body and replace the descriptors for certain -# fields with ones that are tweaked for use with open sauce -scnr_body = desc_variant( - scnr_body, +scnr_body = desc_variant(scnr_body, ("DONT_USE", dependency_os("project_yellow_definitions", 'yelo')), - ("player_starting_profiles", - reflexive("player_starting_profiles", - player_starting_profile, 256, DYN_NAME_PATH='.name') - ), - ("ai_animation_references", - reflexive("ai_animation_references", - ai_anim_reference, 128, DYN_NAME_PATH='.animation_name') - ), - ("script_syntax_data", rawdata_ref("script_syntax_data", max_size=570076, IGNORE_SAFE_MODE=True)), - ("script_string_data", rawdata_ref("script_string_data", max_size=393216, IGNORE_SAFE_MODE=True)), - ("references", - reflexive("references", - reference, 256, DYN_NAME_PATH='.reference.filepath') - ), - ("structure_bsps", - reflexive("structure_bsps", - structure_bsp, 32, DYN_NAME_PATH='.structure_bsp.filepath') - ) + reflexive("player_starting_profiles", player_starting_profile, 256, DYN_NAME_PATH='.name'), + reflexive("ai_animation_references", ai_anim_reference, 128, DYN_NAME_PATH='.animation_name'), + rawdata_ref("script_syntax_data", max_size=570076, IGNORE_SAFE_MODE=True), + rawdata_ref("script_string_data", max_size=393216, IGNORE_SAFE_MODE=True), + reflexive("references", reference, 256, DYN_NAME_PATH='.reference.filepath'), + reflexive("structure_bsps", structure_bsp, 32, DYN_NAME_PATH='.structure_bsp.filepath'), ) def get(): return scnr_def +# TODO: update dependencies scnr_def = TagDef("scnr", blam_header('scnr', 2), scnr_body, - ext=".scenario", endian=">", tag_cls=HekTag + ext=".scenario", endian=">", tag_cls=OsScnrTag ) diff --git a/reclaimer/os_hek/defs/soso.py b/reclaimer/os_hek/defs/soso.py index 926f2dc3..8b7e52f4 100644 --- a/reclaimer/os_hek/defs/soso.py +++ b/reclaimer/os_hek/defs/soso.py @@ -51,9 +51,7 @@ When the opensauce extension is used the tint values in here are overwritten by the ones in the os extension when the map is loaded.""" -reflection = Struct("reflection", - INCLUDE=reflection, COMMENT=os_reflection_prop_comment - ) +reflection = desc_variant(reflection, COMMENT=os_reflection_prop_comment) os_soso_ext = Struct("shader_model_extension", #Specular Color @@ -102,35 +100,8 @@ SIZE=192, ) -soso_attrs = Struct("soso_attrs", - #Model Shader Properties - model_shader, - - Pad(16), - #Color-Change - SEnum16("color_change_source", *function_names, COMMENT=cc_comment), - - Pad(30), - #Self-Illumination - self_illumination, - - Pad(12), - #Diffuse, Multipurpose, and Detail Maps - maps, - - reflexive("os_shader_model_ext", os_soso_ext, 1), - - #Texture Scrolling Animation - texture_scrolling, - - Pad(8), - #Reflection Properties - reflection, - Pad(16), - - FlFloat("unknown0", VISIBLE=False), - BytesRaw("unknown1", SIZE=16, VISIBLE=False), # little endian dependency - SIZE=400 +soso_attrs = desc_variant(soso_attrs, + ("pad_7", reflexive("os_shader_model_ext", os_soso_ext, 1)), ) soso_body = Struct("tagdata", diff --git a/reclaimer/os_hek/defs/ssce.py b/reclaimer/os_hek/defs/ssce.py index abd90da8..d5926367 100644 --- a/reclaimer/os_hek/defs/ssce.py +++ b/reclaimer/os_hek/defs/ssce.py @@ -8,17 +8,10 @@ # from ...hek.defs.ssce import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=11) - -ssce_body = dict(ssce_body) -ssce_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ssce") +ssce_body = desc_variant(ssce_body, obje_attrs) def get(): return ssce_def diff --git a/reclaimer/os_hek/defs/tagc.py b/reclaimer/os_hek/defs/tagc.py index bfdf97ee..d6dabd90 100644 --- a/reclaimer/os_hek/defs/tagc.py +++ b/reclaimer/os_hek/defs/tagc.py @@ -7,18 +7,14 @@ # See LICENSE for more information. # -from ...common_descs import * -from ...hek.defs.objs.tag import HekTag -from supyr_struct.defs.tag_def import TagDef +from ...hek.defs.tagc import * -tag_reference = Struct("tag_reference", - dependency_os("tag"), - SIZE=16 - ) +tag_reference = desc_variant(tag_reference, dependency_os("tag")) -tagc_body = Struct("tagdata", - reflexive("tag_references", tag_reference, 200), - SIZE=12, +tagc_body = desc_variant(tagc_body, + reflexive("tag_references", tag_reference, 200, + DYN_NAME_PATH='.tag.filepath' + ) ) diff --git a/reclaimer/os_hek/defs/unit.py b/reclaimer/os_hek/defs/unit.py index 57ff5d0b..c4b16198 100644 --- a/reclaimer/os_hek/defs/unit.py +++ b/reclaimer/os_hek/defs/unit.py @@ -9,10 +9,7 @@ from ...hek.defs.unit import * -# replace the grenade types enumerator with an open sauce one -unit_attrs = dict(unit_attrs) -unit_attrs[49] = SEnum16('grenade_type', *grenade_types_os) - +unit_attrs = desc_variant(unit_attrs, SEnum16('grenade_type', *grenade_types_os)) unit_body = Struct('tagdata', unit_attrs) def get(): diff --git a/reclaimer/os_hek/defs/vehi.py b/reclaimer/os_hek/defs/vehi.py index ded29cf0..d59f21af 100644 --- a/reclaimer/os_hek/defs/vehi.py +++ b/reclaimer/os_hek/defs/vehi.py @@ -10,19 +10,14 @@ from ...hek.defs.vehi import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=1) - -vehi_body = Struct("tagdata", +obje_attrs = obje_attrs_variant(obje_attrs, "vehi") +vehi_body = Struct("tagdata", obje_attrs, unit_attrs, vehi_attrs, SIZE=1008, ) - def get(): return vehi_def diff --git a/reclaimer/os_hek/defs/weap.py b/reclaimer/os_hek/defs/weap.py index 276bbaf3..a11a3a15 100644 --- a/reclaimer/os_hek/defs/weap.py +++ b/reclaimer/os_hek/defs/weap.py @@ -8,17 +8,10 @@ # from ...hek.defs.weap import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=2) - -weap_body = dict(weap_body) -weap_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "weap") +weap_body = desc_variant(weap_body, obje_attrs) def get(): return weap_def diff --git a/reclaimer/os_hek/defs/yelo.py b/reclaimer/os_hek/defs/yelo.py index 1aef998f..52bb800c 100644 --- a/reclaimer/os_hek/defs/yelo.py +++ b/reclaimer/os_hek/defs/yelo.py @@ -22,6 +22,9 @@ "release", ), SInt32("revision"), + Timestamp64("timestamp", VISIBLE=False), + StrHex("uuid", SIZE=16, VISIBLE=False), + SIZE=48 ) @@ -73,22 +76,26 @@ reflexive("build_info", build_info, 1), Pad(40), + reflexive("scripted_ui_widgets", scripted_ui_widget, 128, DYN_NAME_PATH='.name'), - Pad(16), + # Physics Float("gravity_scale", MIN=0.0, MAX=2.0, SIDETIP="[0,2]"), Float("player_speed_scale", MIN=0.0, MAX=6.0, SIDETIP="[0,6]"), + Pad(20), + + Bool32("networking_flags", VISIBLE=False), # unused + Pad(20), - Pad(44), Bool32("gameplay_model", "prohibit_multi_team_vehicles", ), - Pad(20), + reflexive("yelo_scripting", yelo_scripting, 1), - Pad(12),#reflexive("unknown", void_desc), + Pad(92), SIZE=312 ) diff --git a/reclaimer/os_v3_hek/defs/antr.py b/reclaimer/os_v3_hek/defs/antr.py index 811f9e16..0d01d23c 100644 --- a/reclaimer/os_v3_hek/defs/antr.py +++ b/reclaimer/os_v3_hek/defs/antr.py @@ -8,3 +8,23 @@ # from ...os_hek.defs.antr import * + +antr_body = desc_variant(antr_body, + # open sauce increases all these + # https://github.com/HaloMods/OpenSauce/blob/master/OpenSauce/shared/Include/blamlib/Halo1/models/model_animation_definitions.hpp + reflexive("units", unit_desc, 64, DYN_NAME_PATH=".label"), + reflexive("sound_references", sound_reference_desc, 257*2, + DYN_NAME_PATH=".sound.filepath" + ), + reflexive("animations", animation_desc, 2048, DYN_NAME_PATH=".name") + ) + +def get(): + return antr_def + +antr_def = TagDef("antr", + blam_header('antr', 4), + antr_body, + + ext=".model_animations", endian=">", tag_cls=AntrTag + ) \ No newline at end of file diff --git a/reclaimer/os_v3_hek/defs/bipd.py b/reclaimer/os_v3_hek/defs/bipd.py index a112e45f..bdc918a1 100644 --- a/reclaimer/os_v3_hek/defs/bipd.py +++ b/reclaimer/os_v3_hek/defs/bipd.py @@ -8,16 +8,10 @@ # from ...os_hek.defs.bipd import * - -#import and use the open saucified obje and unit attrs from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=0) - +obje_attrs = obje_attrs_variant(obje_attrs, "bipd") bipd_body = Struct("tagdata", obje_attrs, unit_attrs, diff --git a/reclaimer/os_v3_hek/defs/cdmg.py b/reclaimer/os_v3_hek/defs/cdmg.py index 3212b0a0..fe7e2e32 100644 --- a/reclaimer/os_v3_hek/defs/cdmg.py +++ b/reclaimer/os_v3_hek/defs/cdmg.py @@ -9,51 +9,32 @@ from ...hek.defs.cdmg import * -damage = Struct("damage", - SEnum16("priority", - "none", - "harmless", - {NAME: "backstab", GUI_NAME: "lethal to the unsuspecting"}, - "emp", - ), - SEnum16("category", *damage_category), - Bool32("flags", - "does_not_hurt_owner", - {NAME: "headshot", GUI_NAME: "causes headshots"}, - "pings_resistant_units", - "does_not_hurt_friends", - "does_not_ping_shields", - "detonates_explosives", - "only_hurts_shields", - "causes_flaming_death", - {NAME: "indicator_points_down", - GUI_NAME: "damage indicator always points down"}, - "skips_shields", - "only_hurts_one_infection_form", - {NAME: "multiplayer_headshot", - GUI_NAME: "causes multiplayer headshots"}, - "infection_form_pop", - "YELO_3D_instantaneous_acceleration" - ), - Pad(4), - Float("damage_lower_bound"), - QStruct("damage_upper_bound", INCLUDE=from_to), - float_zero_to_one("vehicle_passthrough_penalty"), - Pad(4), - float_zero_to_one("stun"), - float_zero_to_one("maximum_stun"), - float_sec("stun_time"), - Pad(4), - QStruct("instantaneous_acceleration", - Float("i", UNIT_SCALE=per_sec_unit_scale), - Float("j", UNIT_SCALE=per_sec_unit_scale), - Float("k", UNIT_SCALE=per_sec_unit_scale), - SIDETIP="[-inf,+inf]", ORIENT="h" - ), +damage_flags = Bool32("flags", + "does_not_hurt_owner", + {NAME: "headshot", GUI_NAME: "can cause headshots"}, + "pings_resistant_units", + "does_not_hurt_friends", + "does_not_ping_units", + "detonates_explosives", + "only_hurts_shields", + "causes_flaming_death", + {NAME: "indicator_points_down", GUI_NAME: "damage indicator always points down"}, + "skips_shields", + "only_hurts_one_infection_form", + {NAME: "multiplayer_headshot", GUI_NAME: "can cause multiplayer headshots"}, + "infection_form_pop", + "YELO_3D_instantaneous_acceleration" ) -cdmg_body = dict(cdmg_body) -cdmg_body[5] = damage +damage = desc_variant(damage, + damage_flags, + QStruct("instantaneous_acceleration", INCLUDE=ijk_float, SIDETIP="[-inf,+inf]"), + ("pad_13", Pad(0)), + # we're doing some weird stuff to make this work, so we're turning off verify + verify=False + ) + +cdmg_body = desc_variant(cdmg_body, damage) def get(): return cdmg_def diff --git a/reclaimer/os_v3_hek/defs/ctrl.py b/reclaimer/os_v3_hek/defs/ctrl.py index 2766cec6..ee9135ee 100644 --- a/reclaimer/os_v3_hek/defs/ctrl.py +++ b/reclaimer/os_v3_hek/defs/ctrl.py @@ -8,17 +8,10 @@ # from ...hek.defs.ctrl import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=8) - -ctrl_body = dict(ctrl_body) -ctrl_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ctrl") +ctrl_body = desc_variant(ctrl_body, obje_attrs) def get(): return ctrl_def diff --git a/reclaimer/os_v3_hek/defs/eqip.py b/reclaimer/os_v3_hek/defs/eqip.py index c89357d0..4c6f9537 100644 --- a/reclaimer/os_v3_hek/defs/eqip.py +++ b/reclaimer/os_v3_hek/defs/eqip.py @@ -8,17 +8,10 @@ # from ...os_hek.defs.eqip import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=3) - -eqip_body = dict(eqip_body) -eqip_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "eqip") +eqip_body = desc_variant(eqip_body, obje_attrs) def get(): return eqip_def diff --git a/reclaimer/os_v3_hek/defs/garb.py b/reclaimer/os_v3_hek/defs/garb.py index 6c6a1bd4..1958794c 100644 --- a/reclaimer/os_v3_hek/defs/garb.py +++ b/reclaimer/os_v3_hek/defs/garb.py @@ -8,17 +8,10 @@ # from ...hek.defs.garb import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=4) - -garb_body = dict(garb_body) -garb_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "garb") +garb_body = desc_variant(garb_body, obje_attrs) def get(): diff --git a/reclaimer/os_v3_hek/defs/jpt_.py b/reclaimer/os_v3_hek/defs/jpt_.py index 421774ed..d153188d 100644 --- a/reclaimer/os_v3_hek/defs/jpt_.py +++ b/reclaimer/os_v3_hek/defs/jpt_.py @@ -8,52 +8,17 @@ # from ...hek.defs.jpt_ import * +from .cdmg import damage_flags -damage = Struct("damage", - SEnum16("priority", - "none", - "harmless", - {NAME: "backstab", GUI_NAME: "lethal to the unsuspecting"}, - "emp", - ), - SEnum16("category", *damage_category), - Bool32("flags", - "does_not_hurt_owner", - {NAME: "headshot", GUI_NAME: "causes headshots"}, - "pings_resistant_units", - "does_not_hurt_friends", - "does_not_ping_units", - "detonates_explosives", - "only_hurts_shields", - "causes_flaming_death", - {NAME: "indicator_points_down", - GUI_NAME: "damage indicator always points down"}, - "skips_shields", - "only_hurts_one_infection_form", - {NAME: "multiplayer_headshot", - GUI_NAME: "causes multiplayer headshots"}, - "infection_form_pop", - "YELO_3D_instantaneous_acceleration" - ), - float_wu("aoe_core_radius"), - Float("damage_lower_bound"), - QStruct("damage_upper_bound", INCLUDE=from_to), - float_zero_to_one("vehicle_passthrough_penalty"), - float_zero_to_one("active_camouflage_damage"), - float_zero_to_one("stun"), - float_zero_to_one("maximum_stun"), - float_sec("stun_time"), - Pad(4), - QStruct("instantaneous_acceleration", - Float("i", UNIT_SCALE=per_sec_unit_scale), - Float("j", UNIT_SCALE=per_sec_unit_scale), - Float("k", UNIT_SCALE=per_sec_unit_scale), - SIDETIP="[-inf,+inf]", ORIENT="h" - ), +damage = desc_variant(damage, + damage_flags, + QStruct("instantaneous_acceleration", INCLUDE=ijk_float, SIDETIP="[-inf,+inf]"), + ("pad_13", Pad(0)), + # we're doing some weird stuff to make this work, so we're turning off verify + verify=False ) -jpt__body = dict(jpt__body) -jpt__body[16] = damage +jpt__body = desc_variant(jpt__body, damage) def get(): return jpt__def diff --git a/reclaimer/os_v3_hek/defs/lifi.py b/reclaimer/os_v3_hek/defs/lifi.py index 505b32d4..bcc0afdb 100644 --- a/reclaimer/os_v3_hek/defs/lifi.py +++ b/reclaimer/os_v3_hek/defs/lifi.py @@ -8,17 +8,10 @@ # from ...hek.defs.lifi import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=9) - -lifi_body = dict(lifi_body) -lifi_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "lifi") +lifi_body = desc_variant(lifi_body, obje_attrs) def get(): return lifi_def diff --git a/reclaimer/os_v3_hek/defs/mach.py b/reclaimer/os_v3_hek/defs/mach.py index 493331fc..73059163 100644 --- a/reclaimer/os_v3_hek/defs/mach.py +++ b/reclaimer/os_v3_hek/defs/mach.py @@ -8,17 +8,10 @@ # from ...hek.defs.mach import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=7) - -mach_body = dict(mach_body) -mach_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "mach") +mach_body = desc_variant(mach_body, obje_attrs) def get(): return mach_def diff --git a/reclaimer/os_v3_hek/defs/plac.py b/reclaimer/os_v3_hek/defs/plac.py index 655d8c49..cffcfd66 100644 --- a/reclaimer/os_v3_hek/defs/plac.py +++ b/reclaimer/os_v3_hek/defs/plac.py @@ -8,17 +8,10 @@ # from ...hek.defs.plac import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=10) - -plac_body = dict(plac_body) -plac_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "plac") +plac_body = desc_variant(plac_body, obje_attrs) def get(): return plac_def diff --git a/reclaimer/os_v3_hek/defs/proj.py b/reclaimer/os_v3_hek/defs/proj.py index 94303bfa..cce9ab18 100644 --- a/reclaimer/os_v3_hek/defs/proj.py +++ b/reclaimer/os_v3_hek/defs/proj.py @@ -8,17 +8,10 @@ # from ...os_hek.defs.proj import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=5) - -proj_body = dict(proj_body) -proj_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "proj") +proj_body = desc_variant(proj_body, obje_attrs) def get(): return proj_def diff --git a/reclaimer/os_v3_hek/defs/scen.py b/reclaimer/os_v3_hek/defs/scen.py index b56c75d3..2109dbf2 100644 --- a/reclaimer/os_v3_hek/defs/scen.py +++ b/reclaimer/os_v3_hek/defs/scen.py @@ -8,17 +8,10 @@ # from ...hek.defs.scen import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=6) - -scen_body = dict(scen_body) -scen_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "scen") +scen_body = desc_variant(scen_body, obje_attrs) def get(): return scen_def diff --git a/reclaimer/os_v3_hek/defs/ssce.py b/reclaimer/os_v3_hek/defs/ssce.py index abd90da8..d5926367 100644 --- a/reclaimer/os_v3_hek/defs/ssce.py +++ b/reclaimer/os_v3_hek/defs/ssce.py @@ -8,17 +8,10 @@ # from ...hek.defs.ssce import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=11) - -ssce_body = dict(ssce_body) -ssce_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ssce") +ssce_body = desc_variant(ssce_body, obje_attrs) def get(): return ssce_def diff --git a/reclaimer/os_v3_hek/defs/vehi.py b/reclaimer/os_v3_hek/defs/vehi.py index 7481a531..37873c5e 100644 --- a/reclaimer/os_v3_hek/defs/vehi.py +++ b/reclaimer/os_v3_hek/defs/vehi.py @@ -8,16 +8,10 @@ # from ...os_hek.defs.vehi import * - -#import and use the open saucified obje and unit attrs from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=1) - +obje_attrs = obje_attrs_variant(obje_attrs, "vehi") vehi_body = Struct("tagdata", obje_attrs, unit_attrs, @@ -25,7 +19,6 @@ SIZE=1008, ) - def get(): return vehi_def diff --git a/reclaimer/os_v3_hek/defs/weap.py b/reclaimer/os_v3_hek/defs/weap.py index d70ee3eb..b0910a97 100644 --- a/reclaimer/os_v3_hek/defs/weap.py +++ b/reclaimer/os_v3_hek/defs/weap.py @@ -8,17 +8,10 @@ # from ...os_hek.defs.weap import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=2) - -weap_body = dict(weap_body) -weap_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "weap") +weap_body = desc_variant(weap_body, obje_attrs) def get(): return weap_def diff --git a/reclaimer/os_v4_hek/defs/antr.py b/reclaimer/os_v4_hek/defs/antr.py index 811f9e16..444296ab 100644 --- a/reclaimer/os_v4_hek/defs/antr.py +++ b/reclaimer/os_v4_hek/defs/antr.py @@ -7,4 +7,27 @@ # See LICENSE for more information. # -from ...os_hek.defs.antr import * +from ...os_v3_hek.defs.antr import * + +unit_desc = desc_variant(unit_desc, + reflexive("animations", anim_enum_desc, + len(unit_animation_names_os), + *unit_animation_names_os + ) + ) + +antr_body = desc_variant(antr_body, + # this was further increased in os v4 + # https://github.com/HaloMods/OpenSauce/blob/master/OpenSauce/shared/Include/blamlib/Halo1/models/model_animation_definitions.hpp#L21 + reflexive("units", unit_desc, 512, DYN_NAME_PATH=".label"), + ) + +def get(): + return antr_def + +antr_def = TagDef("antr", + blam_header('antr', 4), + antr_body, + + ext=".model_animations", endian=">", tag_cls=AntrTag + ) \ No newline at end of file diff --git a/reclaimer/os_v4_hek/defs/bipd.py b/reclaimer/os_v4_hek/defs/bipd.py index 5dd4624e..5d3dc8c2 100644 --- a/reclaimer/os_v4_hek/defs/bipd.py +++ b/reclaimer/os_v4_hek/defs/bipd.py @@ -8,19 +8,11 @@ # from ...os_v3_hek.defs.bipd import * - -#import and use the open saucified obje and unit attrs from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=0) - -bipd_body = dict(bipd_body) -bipd_body[0] = obje_attrs -bipd_body[1] = unit_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "bipd") +bipd_body = desc_variant(bipd_body, obje_attrs, unit_attrs) def get(): return bipd_def diff --git a/reclaimer/os_v4_hek/defs/bitm.py b/reclaimer/os_v4_hek/defs/bitm.py index 860db917..2dce4e33 100644 --- a/reclaimer/os_v4_hek/defs/bitm.py +++ b/reclaimer/os_v4_hek/defs/bitm.py @@ -9,16 +9,14 @@ from ...os_v3_hek.defs.bitm import * -def get(): return bitm_def - -# replace the model animations dependency with an open sauce one -bitm_body = dict(bitm_body) -bitm_body[3] = Bool16("flags", - "enable_diffusion_dithering", - "disable_height_map_compression", - "uniform_sprite_sequences", - "sprite_bug_fix", - ("never_share_resources", 1<<13) +bitm_body = desc_variant(bitm_body, + Bool16("flags", + "enable_diffusion_dithering", + "disable_height_map_compression", + "uniform_sprite_sequences", + "sprite_bug_fix", + ("never_share_resources", 1<<13) + ), ) def get(): diff --git a/reclaimer/os_v4_hek/defs/cont.py b/reclaimer/os_v4_hek/defs/cont.py index 3cda3ff7..8626e058 100644 --- a/reclaimer/os_v4_hek/defs/cont.py +++ b/reclaimer/os_v4_hek/defs/cont.py @@ -9,11 +9,13 @@ from ...os_v3_hek.defs.cont import * -cont_body = dict(cont_body) -cont_body[4] = reflexive( - "shader_extensions", - Struct("shader_extension", INCLUDE=os_shader_extension), - 1) +shader_extensions = reflexive("shader_extensions", + Struct("shader_extension", INCLUDE=os_shader_extension), + 1 + ) +cont_body = desc_variant(cont_body, + ("pad_4", shader_extensions), + ) def get(): diff --git a/reclaimer/os_v4_hek/defs/ctrl.py b/reclaimer/os_v4_hek/defs/ctrl.py index f2cb6d01..ed2e2b95 100644 --- a/reclaimer/os_v4_hek/defs/ctrl.py +++ b/reclaimer/os_v4_hek/defs/ctrl.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.ctrl import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=8) - -ctrl_body = dict(ctrl_body) -ctrl_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ctrl") +ctrl_body = desc_variant(ctrl_body, obje_attrs) def get(): return ctrl_def diff --git a/reclaimer/os_v4_hek/defs/eqip.py b/reclaimer/os_v4_hek/defs/eqip.py index 82f5ba66..b7390e64 100644 --- a/reclaimer/os_v4_hek/defs/eqip.py +++ b/reclaimer/os_v4_hek/defs/eqip.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.eqip import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=3) - -eqip_body = dict(eqip_body) -eqip_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "eqip") +eqip_body = desc_variant(eqip_body, obje_attrs) def get(): return eqip_def diff --git a/reclaimer/os_v4_hek/defs/garb.py b/reclaimer/os_v4_hek/defs/garb.py index 49127703..14c0e6ca 100644 --- a/reclaimer/os_v4_hek/defs/garb.py +++ b/reclaimer/os_v4_hek/defs/garb.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.garb import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=4) - -garb_body = dict(garb_body) -garb_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "garb") +garb_body = desc_variant(garb_body, obje_attrs) def get(): return garb_def diff --git a/reclaimer/os_v4_hek/defs/gelo.py b/reclaimer/os_v4_hek/defs/gelo.py index e7072604..0066aa1f 100644 --- a/reclaimer/os_v4_hek/defs/gelo.py +++ b/reclaimer/os_v4_hek/defs/gelo.py @@ -7,32 +7,11 @@ # See LICENSE for more information. # -from .yelo import * +from ...os_v3_hek.defs.gelo import * -gelo_body = Struct("tagdata", - SInt16("version", DEFAULT=2), - Bool16("flags", - "hide_health_when_zoomed", - "hide_shield_when_zoomed", - "hide_motion_sensor_when_zoomed", - "force_game_to_use_stun_jumping_penalty" - ), - SInt32("base_address"), - ascii_str32("mod_name"), - dependency_os("global_explicit_references", "tagc"), - #dependency_os("chokin_victim_globals", "gelc"), - Pad(16), # removed_chokin_victim_globals - - Pad(16), - Pad(12), #reflexive("unknown1", void_desc), - Pad(52), - reflexive("scripted_ui_widgets", scripted_ui_widget, 128), - - Pad(12), #reflexive("unknown2", void_desc), - Pad(20), - reflexive("yelo_scripting", yelo_scripting, 1), - - SIZE=288 +gelo_body = desc_variant(gelo_body, + # was removed + ("chokin_victim_globals", Pad(16)), ) def get(): diff --git a/reclaimer/os_v4_hek/defs/lifi.py b/reclaimer/os_v4_hek/defs/lifi.py index 1ca371c9..a85fcb15 100644 --- a/reclaimer/os_v4_hek/defs/lifi.py +++ b/reclaimer/os_v4_hek/defs/lifi.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.lifi import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=9) - -lifi_body = dict(lifi_body) -lifi_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "lifi") +lifi_body = desc_variant(lifi_body, obje_attrs) def get(): return lifi_def diff --git a/reclaimer/os_v4_hek/defs/mach.py b/reclaimer/os_v4_hek/defs/mach.py index c6b78a65..2d25cbdb 100644 --- a/reclaimer/os_v4_hek/defs/mach.py +++ b/reclaimer/os_v4_hek/defs/mach.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.mach import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=7) - -mach_body = dict(mach_body) -mach_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "mach") +mach_body = desc_variant(mach_body, obje_attrs) def get(): return mach_def diff --git a/reclaimer/os_v4_hek/defs/obje.py b/reclaimer/os_v4_hek/defs/obje.py index 11beced7..7813138d 100644 --- a/reclaimer/os_v4_hek/defs/obje.py +++ b/reclaimer/os_v4_hek/defs/obje.py @@ -9,15 +9,14 @@ from ...os_v3_hek.defs.obje import * -obje_attrs = dict(obje_attrs) -obje_attrs[1] = Bool16('flags', - 'does_not_cast_shadow', - 'transparent_self_occlusion', - 'brighter_than_it_should_be', - 'not_a_pathfinding_obstacle', - 'cast_shadow_by_default', - {NAME: 'xbox_unknown_bit_8', VALUE: 1<<8, VISIBLE: False}, - {NAME: 'xbox_unknown_bit_11', VALUE: 1<<11, VISIBLE: False}, +obje_attrs = desc_variant(obje_attrs, + Bool16('flags', + 'does_not_cast_shadow', + 'transparent_self_occlusion', + 'brighter_than_it_should_be', + 'not_a_pathfinding_obstacle', + 'cast_shadow_by_default', + ), ) obje_body = Struct('tagdata', diff --git a/reclaimer/os_v4_hek/defs/part.py b/reclaimer/os_v4_hek/defs/part.py index 31543beb..130df894 100644 --- a/reclaimer/os_v4_hek/defs/part.py +++ b/reclaimer/os_v4_hek/defs/part.py @@ -7,13 +7,16 @@ # See LICENSE for more information. # -from ...hek.defs.part import * +from ...os_v3_hek.defs.part import * -part_body = dict(part_body) -part_body[11] = reflexive( - "particle_shader_extensions", +particle_shader_extensions = reflexive("particle_shader_extensions", Struct("particle_shader_extension", INCLUDE=os_shader_extension), - 1) + 1 + ) + +part_body = desc_variant(part_body, + ("pad_11", particle_shader_extensions) + ) def get(): @@ -23,5 +26,5 @@ def get(): blam_header("part", 2), part_body, - ext=".particle", endian=">", tag_cls=HekTag, + ext=".particle", endian=">", tag_cls=PartTag, ) diff --git a/reclaimer/os_v4_hek/defs/pctl.py b/reclaimer/os_v4_hek/defs/pctl.py index ab8f45c5..571260f4 100644 --- a/reclaimer/os_v4_hek/defs/pctl.py +++ b/reclaimer/os_v4_hek/defs/pctl.py @@ -7,21 +7,20 @@ # See LICENSE for more information. # -from ...hek.defs.pctl import * - -particle_state = dict(particle_state) -particle_type = dict(particle_type) -pctl_body = dict(pctl_body) - -particle_state[19] = reflexive( - "shader_extensions", - Struct("shader_extension", INCLUDE=os_shader_extension), - 1) -particle_type[12] = reflexive( - "particle_states", particle_state, 8, DYN_NAME_PATH='.name') -pctl_body[5] = reflexive( - "particle_types", particle_type, 4, DYN_NAME_PATH='.name') +from ...os_v3_hek.defs.pctl import * +shader_extensions = reflexive("shader_extensions", + Struct("shader_extension", INCLUDE=os_shader_extension), 1 + ) +particle_state = desc_variant(particle_state, + ("pad_19", shader_extensions), + ) +particle_type = desc_variant(particle_type, + reflexive("particle_states", particle_state, 8, DYN_NAME_PATH='.name'), + ) +pctl_body = desc_variant(pctl_body, + reflexive("particle_types", particle_type, 4, DYN_NAME_PATH='.name'), + ) def get(): return pctl_def diff --git a/reclaimer/os_v4_hek/defs/plac.py b/reclaimer/os_v4_hek/defs/plac.py index 3a1d3673..39033345 100644 --- a/reclaimer/os_v4_hek/defs/plac.py +++ b/reclaimer/os_v4_hek/defs/plac.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.plac import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=10) - -plac_body = dict(plac_body) -plac_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "plac") +plac_body = desc_variant(plac_body, obje_attrs) def get(): return plac_def diff --git a/reclaimer/os_v4_hek/defs/proj.py b/reclaimer/os_v4_hek/defs/proj.py index 5aa6faf0..c77c03f3 100644 --- a/reclaimer/os_v4_hek/defs/proj.py +++ b/reclaimer/os_v4_hek/defs/proj.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.proj import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=5) - -proj_body = dict(proj_body) -proj_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "proj") +proj_body = desc_variant(proj_body, obje_attrs) def get(): return proj_def diff --git a/reclaimer/os_v4_hek/defs/scen.py b/reclaimer/os_v4_hek/defs/scen.py index 0ab5bb32..f1b103b0 100644 --- a/reclaimer/os_v4_hek/defs/scen.py +++ b/reclaimer/os_v4_hek/defs/scen.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.scen import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=6) - -scen_body = dict(scen_body) -scen_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "scen") +scen_body = desc_variant(scen_body, obje_attrs) def get(): return scen_def diff --git a/reclaimer/os_v4_hek/defs/scnr.py b/reclaimer/os_v4_hek/defs/scnr.py index e8e2e8c5..b9e8c49a 100644 --- a/reclaimer/os_v4_hek/defs/scnr.py +++ b/reclaimer/os_v4_hek/defs/scnr.py @@ -19,7 +19,6 @@ SIZE=124 ) - sky_set_sky = Struct("sky", Pad(2), dyn_senum16("sky_index", @@ -28,14 +27,12 @@ SIZE=20 ) - sky_set = Struct("sky_set", ascii_str32("name"), reflexive("skies", sky_set_sky, 8), SIZE=44 ) - bsp_modifier = Struct("bsp_modifier", Pad(2), dyn_senum16("bsp_index", @@ -45,9 +42,9 @@ SIZE=64 ) - -scnr_body = dict(scnr_body) -scnr_body[64] = reflexive("bsp_modifiers", bsp_modifier, 32) +scnr_body = desc_variant(scnr_body, + ("pad_65", reflexive("bsp_modifiers", bsp_modifier, 32)), + ) def get(): return scnr_def @@ -56,5 +53,5 @@ def get(): blam_header('scnr', 2), scnr_body, - ext=".scenario", endian=">", tag_cls=HekTag + ext=".scenario", endian=">", tag_cls=ScnrTag ) diff --git a/reclaimer/os_v4_hek/defs/senv.py b/reclaimer/os_v4_hek/defs/senv.py index 32e638b2..0b00e968 100644 --- a/reclaimer/os_v4_hek/defs/senv.py +++ b/reclaimer/os_v4_hek/defs/senv.py @@ -7,7 +7,7 @@ # See LICENSE for more information. # -from ...os_hek.defs.senv import * +from ...os_v3_hek.defs.senv import * dlm_comment = """DIRECTIONAL LIGHTMAP PROPERTIES Special shader settings for when your map has directional lightmaps rendered for it.""" @@ -51,8 +51,9 @@ ) # replace the padding with an open sauce shader environment extension reflexive -senv_attrs = dict(senv_attrs) -senv_attrs[3] = reflexive("os_shader_environment_ext", os_senv_ext, 1) +senv_attrs = desc_variant(senv_attrs, + ("pad_3", reflexive("os_shader_environment_ext", os_senv_ext, 1)), + ) senv_body = Struct("tagdata", shdr_attrs, diff --git a/reclaimer/os_v4_hek/defs/ssce.py b/reclaimer/os_v4_hek/defs/ssce.py index bb7da50a..32129e80 100644 --- a/reclaimer/os_v4_hek/defs/ssce.py +++ b/reclaimer/os_v4_hek/defs/ssce.py @@ -8,17 +8,10 @@ # from ...os_v3_hek.defs.ssce import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=11) - -ssce_body = dict(ssce_body) -ssce_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ssce") +ssce_body = desc_variant(ssce_body, obje_attrs) def get(): return ssce_def diff --git a/reclaimer/os_v4_hek/defs/unit.py b/reclaimer/os_v4_hek/defs/unit.py index 063a4e02..e38c7479 100644 --- a/reclaimer/os_v4_hek/defs/unit.py +++ b/reclaimer/os_v4_hek/defs/unit.py @@ -220,10 +220,7 @@ SIZE=100 ) -seat = dict(seat) -unit_attrs = dict(unit_attrs) - -seat[0] = Bool32("flags", +seat_flags = Bool32("flags", "invisible", "locked", "driver", @@ -237,9 +234,46 @@ "allow_ai_noncombatants", ("allows_melee", 1<<20) ) -seat[20] = reflexive("seat_extensions", seat_extension, 1) -unit_attrs[45] = reflexive("unit_extensions", unit_extension, 1) -unit_attrs[54] = reflexive("seats", seat, 16, DYN_NAME_PATH='.label') + +seat = desc_variant(seat, + seat_flags, + ("pad_20", reflexive("seat_extensions", seat_extension, 1)), + ) + +unit_flags = Bool32("flags", + "circular_aiming", + "destroyed_after_dying", + "half_speed_interpolation", + "fires_from_camera", + "entrance_inside_bounding_sphere", + "unused", + "causes_passenger_dialogue", + "resists_pings", + "melee_attack_is_fatal", + "dont_reface_during_pings", + "has_no_aiming", + "simple_creature", + "impact_melee_attaches_to_unit", + "impact_melee_dies_on_shields", + "cannot_open_doors_automatically", + "melee_attackers_cannot_attach", + "not_instantly_killed_by_melee", + "shield_sapping", + "runs_around_flaming", + "inconsequential", + "special_cinematic_unit", + "ignored_by_autoaiming", + "shields_fry_infection_forms", + "integrated_light_controls_weapon", + "integrated_light_lasts_forever", + ("has_boarding_seats", 1<<31) + ) + +unit_attrs = desc_variant(unit_attrs, + unit_flags, + ("pad_45", reflexive("unit_extensions", unit_extension, 1)), + reflexive("seats", seat, 16, DYN_NAME_PATH='.label'), + ) unit_body = Struct('tagdata', unit_attrs) diff --git a/reclaimer/os_v4_hek/defs/vehi.py b/reclaimer/os_v4_hek/defs/vehi.py index 265ab4e3..9ac5cc60 100644 --- a/reclaimer/os_v4_hek/defs/vehi.py +++ b/reclaimer/os_v4_hek/defs/vehi.py @@ -8,19 +8,11 @@ # from ...os_v3_hek.defs.vehi import * - -#import and use the open saucified obje and unit attrs from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=1) - -vehi_body = dict(vehi_body) -vehi_body[0] = obje_attrs -vehi_body[1] = unit_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "vehi") +vehi_body = desc_variant(vehi_body, obje_attrs, unit_attrs) def get(): return vehi_def diff --git a/reclaimer/os_v4_hek/defs/weap.py b/reclaimer/os_v4_hek/defs/weap.py index 95b114a5..ee92f91d 100644 --- a/reclaimer/os_v4_hek/defs/weap.py +++ b/reclaimer/os_v4_hek/defs/weap.py @@ -11,13 +11,21 @@ from .obje import * from .item import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=2) +magazine = desc_variant(magazine, + reflexive("magazine_items", magazine_item, 8, + "primary", + "secondary_primary_2", + # so uh, spv3 does a thing with extra ammo pickups that + # act as alternate ammo counts for the primary magazine + *("primary_%s" % (3 + i) for i in range(6)) + ), + ) +weap_attrs = desc_variant(weap_attrs, + reflexive("magazines", magazine, 2, "primary", "secondary") + ) -weap_body = dict(weap_body) -weap_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "weap") +weap_body = desc_variant(weap_body, obje_attrs, weap_attrs) def get(): return weap_def diff --git a/reclaimer/shadowrun_prototype/common_descs.py b/reclaimer/shadowrun_prototype/common_descs.py index c7b359b5..f8a97ea8 100644 --- a/reclaimer/shadowrun_prototype/common_descs.py +++ b/reclaimer/shadowrun_prototype/common_descs.py @@ -10,6 +10,19 @@ from reclaimer.common_descs import * from reclaimer.shadowrun_prototype.constants import * +# TODO: move shared enumerators into separate enums.py module +# ########################################################################### +# The order of element in all the enumerators is important(DONT SHUFFLE THEM) +# ########################################################################### + +#Shared Enumerator options + +# TODO: update these if any of the new shadowrun tag types are found +# to be used in scripts, or if there are new builtin functions +# NOTE: we're re-defining these here simply as a placeholder +script_types = tuple(script_types) +script_object_types = tuple(script_object_types) + def sr_tag_class(*args, **kwargs): ''' @@ -19,7 +32,6 @@ def sr_tag_class(*args, **kwargs): kwargs["class_mapping"] = sr_tag_class_fcc_to_ext return tag_class(*args, **kwargs) - def dependency(name='tag_ref', valid_ids=None, **kwargs): '''This function serves to macro the creation of a tag dependency''' if isinstance(valid_ids, tuple): @@ -27,16 +39,20 @@ def dependency(name='tag_ref', valid_ids=None, **kwargs): elif isinstance(valid_ids, str): valid_ids = sr_tag_class(valid_ids) elif valid_ids is None: - valid_ids = valid_tags + valid_ids = sr_valid_tags - return TagRef(name, + return desc_variant(tag_ref_struct, valid_ids, - INCLUDE=tag_ref_struct, STEPTREE=StrTagRef( - "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=234), - **kwargs + "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=254 + ), + NAME=name, **kwargs ) +# should really rename "dependency" above to this. +# until then, make an alias so it's clear what we're referencing +dependency_sr = dependency + def blam_header(tagid, version=1): '''This function serves to macro the creation of a tag header''' diff --git a/reclaimer/shadowrun_prototype/defs/bipd.py b/reclaimer/shadowrun_prototype/defs/bipd.py index c8c876e9..7ca9ff98 100644 --- a/reclaimer/shadowrun_prototype/defs/bipd.py +++ b/reclaimer/shadowrun_prototype/defs/bipd.py @@ -11,12 +11,8 @@ from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=0) - -bipd_body = Struct("tagdata", +obje_attrs = obje_attrs_variant(obje_attrs, "bipd") +bipd_body = Struct("tagdata", obje_attrs, unit_attrs, bipd_attrs, diff --git a/reclaimer/shadowrun_prototype/defs/ctrl.py b/reclaimer/shadowrun_prototype/defs/ctrl.py index 3ff1a7de..dbbd4cca 100644 --- a/reclaimer/shadowrun_prototype/defs/ctrl.py +++ b/reclaimer/shadowrun_prototype/defs/ctrl.py @@ -10,13 +10,8 @@ from ...hek.defs.ctrl import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=8) - -ctrl_body = dict(ctrl_body) -ctrl_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ctrl") +ctrl_body = desc_variant(ctrl_body, obje_attrs) def get(): return ctrl_def diff --git a/reclaimer/shadowrun_prototype/defs/eqip.py b/reclaimer/shadowrun_prototype/defs/eqip.py index 1513134d..719b7068 100644 --- a/reclaimer/shadowrun_prototype/defs/eqip.py +++ b/reclaimer/shadowrun_prototype/defs/eqip.py @@ -10,13 +10,8 @@ from ...hek.defs.eqip import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=3) - -eqip_body = dict(eqip_body) -eqip_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "eqip") +eqip_body = desc_variant(eqip_body, obje_attrs) def get(): return eqip_def diff --git a/reclaimer/shadowrun_prototype/defs/garb.py b/reclaimer/shadowrun_prototype/defs/garb.py index 18fb1f59..0297e5f3 100644 --- a/reclaimer/shadowrun_prototype/defs/garb.py +++ b/reclaimer/shadowrun_prototype/defs/garb.py @@ -10,14 +10,8 @@ from ...hek.defs.garb import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=4) - -garb_body = dict(garb_body) -garb_body[0] = obje_attrs - +obje_attrs = obje_attrs_variant(obje_attrs, "garb") +garb_body = desc_variant(garb_body, obje_attrs) def get(): return garb_def diff --git a/reclaimer/shadowrun_prototype/defs/lifi.py b/reclaimer/shadowrun_prototype/defs/lifi.py index 6f110a61..bcc0afdb 100644 --- a/reclaimer/shadowrun_prototype/defs/lifi.py +++ b/reclaimer/shadowrun_prototype/defs/lifi.py @@ -10,13 +10,8 @@ from ...hek.defs.lifi import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=9) - -lifi_body = dict(lifi_body) -lifi_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "lifi") +lifi_body = desc_variant(lifi_body, obje_attrs) def get(): return lifi_def diff --git a/reclaimer/shadowrun_prototype/defs/mach.py b/reclaimer/shadowrun_prototype/defs/mach.py index c2381757..73059163 100644 --- a/reclaimer/shadowrun_prototype/defs/mach.py +++ b/reclaimer/shadowrun_prototype/defs/mach.py @@ -10,13 +10,8 @@ from ...hek.defs.mach import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=7) - -mach_body = dict(mach_body) -mach_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "mach") +mach_body = desc_variant(mach_body, obje_attrs) def get(): return mach_def diff --git a/reclaimer/shadowrun_prototype/defs/matg.py b/reclaimer/shadowrun_prototype/defs/matg.py index 780084e6..2c383d66 100644 --- a/reclaimer/shadowrun_prototype/defs/matg.py +++ b/reclaimer/shadowrun_prototype/defs/matg.py @@ -8,3 +8,18 @@ # from ...hek.defs.matg import * + +def get(): + return matg_def + +# shadowrun has more grenade types +matg_body = desc_variant(matg_body, + reflexive("grenades", grenade, 5) + ) + +matg_def = TagDef("matg", + blam_header_os('matg', 3), + matg_body, + + ext=".globals", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/shadowrun_prototype/defs/objs/scnr.py b/reclaimer/shadowrun_prototype/defs/objs/scnr.py new file mode 100644 index 00000000..22ad42cf --- /dev/null +++ b/reclaimer/shadowrun_prototype/defs/objs/scnr.py @@ -0,0 +1,13 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.hek.defs.objs.scnr import ScnrTag + +class SrProtoScnrTag(ScnrTag): + engine = "shadowrun_proto" diff --git a/reclaimer/shadowrun_prototype/defs/plac.py b/reclaimer/shadowrun_prototype/defs/plac.py index e71ceeef..cffcfd66 100644 --- a/reclaimer/shadowrun_prototype/defs/plac.py +++ b/reclaimer/shadowrun_prototype/defs/plac.py @@ -10,13 +10,8 @@ from ...hek.defs.plac import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=10) - -plac_body = dict(plac_body) -plac_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "plac") +plac_body = desc_variant(plac_body, obje_attrs) def get(): return plac_def diff --git a/reclaimer/shadowrun_prototype/defs/proj.py b/reclaimer/shadowrun_prototype/defs/proj.py index ad6f38db..062b16f6 100644 --- a/reclaimer/shadowrun_prototype/defs/proj.py +++ b/reclaimer/shadowrun_prototype/defs/proj.py @@ -10,13 +10,8 @@ from ...hek.defs.proj import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=5) - -proj_body = dict(proj_body) -proj_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "proj") +proj_body = desc_variant(proj_body, obje_attrs) def get(): return proj_def diff --git a/reclaimer/shadowrun_prototype/defs/scen.py b/reclaimer/shadowrun_prototype/defs/scen.py index 020bf262..2109dbf2 100644 --- a/reclaimer/shadowrun_prototype/defs/scen.py +++ b/reclaimer/shadowrun_prototype/defs/scen.py @@ -10,13 +10,8 @@ from ...hek.defs.scen import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=6) - -scen_body = dict(scen_body) -scen_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "scen") +scen_body = desc_variant(scen_body, obje_attrs) def get(): return scen_def diff --git a/reclaimer/shadowrun_prototype/defs/scnr.py b/reclaimer/shadowrun_prototype/defs/scnr.py index 5ccac64f..e24147f1 100644 --- a/reclaimer/shadowrun_prototype/defs/scnr.py +++ b/reclaimer/shadowrun_prototype/defs/scnr.py @@ -8,3 +8,22 @@ # from ...hek.defs.scnr import * +from ..common_descs import * +from .objs.scnr import SrProtoScnrTag + +reference = desc_variant(reference, dependency_sr("reference")) + +scnr_body = desc_variant(scnr_body, + reflexive("references", reference, 256, DYN_NAME_PATH='.reference.filepath'), + ) + +def get(): + return scnr_def + +# TODO: update dependencies +scnr_def = TagDef("scnr", + blam_header('scnr', 2), + scnr_body, + + ext=".scenario", endian=">", tag_cls=SrProtoScnrTag + ) diff --git a/reclaimer/shadowrun_prototype/defs/ssce.py b/reclaimer/shadowrun_prototype/defs/ssce.py index 001a3e4b..d5926367 100644 --- a/reclaimer/shadowrun_prototype/defs/ssce.py +++ b/reclaimer/shadowrun_prototype/defs/ssce.py @@ -10,13 +10,8 @@ from ...hek.defs.ssce import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=11) - -ssce_body = dict(ssce_body) -ssce_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ssce") +ssce_body = desc_variant(ssce_body, obje_attrs) def get(): return ssce_def diff --git a/reclaimer/shadowrun_prototype/defs/tagc.py b/reclaimer/shadowrun_prototype/defs/tagc.py index 6e3f05af..2fa3dac5 100644 --- a/reclaimer/shadowrun_prototype/defs/tagc.py +++ b/reclaimer/shadowrun_prototype/defs/tagc.py @@ -8,3 +8,23 @@ # from ...hek.defs.tagc import * +from ..common_descs import dependency_sr + +tag_reference = desc_variant(tag_reference, dependency_sr("tag")) + +tagc_body = desc_variant(tagc_body, + reflexive("tag_references", tag_reference, 200, + DYN_NAME_PATH='.tag.filepath' + ) + ) + + +def get(): + return tagc_def + +tagc_def = TagDef("tagc", + blam_header('tagc'), + tagc_body, + + ext=".tag_collection", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/shadowrun_prototype/defs/weap.py b/reclaimer/shadowrun_prototype/defs/weap.py index a98e2e94..a11a3a15 100644 --- a/reclaimer/shadowrun_prototype/defs/weap.py +++ b/reclaimer/shadowrun_prototype/defs/weap.py @@ -10,13 +10,8 @@ from ...hek.defs.weap import * from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=2) - -weap_body = dict(weap_body) -weap_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "weap") +weap_body = desc_variant(weap_body, obje_attrs) def get(): return weap_def diff --git a/reclaimer/sounds/adpcm.py b/reclaimer/sounds/adpcm.py index ec11e808..0303367e 100644 --- a/reclaimer/sounds/adpcm.py +++ b/reclaimer/sounds/adpcm.py @@ -15,13 +15,13 @@ # IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import array -import audioop import sys + from struct import unpack_from from types import MethodType -from . import constants +from . import constants, util, audioop try: from .ext import adpcm_ext @@ -48,7 +48,6 @@ def _slow_decode_xbadpcm_samples(in_data, out_data, channel_ct): pcm_blocksize = channel_ct * constants.XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE // 2 adpcm_blocksize = channel_ct * constants.XBOX_ADPCM_COMPRESSED_BLOCKSIZE - adpcm2lin = audioop.adpcm2lin all_codes = memoryview(in_data) state_unpacker = MethodType(unpack_from, "<" + "hBx" * channel_ct) @@ -73,7 +72,7 @@ def _slow_decode_xbadpcm_samples(in_data, out_data, channel_ct): for j in range(c * 4, len(swapped_codes), code_block_size) ) decoded_samples = memoryview( - adpcm2lin(swapped_codes, 2, all_states[c])[0]).cast("h") + audioop.adpcm2lin(swapped_codes, 2, all_states[c])[0]).cast("h") # interleave the samples for each channel out_data[k + c] = all_states[c][0] @@ -104,7 +103,8 @@ def decode_adpcm_samples(in_data, channel_ct, output_big_endian=False): def encode_adpcm_samples(in_data, channel_ct, input_big_endian=False, - noise_shaping=NOISE_SHAPING_OFF, lookahead=3): + noise_shaping=NOISE_SHAPING_OFF, lookahead=3, + fit_to_blocksize=True): assert noise_shaping in (NOISE_SHAPING_OFF, NOISE_SHAPING_STATIC, NOISE_SHAPING_DYNAMIC) assert lookahead in range(6) @@ -115,16 +115,19 @@ def encode_adpcm_samples(in_data, channel_ct, input_big_endian=False, "Accelerator module not detected. Cannot compress to ADPCM.") if (sys.byteorder == "big") != input_big_endian: - out_data = audioop.byteswap(out_data, 2) + in_data = audioop.byteswap(in_data, 2) adpcm_blocksize = constants.XBOX_ADPCM_COMPRESSED_BLOCKSIZE * channel_ct - pcm_blocksize = constants.XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE * channel_ct - - pad_size = len(in_data) % pcm_blocksize - if pad_size: - pad_size = pcm_blocksize - pad_size + pcm_blocksize = constants.XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE * channel_ct + pcm_block_count = len(in_data) // pcm_blocksize + + pad_size = (pcm_blocksize - (len(in_data) % pcm_blocksize)) % pcm_blocksize + if fit_to_blocksize and pad_size: + # truncate to fit to blocksize + in_data = in_data[:pcm_block_count * pcm_blocksize] + elif pad_size: # repeat the last sample to the end to pad to a multiple of blocksize - pad_piece_size = (channel_ct * 2) + pad_piece_size = channel_ct * 2 in_data += in_data[-pad_piece_size: ] * (pad_size // pad_piece_size) out_data = bytearray( @@ -134,4 +137,4 @@ def encode_adpcm_samples(in_data, channel_ct, input_big_endian=False, adpcm_ext.encode_xbadpcm_samples( in_data, out_data, channel_ct, noise_shaping, lookahead) - return bytes(out_data) + return bytes(out_data) \ No newline at end of file diff --git a/reclaimer/sounds/audioop.py b/reclaimer/sounds/audioop.py new file mode 100644 index 00000000..9fbb2798 --- /dev/null +++ b/reclaimer/sounds/audioop.py @@ -0,0 +1,318 @@ +''' +This module provides a pure-python replacement for MOST of the audioop functions +that Reclaimer relies on for audio conversion. This module exists because audioop +is being deprecated in python 3.11, and will be removed in 3.13. Some of this +functionality isn't necessary for audio compiling, and those methods that arent +have not been reimplemented. + +A better solution would be to use a pypi library replacement, however the +pure-python versions out there are extremely slow. Example: + https://github.com/jiaaro/pydub/blob/master/pydub/pyaudioop.py + +To learn about the deprecation, see here: + https://peps.python.org/pep-0594/#audioop +''' +from types import MethodType +import array +import itertools + +try: + import audioop +except ModuleNotFoundError: + # THEY TOOK MY FUCKING BATTERIES!!!!! + audioop = None + +SAMPLE_WIDTH_BOUNDS = ( + ( -0x80, 0x7F), + ( -0x8000, 0x7Fff), + ( -0x800000, 0x7FffFF), + (-0x80000000, 0x7FffFFff), + ) + +SAMPLE_TYPECODES = ( + "b", + "h", + "t", # NOTE: not a real typecode + "i" + ) + +def adpcm2lin(fragment, width, state): + # TODO: replace with python fallback + raise NotImplementedError( + "No accelerators found, and audioop is removed in this " + "version of Python. Cannot decode ADPCM audio samples." + ) + + +def bias(fragment, width, bias): + if width not in (1, 2, 4): + raise NotImplementedError("Cannot bias %s-byte width samples." % width) + + # ensure fragment is the correct data type + if width > 1 and not(isinstance(fragment, array.array) and + fragment.itemsize == 1): + fragment = array.array("I" if width == 4 else "H", fragment) + + modulous = 1 << (8*width) + mapper = map( + modulous.__rmod__, + map(int(bias).__add__, fragment) + ) + + if width == 1: + fragment = bytes(mapper) + elif width in (2, 4): + fragment = array.array("I" if width == 4 else "H", mapper).tobytes() + + return fragment + + +def byteswap(fragment, width): + if width not in (1, 2, 3, 4): + raise NotImplementedError( + "Cannot byteswap %s-byte width samples." % width + ) + + orig_fragment = ( + fragment if isinstance(fragment, (bytes, bytearray)) else + fragment.tobytes() if isinstance(fragment, array.array) else + bytes(fragment) + ) + if width == 3: + # for 24bit we gotta do some clever shit to do this reasonable fast. + # sample data must be bytes or bytearray if we want to use slices + fragment = bytearray(orig_fragment) + # use slices to move byte 0 of each original word into byte 2 + # of the byteswapped word, and vice versa for the other bytes + fragment[::3] = orig_fragment[2::3] + fragment[2::3] = orig_fragment[::3] + fragment = bytes(fragment) + elif width == 2 or width == 4: + # we can use array.array to byteswap 16/32 bit pcm + fragment = array.array("i" if width == 4 else "h", fragment) + fragment.byteswap() + fragment = fragment.tobytes() + else: + fragment = orig_fragment + + return fragment + + +def lin2lin(fragment, width, newwidth): + if width not in (1, 2, 4): + raise NotImplementedError("Cannot convert from %s-byte width samples." % width) + elif newwidth not in (1, 2, 4): + raise NotImplementedError("Cannot convert to %s-byte width samples." % newwidth) + + typecode = SAMPLE_TYPECODES[width-1] + new_typecode = SAMPLE_TYPECODES[newwidth-1] + if not(isinstance(fragment, array.array) and + fragment.typecode == typecode): + fragment = array.array(typecode, fragment) + + if width == newwidth: + # same width? why? whatever.... + new_fragment = fragment + else: + shift_diff = abs(width - newwidth) * 8 + shift_func = ( + shift_diff.__rrshift__ # shift samples right(divide) + if width > newwidth else + shift_diff.__rlshift__ # shift samples left(multiply) + ) + + min_val_clip = MethodType(max, SAMPLE_WIDTH_BOUNDS[newwidth-1][0]) + max_val_clip = MethodType(min, SAMPLE_WIDTH_BOUNDS[newwidth-1][1]) + new_fragment = array.array(new_typecode, + map(max_val_clip, + map(min_val_clip, + map(round, + map(shift_func, fragment + ))))) + + return new_fragment.tobytes() + + +def ratecv(fragment, width, nchannels, inrate, outrate, state, weightA=1, weightB=0): + # TODO: replace with python fallback + raise NotImplementedError( + "audioop is removed in this version of Python. " + "Cannot convert sample rate to target." + ) + + +def tomono(fragment, width, lfactor, rfactor): + if width not in (1, 2, 4): + raise NotImplementedError( + "Cannot convert %s-byte width samples to mono." % width + ) + + typecode = SAMPLE_TYPECODES[width-1] + if not(isinstance(fragment, array.array) and + fragment.typecode == typecode): + fragment = array.array(typecode, fragment) + + min_val_clip = MethodType(max, SAMPLE_WIDTH_BOUNDS[width-1][0]) + max_val_clip = MethodType(min, SAMPLE_WIDTH_BOUNDS[width-1][1]) + + # these are some pretty simple map chains that just grab odd/even + # audio channel values and multiply them by their channel factor + left_channel_data = map(lfactor.__mul__, fragment[0::2]) + right_channel_data = map(rfactor.__mul__, fragment[1::2]) + + # WARNING: MAP FROM HELL + # now that we have our audio channels separated into maps, + # we can zip them together, pass that zip into a mapped sum + # to add the left/right channels, pass the result into a + # min/max clip map, and finally fast everything to integers. + new_fragment = array.array(typecode, + map(max_val_clip, + map(min_val_clip, + map(round, + map(sum, + zip(left_channel_data, right_channel_data) + ))))) + + return new_fragment.tobytes() + + +def tostereo(fragment, width, lfactor, rfactor): + if width not in (1, 2, 4): + raise NotImplementedError( + "Cannot convert %s-byte width samples to stereo." % width + ) + + typecode = SAMPLE_TYPECODES[width-1] + if not(isinstance(fragment, array.array) and + fragment.typecode == typecode): + fragment = array.array(typecode, fragment) + + min_val_clip = MethodType(max, SAMPLE_WIDTH_BOUNDS[width-1][0]) + max_val_clip = MethodType(min, SAMPLE_WIDTH_BOUNDS[width-1][1]) + + # WARNING: MAPS FROM HELL + # essentially we're doing the opposite of the tomono maps above. + # we're multiplying the input samples by the left/right factors, + # rounding to integers, and then clipping to out min/max values. + interleaved_fragment = array.array( + typecode, b'\x00'*(len(fragment)*width*2) + ) + interleaved_fragment[0::2] = fragment + interleaved_fragment[1::2] = fragment + + new_fragment = array.array(typecode, + map(max_val_clip, + map(min_val_clip, + map(round, + itertools.starmap(float.__mul__, + zip( + itertools.cycle((lfactor, rfactor)), + interleaved_fragment, + )))))) + return new_fragment.tobytes() + + +# TESTS +# these tests show that behavior is unchanged between the audioop +# implementation and the pure python version we've developed +def _run_tests(): + + for test_vals in [ + # NOTE: we use steps here to ensure we generate enough data + # for the test, but not TOO MUCH data(i.e. multiple MB) + ( -0x80, 0x7F, 0x01, 1, 2, "b", 0x80), + ( -0x80, 0x7F, 0x01, 1, 4, "b", 0x80), + ( 0x00, 0xFF, 0x01, 1, 2, "B", 0x80), + ( 0x00, 0xFF, 0x01, 1, 4, "B", 0x80), + ( -0x8000, 0x7Fff, 0x01, 2, 1, "h", 0x8000), + ( -0x8000, 0x7Fff, 0x01, 2, 4, "h", 0x8000), + ( 0x00, 0xFFff, 0x01, 2, 1, "H", 0x8000), + ( 0x00, 0xFFff, 0x01, 2, 4, "H", 0x8000), + ( -0x800000, 0x7FffFF, 0x80, 3, 1, "i", 0x800000), + ( -0x800000, 0x7FffFF, 0x80, 3, 4, "i", 0x800000), + ( 0x00, 0xFFffFF, 0x80, 3, 1, "I", 0x800000), + ( 0x00, 0xFFffFF, 0x80, 3, 4, "I", 0x800000), + (-0x80000000, 0x7FffFFff, 0x8000, 4, 1, "i", 0x7FffFFff), + (-0x80000000, 0x7FffFFff, 0x8000, 4, 2, "i", 0x7FffFFff), + ( 0x00, 0xFFffFFff, 0x8000, 4, 1, "I", 0x7FffFFff), + ( 0x00, 0xFFffFFff, 0x8000, 4, 2, "I", 0x7FffFFff), + #(-0x80000000, 0x7FffFFff, 0x100, 4, 1, "i", 0x7FffFFff), + #(-0x80000000, 0x7FffFFff, 0x100, 4, 2, "i", 0x7FffFFff), + #( 0x00, 0xFFffFFff, 0x100, 4, 1, "I", 0x7FffFFff), + #( 0x00, 0xFFffFFff, 0x100, 4, 2, "I", 0x7FffFFff), + ]: + get_delta = lambda: list( + itertools.starmap(int.__sub__, + zip( + array.array(typecode, audioop_data), + array.array(typecode, reclaimer_data) + )) + ) + + min_val, max_val, step_val, width, new_width, typecode, bias_val = test_vals + test_vals_str = ", ".join(str(v) for v in test_vals) + + test_data = array.array(typecode, range(min_val, max_val+1, step_val)) + # ensure test data is multiple of sample width + # also ensure audio data is a multiple of 2 for stereo/mono tests + test_data = test_data.tobytes()[:width*2*(len(test_data)//(width*2))] + + # test 1: audioop.byteswap vs byteswap + audioop_data = audioop.byteswap(test_data, width) + reclaimer_data = byteswap(test_data, width) + + if width == 3: + # NOTE: can't do much ATM with 24bit samples cause of how awkward they are + continue + + delta = get_delta() + if max(map(abs, delta)) > 1: + print(delta) + print("Test failure: Inconsistency in byteswap(%s)" % test_vals_str) + + # test 2: audioop.bias vs bias + audioop_data = audioop.bias(test_data, width, bias_val) + reclaimer_data = bias(test_data, width, bias_val) + delta = get_delta() + if max(map(abs, delta)) > 1: + print(delta) + print("Test failure: Inconsistency in bias(%s)" % test_vals_str) + + + # test 3: audioop.tomono vs tomono + audioop_data = audioop.tomono(test_data, width, 0.5, 0.5) + reclaimer_data = tomono(test_data, width, 0.5, 0.5) + delta = get_delta() + if max(map(abs, delta)) > 1: + print(delta) + print("Test failure: Inconsistency in tomono(%s)" % test_vals_str) + + + # test 4: audioop.tostereo vs tostereo + audioop_data = audioop.tostereo(test_data, width, 1.0, 1.0) + reclaimer_data = tostereo(test_data, width, 1.0, 1.0) + delta = get_delta() + if max(map(abs, delta)) > 1: + print(delta) + print("Test failure: Inconsistency in tostereo(%s)" % test_vals_str) + input() + + # test 5: audioop.lin2lin vs lin2lin + audioop_data = audioop.lin2lin(test_data, width, new_width) + reclaimer_data = lin2lin(test_data, width, new_width) + delta = get_delta() + if max(map(abs, delta)) > 1: + print(delta) + print("Test failure: Inconsistency in lin2lin(%s)" % test_vals_str) + input() + + +if __name__ == "__main__" and audioop is not None: + _run_tests() + #import cProfile + #cProfile.runctx("_run_tests()", locals(), globals()) + + +if audioop is not None: + # if audioop is loaded, use its methods + from audioop import * diff --git a/reclaimer/sounds/blam_sound_bank.py b/reclaimer/sounds/blam_sound_bank.py index 49034382..9f67b7e2 100644 --- a/reclaimer/sounds/blam_sound_bank.py +++ b/reclaimer/sounds/blam_sound_bank.py @@ -13,7 +13,7 @@ from traceback import format_exc from reclaimer.sounds.blam_sound_permutation import BlamSoundPermutation -from reclaimer.sounds import constants, ogg, adpcm +from reclaimer.sounds import constants, adpcm class BlamSoundPitchRange: @@ -55,11 +55,13 @@ def create_from_directory(directory): return new_pitch_range def export_to_directory(self, directory, overwrite=False, - export_source=True, decompress=True): + export_source=True, decompress=True, + ext=constants.CONTAINER_EXT_WAV, **kwargs): for name, perm in self.permutations.items(): perm.export_to_file( Path(directory, name), overwrite, - export_source, decompress) + export_source, decompress, ext, **kwargs + ) def import_from_directory(self, directory, clear_existing=True, replace_existing=True): @@ -70,13 +72,15 @@ def import_from_directory(self, directory, clear_existing=True, # import each sound file to a new sound permutation for filename in files: filepath = Path(root, filename) - if filepath.suffix.lower() != ".wav": - # only import wav files + if filepath.suffix.lower() not in constants.SUPPORTED_IMPORT_EXTS: continue - name_key = filepath.stem.lower().strip() perm = BlamSoundPermutation.create_from_file(filepath) + if perm is None: + continue + perm.name = filepath.stem.strip() + name_key = perm.name.lower() if (perm and perm.source_sample_data) and ( replace_existing or name_key not in self.permutations): @@ -97,21 +101,54 @@ class BlamSoundBank: # this value is fine to bump under most circumstances. chunk_size = constants.DEF_SAMPLE_CHUNK_SIZE - vorbis_bitrate_info = None - + # if True, truncates samples to fit to a multiple of 130. + # otherwise pads the block up with the last sample in it. + adpcm_fit_to_blocksize = True adpcm_noise_shaping = adpcm.NOISE_SHAPING_OFF adpcm_lookahead = 3 + # Combinations of nominal/lower/upper carry these implications: + # all three set to the same value: + # implies a fixed rate bitstream + # only nominal set: + # implies a VBR stream that has this nominal bitrate. + # No hard upper/lower limit + # upper and or lower set: + # implies a VBR bitstream that obeys the bitrate limits. + # nominal may also be set to give a nominal rate. + # none set: + # the coder does not care to speculate. + ogg_bitrate_lower = -1 + ogg_bitrate_nominal = -1 + ogg_bitrate_upper = -1 + ogg_quality_setting = 0 + ogg_use_quality_value = False + _pitch_ranges = () def __init__(self): self._pitch_ranges = {} - self.vorbis_bitrate_info = ogg.VorbisBitrateInfo() @property def pitch_ranges(self): return self._pitch_ranges + @property + def adpcm_kwargs(self): return dict( + noise_shaping = self.adpcm_noise_shaping, + lookahead = self.adpcm_lookahead, + fit_to_blocksize = self.adpcm_fit_to_blocksize, + ) + + @property + def ogg_kwargs(self): return dict( + bitrate_lower = self.ogg_bitrate_lower, + bitrate_upper = self.ogg_bitrate_upper, + bitrate_nominal = self.ogg_bitrate_nominal, + quality_setting = self.ogg_quality_setting, + use_quality_value = self.ogg_use_quality_value, + ) + def generate_mouth_data(self): for pitch_range in self.pitch_ranges.values(): pitch_range.generate_mouth_data() @@ -121,18 +158,10 @@ def compress_samples(self): if self.split_into_smaller_chunks: chunk_size = self.chunk_size - adpcm_kwargs = dict( - noise_shaping=self.adpcm_noise_shaping, - lookahead=self.adpcm_lookahead - ) - ogg_kwargs = dict( - bitrate_info=self.vorbis_bitrate_info - ) - for pitch_range in self.pitch_ranges.values(): pitch_range.compress_samples( - self.compression, self.sample_rate, self.encoding, - chunk_size, adpcm_kwargs=adpcm_kwargs, ogg_kwargs=ogg_kwargs, + self.compression, self.sample_rate, self.encoding, chunk_size, + adpcm_kwargs=self.adpcm_kwargs, ogg_kwargs=self.ogg_kwargs ) def regenerate_source(self): @@ -151,7 +180,8 @@ def create_from_directory(directory): return new_sound_bank def export_to_directory(self, directory, overwrite=False, - export_source=True, decompress=True): + export_source=True, decompress=True, + ext=constants.CONTAINER_EXT_WAV): for name, pitch_range in self.pitch_ranges.items(): if len(self.pitch_ranges) > 1: pitch_directory = Path(directory, name) @@ -159,7 +189,9 @@ def export_to_directory(self, directory, overwrite=False, pitch_directory = Path(directory) pitch_range.export_to_directory( - pitch_directory, overwrite, export_source, decompress) + pitch_directory, overwrite, export_source, decompress, ext, + adpcm_kwargs=self.adpcm_kwargs, ogg_kwargs=self.ogg_kwargs + ) def import_from_directory(self, directory, clear_existing=True, merge_same_names=False): diff --git a/reclaimer/sounds/blam_sound_permutation.py b/reclaimer/sounds/blam_sound_permutation.py index c64694c1..1f0bb695 100644 --- a/reclaimer/sounds/blam_sound_permutation.py +++ b/reclaimer/sounds/blam_sound_permutation.py @@ -10,7 +10,7 @@ from pathlib import Path from traceback import format_exc -from reclaimer.sounds import constants, util, blam_sound_samples +from reclaimer.sounds import blam_sound_samples, constants as const, ogg, util from supyr_struct.defs.audio.wav import wav_def from supyr_struct.util import is_path_empty @@ -19,18 +19,19 @@ class BlamSoundPermutation: name = "" # permutation properties + _source_filename = "" _source_sample_data = b'' - _source_compression = constants.COMPRESSION_PCM_16_LE - _source_sample_rate = constants.SAMPLE_RATE_22K - _source_encoding = constants.ENCODING_MONO + _source_compression = const.COMPRESSION_PCM_16_LE + _source_sample_rate = const.SAMPLE_RATE_22K + _source_encoding = const.ENCODING_MONO # processed properties _processed_samples = () def __init__(self, sample_data=b'', - compression=constants.COMPRESSION_PCM_16_LE, - sample_rate=constants.SAMPLE_RATE_22K, - encoding=constants.ENCODING_MONO, **kwargs): + compression=const.COMPRESSION_PCM_16_LE, + sample_rate=const.SAMPLE_RATE_22K, + encoding=const.ENCODING_MONO, **kwargs): self.load_source_samples( sample_data, compression, sample_rate, encoding) @@ -46,6 +47,9 @@ def source_sample_rate(self): @property def source_encoding(self): return self._source_encoding + @property + def source_filename(self): + return self._source_filename @property def processed_samples(self): @@ -70,12 +74,13 @@ def encoding(self): return self._source_encoding def load_source_samples(self, sample_data, compression, - sample_rate, encoding): + sample_rate, encoding, filename=""): self._source_sample_data = sample_data self._source_compression = compression self._source_sample_rate = sample_rate - self._source_encoding = encoding - self._processed_samples = [] + self._source_encoding = encoding + self._source_filename = filename + self._processed_samples = [] def partition_samples(self, target_compression=None, target_sample_rate=None, target_encoding=None, @@ -89,23 +94,20 @@ def partition_samples(self, target_compression=None, if target_encoding is None: target_encoding = self.source_encoding - if self.source_compression == constants.COMPRESSION_OGG: - raise NotImplementedError( - "Cannot partition Ogg Vorbis samples.") - elif (target_compression not in constants.PCM_FORMATS and - target_compression != constants.COMPRESSION_IMA_ADPCM and - target_compression != constants.COMPRESSION_XBOX_ADPCM and - target_compression != constants.COMPRESSION_OGG): + if (target_compression not in const.PCM_FORMATS and + target_compression != const.COMPRESSION_IMA_ADPCM and + target_compression != const.COMPRESSION_XBOX_ADPCM and + target_compression != const.COMPRESSION_OGG): raise ValueError('Unknown compression type "%s"' % target_compression) - elif target_encoding not in (constants.ENCODING_MONO, - constants.ENCODING_STEREO): + elif target_encoding not in (const.ENCODING_MONO, + const.ENCODING_STEREO): raise ValueError("Compression encoding must be mono or stereo.") elif target_sample_rate <= 0: raise ValueError("Sample rate must be greater than zero.") source_compression = self.source_compression source_sample_rate = self.source_sample_rate - source_encoding = self.source_encoding + source_encoding = self.source_encoding source_sample_data = self.source_sample_data target_chunk_size = util.get_sample_chunk_size( @@ -113,9 +115,9 @@ def partition_samples(self, target_compression=None, if (source_compression == target_compression and source_sample_rate == target_sample_rate and source_encoding == target_encoding and - (source_compression in constants.PCM_FORMATS or - source_compression == constants.COMPRESSION_IMA_ADPCM or - source_compression == constants.COMPRESSION_XBOX_ADPCM)): + (source_compression in const.PCM_FORMATS or + source_compression == const.COMPRESSION_IMA_ADPCM or + source_compression == const.COMPRESSION_XBOX_ADPCM)): # compressing to same settings and can split at target_chunk_size # because format has fixed compression ratio. recompression not # necessary. Just split source into pieces at target_chunk_size. @@ -124,15 +126,12 @@ def partition_samples(self, target_compression=None, else: # decompress samples so we can partition to a # different compression/encoding/sample rate - decompressor = blam_sound_samples.BlamSoundSamples( - source_sample_data, 0, source_compression, - source_sample_rate, source_encoding - ) - source_compression = constants.DEFAULT_UNCOMPRESSED_FORMAT + source_compression = const.DEFAULT_UNCOMPRESSED_FORMAT source_sample_rate = target_sample_rate - source_encoding = target_encoding - source_sample_data = decompressor.get_decompressed( - source_compression, source_sample_rate, source_encoding) + source_encoding = target_encoding + source_sample_data = self.decompress_source_samples( + source_compression, source_sample_rate, source_encoding + ) source_bytes_per_sample = util.get_block_size( source_compression, source_encoding) @@ -173,6 +172,20 @@ def compress_samples(self, compression, sample_rate=None, encoding=None, samples.compress(compression, sample_rate, encoding, **compressor_kwargs) + def decompress_source_samples(self, compression, sample_rate, encoding): + assert compression in const.PCM_FORMATS + assert encoding in const.channel_counts + + # decompress samples so we can partition to a + # different compression/encoding/sample rate + decompressor = blam_sound_samples.BlamSoundSamples( + self.source_sample_data, 0, self.source_compression, + self.source_sample_rate, self.source_encoding + ) + return decompressor.get_decompressed( + compression, sample_rate, encoding + ) + def get_concatenated_sample_data(self, target_compression=None, target_sample_rate=None, target_encoding=None): @@ -185,11 +198,11 @@ def get_concatenated_sample_data(self, target_compression=None, if target_encoding is None: target_encoding = self.source_encoding - assert target_encoding in constants.channel_counts + assert target_encoding in const.channel_counts if (target_compression != self.compression or target_sample_rate != self.sample_rate or - target_encoding != self.encoding): + target_encoding != self.encoding): # decompress processed samples to the target compression sample_data = b''.join( p.get_decompressed( @@ -203,7 +216,7 @@ def get_concatenated_sample_data(self, target_compression=None, if piece.compression != compression: raise ValueError( "Cannot combine differently compressed samples without decompressing.") - elif piece.compression == constants.COMPRESSION_OGG: + elif piece.compression == const.COMPRESSION_OGG: raise ValueError( "Cannot combine ogg samples without decompressing.") @@ -214,21 +227,27 @@ def get_concatenated_sample_data(self, target_compression=None, def get_concatenated_mouth_data(self): return b''.join(p.mouth_data for p in self.processed_samples) - def regenerate_source(self): + def regenerate_source( + self, compression=None, sample_rate=None, encoding=None + ): ''' Regenerates an uncompressed, concatenated audio stream from the compressed samples. Use when loading a sound tag for re-compression, re-sampling, or re-encoding. ''' - # always regenerate to constants.DEFAULT_UNCOMPRESSED_FORMAT + # default to regenerating to const.DEFAULT_UNCOMPRESSED_FORMAT # because, technically speaking, that is highest sample depth # we can ever possibly see in Halo CE. + if compression is None: compression = const.DEFAULT_UNCOMPRESSED_FORMAT + if sample_rate is None: sample_rate = self.sample_rate + if encoding is None: encoding = self.encoding + self._source_sample_data = self.get_concatenated_sample_data( - constants.DEFAULT_UNCOMPRESSED_FORMAT, - self.sample_rate, self.encoding) - self._source_compression = constants.DEFAULT_UNCOMPRESSED_FORMAT - self._source_sample_rate = self.sample_rate - self._source_encoding = self.encoding + compression, sample_rate, encoding + ) + self._source_compression = compression + self._source_sample_rate = sample_rate + self._source_encoding = encoding @staticmethod def create_from_file(filepath): @@ -242,132 +261,177 @@ def create_from_file(filepath): return new_perm def export_to_file(self, filepath_base, overwrite=False, - export_source=True, decompress=True): + export_source=True, decompress=True, + ext=const.CONTAINER_EXT_WAV, **kwargs): perm_chunks = [] - encoding = self.encoding - sample_rate = self.sample_rate + + filepath = Path(util.BAD_PATH_CHAR_REMOVAL.sub("_", str(filepath_base))) + filepath = Path("unnamed" if is_path_empty(filepath) else filepath) + filepath = filepath.with_suffix(ext) + if export_source and self.source_sample_data: # export the source data - perm_chunks.append( - (self.compression, self.source_encoding, self.source_sample_data) - ) - sample_rate = self.source_sample_rate + perm_chunks.append(( + filepath, self.compression, self.source_sample_rate, + self.source_encoding, self.source_sample_data, + )) elif self.processed_samples: # concatenate processed samples if source samples don't exist. - # also, if compression is ogg, we have to decompress + # if compression isn't some form of PCM, need to decompress it compression = self.compression - if decompress or compression == constants.COMPRESSION_OGG: - compression = constants.COMPRESSION_PCM_16_LE + if decompress or compression not in const.PCM_FORMATS: + compression = const.COMPRESSION_PCM_16_LE try: sample_data = self.get_concatenated_sample_data( - compression, sample_rate, encoding) + compression, self.sample_rate, self.encoding + ) if sample_data: - perm_chunks.append((compression, self.encoding, sample_data)) + perm_chunks.append(( + filepath, compression, self.sample_rate, + self.encoding, sample_data, + )) except Exception: - print("Could not decompress permutation pieces. Concatenating.") - perm_chunks.extend( - (piece.compression, piece.encoding, piece.sample_data) - for piece in self.processed_samples - ) - - i = -1 - wav_file = wav_def.build() - for compression, encoding, sample_data in perm_chunks: - i += 1 - filepath = Path(util.BAD_PATH_CHAR_REMOVAL.sub("_", str(filepath_base))) - - if is_path_empty(filepath): - filepath = Path("unnamed") - - if len(perm_chunks) > 1: - filepath = filepath.parent.joinpath(filepath.stem + "__%s" % i) - - # figure out if the sample data is already encapsulated in a - # container, or if it'll need to be encapsulated in a wav file. - is_container_format = True - if compression == constants.COMPRESSION_OGG: - ext = ".ogg" - elif compression == constants.COMPRESSION_WMA: - ext = ".wma" - elif compression == constants.COMPRESSION_UNKNOWN: - ext = ".bin" + print("Could not decompress permutation pieces. Exporting in pieces.") + # gotta switch format to the container it's compressed as + for i, piece in enumerate(self.processed_samples): + ext = { + const.COMPRESSION_OGG: const.CONTAINER_EXT_OGG, + const.COMPRESSION_OPUS: const.CONTAINER_EXT_OPUS, + const.COMPRESSION_FLAC: const.CONTAINER_EXT_FLAC, + }.get(piece.compression, ext) + + name_base = "%s__%%s%s" % (filepath.stem, ext) + perm_chunks.append(( + filepath.with_name(name_base % i), piece.compression, + piece.sample_rate, piece.encoding, piece.sample_data + )) + + for perm_info in perm_chunks: + filepath, comp, rate, enc, data = perm_info + ext = filepath.suffix.lower() + if not data or (not overwrite and filepath.is_file()): + return + elif ext not in const.SUPPORTED_EXPORT_EXTS: + raise ValueError("Unsupported audio extension '%s'." % ext) + elif ext in const.PYOGG_CONTAINER_EXTS: + exporter = BlamSoundPermutation._export_to_pyogg_file else: - is_container_format = False - ext = ".wav" - - if filepath.suffix.lower() != ext: - filepath = filepath.with_suffix(ext) - - if not sample_data or (not overwrite and filepath.is_file()): - continue - - if is_container_format: - try: - filepath.parent.mkdir(exist_ok=True, parents=True) - with filepath.open("wb") as f: - f.write(sample_data) - except Exception: - print(format_exc()) - - continue - - wav_file.filepath = filepath - - wav_fmt = wav_file.data.wav_format - wav_chunks = wav_file.data.wav_chunks - wav_chunks.append(case="data") - data_chunk = wav_chunks[-1] - - wav_fmt.fmt.data = constants.WAV_FORMAT_PCM - wav_fmt.channels = constants.channel_counts.get(encoding, 1) - wav_fmt.sample_rate = sample_rate - - samples_len = len(sample_data) - if compression in constants.PCM_FORMATS: - # one of the uncompressed pcm formats - if util.is_big_endian_pcm(compression): - sample_data = util.convert_pcm_to_pcm( - sample_data, compression, - util.change_pcm_endianness(compression)) - - sample_width = constants.sample_widths[compression] - wav_fmt.bits_per_sample = sample_width * 8 - wav_fmt.block_align = sample_width * wav_fmt.channels - wav_fmt.byte_rate = wav_fmt.sample_rate * wav_fmt.block_align - elif compression == constants.COMPRESSION_IMA_ADPCM: - # 16bit adpcm - wav_fmt.fmt.data = constants.WAV_FORMAT_IMA_ADPCM - wav_fmt.bits_per_sample = 16 - wav_fmt.block_align = constants.IMA_ADPCM_COMPRESSED_BLOCKSIZE * wav_fmt.channels - wav_fmt.byte_rate = int( - (wav_fmt.sample_rate * wav_fmt.block_align / - (constants.IMA_ADPCM_DECOMPRESSED_BLOCKSIZE // 2)) - ) - elif compression == constants.COMPRESSION_XBOX_ADPCM: - # 16bit adpcm - wav_fmt.fmt.data = constants.WAV_FORMAT_XBOX_ADPCM - wav_fmt.bits_per_sample = 16 - wav_fmt.block_align = constants.XBOX_ADPCM_COMPRESSED_BLOCKSIZE * wav_fmt.channels - wav_fmt.byte_rate = int( - (wav_fmt.sample_rate * wav_fmt.block_align / - (constants.XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE // 2)) - ) - else: - print("Unknown compression method:", compression) - continue + exporter = BlamSoundPermutation._export_to_wav + + try: + exporter(*perm_info, **kwargs) + except Exception: + print(format_exc()) + + @staticmethod + def _export_to_pyogg_file( + filepath, compression, sample_rate, + encoding, sample_data, **kwargs + ): + ext = filepath.suffix.lower() + if ext != const.CONTAINER_EXT_OGG: + raise ValueError("Exporting '%s' is currently unsupported" % ext) + elif compression != const.COMPRESSION_OGG: + # need to encode data to oggvorbis first + sample_data = ogg.encode_oggvorbis( + sample_data, sample_rate, + constants.sample_widths[compression], + constants.channel_counts[encoding], + util.is_big_endian_pcm(compression), + **kwargs.get("ogg_kwargs", {}) + ) - data_chunk.data = sample_data - wav_file.data.wav_header.filesize = 36 + samples_len + filepath.parent.mkdir(exist_ok=True, parents=True) + with filepath.open("wb") as f: + f.write(sample_data) - wav_file.serialize(temp=False, backup=False) + @staticmethod + def _export_to_wav( + filepath, compression, sample_rate, encoding, + sample_data, **kwargs + ): + if not (compression in const.PCM_FORMATS or + compression == const.COMPRESSION_IMA_ADPCM or + compression == const.COMPRESSION_XBOX_ADPCM + ): + print("Unknown compression method:", compression) + return + + block_ratio = 1 + block_size = const.sample_widths[compression] + if compression == const.COMPRESSION_IMA_ADPCM: + # 16bit imaadpcm + block_ratio /= const.IMA_ADPCM_DECOMPRESSED_BLOCKSIZE * 2 + block_size = const.IMA_ADPCM_COMPRESSED_BLOCKSIZE + elif compression == const.COMPRESSION_XBOX_ADPCM: + # 16bit xbox adpcm + block_ratio /= const.XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE * 2 + block_size = const.XBOX_ADPCM_COMPRESSED_BLOCKSIZE + elif util.is_big_endian_pcm(compression): + # one of the big-endian uncompressed pcm formats + sample_data = util.convert_pcm_to_pcm( + sample_data, compression, + util.change_pcm_endianness(compression) + ) + + wav_file = wav_def.build() + wav_file.filepath = filepath + + wav_fmt = wav_file.data.wav_format + wav_chunks = wav_file.data.wav_chunks + wav_chunks.append(case="data") + data_chunk = wav_chunks[-1] + + wav_fmt.channels = const.channel_counts.get(encoding, 1) + wav_fmt.sample_rate = sample_rate + wav_fmt.bits_per_sample = block_size * 8 + wav_fmt.block_align = block_size * wav_fmt.channels + wav_fmt.byte_rate = int( + sample_rate * wav_fmt.block_align * block_ratio + ) + wav_fmt.fmt.data = { + const.COMPRESSION_IMA_ADPCM: const.WAV_FORMAT_IMA_ADPCM, + const.COMPRESSION_XBOX_ADPCM: const.WAV_FORMAT_XBOX_ADPCM, + }.get(compression, const.WAV_FORMAT_PCM) + + data_chunk.data = sample_data + wav_file.data.wav_header.filesize = 36 + len(sample_data) + + wav_file.serialize(temp=False, backup=False) def import_from_file(self, filepath): filepath = Path(filepath) if not filepath.is_file(): raise OSError('File "%s" does not exist. Cannot import.' % filepath) - wav_file = wav_def.build(filepath=filepath) + ext = filepath.suffix.lower() + if ext == const.CONTAINER_EXT_WAV: + self._import_from_wav(filepath) + elif (ext in const.PYOGG_CONTAINER_EXTS and + ext in const.SUPPORTED_IMPORT_EXTS): + self._import_from_pyogg_file(filepath, ext) + else: + raise ValueError("Unsupported audio extension '%s'." % ext) + + self._source_filename = filepath.name + + def _import_from_pyogg_file(self, filepath, ext): + pyogg_audio_file = ogg.pyogg_audiofile_from_filepath( + filepath, ext, streaming=False + ) + sample_data = pyogg_audio_file.buffer + sample_rate = pyogg_audio_file.frequency + compression = const.OGG_DECOMPRESSED_FORMAT + encoding = ( + const.ENCODING_STEREO if pyogg_audio_file.channels == 2 else + const.ENCODING_MONO if pyogg_audio_file.channels == 1 else + const.ENCODING_UNKNOWN + ) + self.load_source_samples(sample_data, compression, sample_rate, encoding) + + def _import_from_wav(self, filepath): + wav_file = wav_def.build(filepath=filepath) wav_header = wav_file.data.wav_header wav_format = wav_file.data.wav_format wav_chunks = wav_file.data.wav_chunks @@ -388,7 +452,7 @@ def import_from_file(self, filepath): "Format signature is invalid. Not a valid wav file.") elif data_chunk is None: raise ValueError("Data chunk not present. Not a valid wav file.") - elif wav_format.fmt.data not in constants.ALLOWED_WAV_FORMATS: + elif wav_format.fmt.data not in const.ALLOWED_WAV_FORMATS: raise ValueError( 'Invalid compression format "%s".' % wav_format.fmt.data) elif wav_format.channels not in (1, 2): @@ -398,12 +462,12 @@ def import_from_file(self, filepath): elif wav_format.sample_rate == 0: raise ValueError( "Sample rate cannot be zero. Not a valid wav file") - elif (wav_format.fmt.data == constants.WAV_FORMAT_PCM_FLOAT and + elif (wav_format.fmt.data == const.WAV_FORMAT_PCM_FLOAT and wav_format.bits_per_sample != 32): raise ValueError( "Pcm float sample width must be 32, not %s." % wav_format.bits_per_sample) - elif (wav_format.fmt.data == constants.WAV_FORMAT_PCM and + elif (wav_format.fmt.data == const.WAV_FORMAT_PCM and wav_format.bits_per_sample not in (8, 16, 24, 32)): raise ValueError( "Pcm sample width must be 8, 16, 24, or 32, not %s." % @@ -419,20 +483,20 @@ def import_from_file(self, filepath): "Sample data may be truncated.") if wav_format.channels == 2: - encoding = constants.ENCODING_STEREO + encoding = const.ENCODING_STEREO else: - encoding = constants.ENCODING_MONO + encoding = const.ENCODING_MONO sample_data = data_chunk.data - if wav_format.fmt.data == constants.WAV_FORMAT_PCM_FLOAT: - sample_data = util.convert_pcm_float32_to_pcm_32(sample_data) - compression = constants.COMPRESSION_PCM_32_LE + if wav_format.fmt.data == const.WAV_FORMAT_PCM_FLOAT: + sample_data = util.convert_pcm_float32_to_pcm_int(sample_data, 4) + compression = const.COMPRESSION_PCM_32_LE else: sample_width = None - if wav_format.fmt.data == constants.WAV_FORMAT_PCM: + if wav_format.fmt.data == const.WAV_FORMAT_PCM: sample_width = wav_format.bits_per_sample // 8 - compression = constants.wav_format_mapping.get( + compression = const.wav_format_mapping.get( (wav_format.fmt.data, sample_width)) self.load_source_samples( diff --git a/reclaimer/sounds/blam_sound_samples.py b/reclaimer/sounds/blam_sound_samples.py index 0132d4b8..1ab1b888 100644 --- a/reclaimer/sounds/blam_sound_samples.py +++ b/reclaimer/sounds/blam_sound_samples.py @@ -9,7 +9,7 @@ from traceback import format_exc -from reclaimer.sounds import constants, ogg, util, adpcm +from reclaimer.sounds import adpcm, constants, ogg, util class BlamSoundSamples: @@ -65,9 +65,10 @@ def compress(self, target_compression, target_sample_rate=None, return if (target_compression == constants.COMPRESSION_OGG and - not constants.OGG_VORBIS_AVAILABLE): + not constants.OGGVORBIS_AVAILABLE): raise NotImplementedError( - "Ogg encoder not available. Cannot compress.") + "Ogg encoder not available. Cannot compress." + ) elif (target_compression not in constants.PCM_FORMATS and target_compression != constants.COMPRESSION_XBOX_ADPCM and target_compression != constants.COMPRESSION_IMA_ADPCM and @@ -112,21 +113,25 @@ def compress(self, target_compression, target_sample_rate=None, constants.ADPCM_DECOMPRESSED_FORMAT) compression = constants.ADPCM_DECOMPRESSED_FORMAT - adpcm_kwargs = compressor_kwargs.get("adpcm_kwargs", {}) - sample_data = adpcm.encode_adpcm_samples( sample_data, constants.channel_counts[target_encoding], - util.is_big_endian_pcm(compression), **adpcm_kwargs) + util.is_big_endian_pcm(compression), + **compressor_kwargs.get("adpcm_kwargs", {}) + ) elif target_compression == constants.COMPRESSION_OGG: # compress to ogg vorbis - # TODO: Finish this - ogg_kwargs = compressor_kwargs.get("ogg_kwargs", {}) - - raise NotImplementedError("Whoops, ogg is not implemented.") + sample_data = ogg.encode_oggvorbis( + sample_data, target_sample_rate, + constants.sample_widths[compression], + constants.channel_counts[target_encoding], + util.is_big_endian_pcm(compression), + **compressor_kwargs.get("ogg_kwargs", {}) + ) elif target_compression != self.compression: # convert to a different pcm format sample_data = util.convert_pcm_to_pcm( - sample_data, self.compression, target_compression) + sample_data, self.compression, target_compression + ) self._sample_data = sample_data self._compression = target_compression @@ -146,21 +151,22 @@ def get_decompressed(self, target_compression, target_sample_rate=None, assert target_sample_rate > 0 curr_compression = self.compression + curr_encoding = self.encoding + curr_sample_rate = self.sample_rate if curr_compression in (constants.COMPRESSION_XBOX_ADPCM, constants.COMPRESSION_IMA_ADPCM): # decompress adpcm to 16bit pcm sample_data = adpcm.decode_adpcm_samples( - self.sample_data, constants.channel_counts[self.encoding], + self.sample_data, constants.channel_counts[curr_encoding], util.is_big_endian_pcm(target_compression)) curr_compression = constants.ADPCM_DECOMPRESSED_FORMAT elif not self.is_compressed: # samples are decompressed. use as-is sample_data = self.sample_data elif curr_compression == constants.COMPRESSION_OGG: - if not constants.OGG_VORBIS_AVAILABLE: - raise NotImplementedError( - "Ogg decoder not available. Cannot decompress.") - # TODO: Finish this + sample_data, curr_compression, curr_encoding, curr_sample_rate =\ + ogg.decode_oggvorbis(self.sample_data) + elif curr_compression == constants.COMPRESSION_WMA: if not constants.WMA_AVAILABLE: raise NotImplementedError( @@ -170,12 +176,14 @@ def get_decompressed(self, target_compression, target_sample_rate=None, raise ValueError("Unknown compression format.") if (curr_compression != target_compression or - self.encoding != target_encoding or - self.sample_rate != target_sample_rate): + curr_encoding != target_encoding or + curr_sample_rate != target_sample_rate): sample_data = util.convert_pcm_to_pcm( - sample_data, curr_compression, target_compression, - self.encoding, target_encoding, - self.sample_rate, target_sample_rate) + sample_data, + curr_compression, target_compression, + curr_encoding, target_encoding, + curr_sample_rate, target_sample_rate + ) return sample_data diff --git a/reclaimer/sounds/constants.py b/reclaimer/sounds/constants.py index 349ad1f9..c61766f4 100644 --- a/reclaimer/sounds/constants.py +++ b/reclaimer/sounds/constants.py @@ -7,26 +7,56 @@ # See LICENSE for more information. # +import pathlib import sys +import tempfile + +PLAYBACK_AVAILABLE = False +OGGVORBIS_AVAILABLE = False +FLAC_AVAILABLE = False +OPUS_AVAILABLE = False +WMA_AVAILABLE = False + +OGGVORBIS_ENCODING_AVAILABLE = False + +TEMP_ROOT = pathlib.Path(tempfile.gettempdir(), "reclaimer_tmp") +OGGVORBIS_TMPNAME_FORMAT = "pyogg_tmpfile_%s%s" try: - from reclaimer.sounds.ext import ogg_ext - OGG_VORBIS_AVAILABLE = True + import pyogg + OGGVORBIS_AVAILABLE = ( + pyogg.PYOGG_OGG_AVAIL and + pyogg.PYOGG_VORBIS_AVAIL and + pyogg.PYOGG_VORBIS_FILE_AVAIL + ) + FLAC_AVAILABLE = pyogg.PYOGG_FLAC_AVAIL + OPUS_AVAILABLE = ( + pyogg.PYOGG_OPUS_AVAIL and + pyogg.PYOGG_OPUS_FILE_AVAIL + ) + # encoding isn't available in all releases + OGGVORBIS_ENCODING_AVAILABLE = OGGVORBIS_AVAILABLE and getattr( + pyogg, "PYOGG_VORBIS_ENC_AVAIL", False + ) + + # NOTE: for right now this won't be available. + # still need to implement it. + #OGGVORBIS_ENCODING_AVAILABLE = False except ImportError: - OGG_VORBIS_AVAILABLE = False + pass try: - from reclaimer.sounds.ext import wma_ext - WMA_AVAILABLE = True + import simpleaudio + del simpleaudio + PLAYBACK_AVAILABLE = True except ImportError: - WMA_AVAILABLE = False + pass SOUND_COMPILE_MODE_NEW = 0 SOUND_COMPILE_MODE_PRESERVE = 1 SOUND_COMPILE_MODE_ADDITIVE = 2 - COMPRESSION_UNKNOWN = -1 # NOTE: the ordering of these constants is such that their endianess # can be swapped by flipping the first bit. This is used in util.py @@ -45,14 +75,18 @@ COMPRESSION_OGG = 18 # halo pc only COMPRESSION_WMA = 19 # halo 2 only +# picking much higher enum values that will never actually be used +COMPRESSION_OPUS = 1024 +COMPRESSION_FLAC = 1025 + # these encoding constants mirror halo 1/2 enum values. ENCODING_UNKNOWN = -1 -ENCODING_MONO = 0 -ENCODING_STEREO = 1 -ENCODING_CODEC = 2 +ENCODING_MONO = 0 +ENCODING_STEREO = 1 +ENCODING_CODEC = 2 SAMPLE_RATE_22K = 22050 -SAMPLE_RATE_32K = 32000 +SAMPLE_RATE_32K = 32000 # halo 2 SAMPLE_RATE_44K = 44100 # Halo constants @@ -70,8 +104,8 @@ XBOX_ADPCM_COMPRESSED_BLOCKSIZE = 36 XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE = 128 -IMA_ADPCM_COMPRESSED_BLOCKSIZE = 36 # not correct -IMA_ADPCM_DECOMPRESSED_BLOCKSIZE = 128 # not correct +IMA_ADPCM_COMPRESSED_BLOCKSIZE = 36 # not correct +IMA_ADPCM_DECOMPRESSED_BLOCKSIZE = 128 # not correct # Wave file format constants WAV_FORMAT_PCM = 0x0001 @@ -85,6 +119,34 @@ WAV_FORMAT_IMA_ADPCM, WAV_FORMAT_XBOX_ADPCM )) +PYOGG_CONTAINER_FORMATS = set(( + COMPRESSION_OGG, + COMPRESSION_OPUS, + COMPRESSION_FLAC, + )) + +CONTAINER_EXT_WAV = ".wav" +CONTAINER_EXT_OGG = ".ogg" +CONTAINER_EXT_OPUS = ".opus" +CONTAINER_EXT_FLAC = ".flac" +PYOGG_CONTAINER_EXTS = frozenset(( + CONTAINER_EXT_OGG, + CONTAINER_EXT_OPUS, + CONTAINER_EXT_FLAC + )) +SUPPORTED_IMPORT_EXTS = frozenset(( + CONTAINER_EXT_WAV, + *([CONTAINER_EXT_OGG] if OGGVORBIS_AVAILABLE else []), + *([CONTAINER_EXT_OPUS] if OPUS_AVAILABLE else []), + *([CONTAINER_EXT_FLAC] if FLAC_AVAILABLE else []), + )) +SUPPORTED_EXPORT_EXTS = frozenset(( + CONTAINER_EXT_WAV, CONTAINER_EXT_OGG + )) + +# for all our purposes we only care about +# decoding ogg to little-endian 16bit pcm +OGG_DECOMPRESSED_FORMAT = COMPRESSION_PCM_16_LE # Endianness interop constants if sys.byteorder == "little": @@ -129,6 +191,9 @@ COMPRESSION_PCM_24_BE: 3, COMPRESSION_PCM_32_LE: 4, COMPRESSION_PCM_32_BE: 4, + # this is what width they get decompressed to + COMPRESSION_XBOX_ADPCM: 2, + COMPRESSION_IMA_ADPCM: 2, } # maps wave format enum options and sample widths to our compression constants @@ -153,6 +218,10 @@ 0: SAMPLE_RATE_22K, 1: SAMPLE_RATE_44K, } +halo_1_encodings = { + 0: ENCODING_MONO, + 1: ENCODING_STEREO, + } # these mappings key halo 2 compression enums # to the compression/sample rate constants @@ -168,6 +237,13 @@ 1: SAMPLE_RATE_44K, 2: SAMPLE_RATE_32K, } +halo_2_encodings = { + 0: ENCODING_MONO, + 1: ENCODING_STEREO, + 2: ENCODING_CODEC, + } # unneeded for export +del pathlib del sys +del tempfile \ No newline at end of file diff --git a/reclaimer/sounds/ogg.py b/reclaimer/sounds/ogg.py index f5c2dc26..ca2be66f 100644 --- a/reclaimer/sounds/ogg.py +++ b/reclaimer/sounds/ogg.py @@ -1,69 +1,316 @@ -# -# This file is part of Reclaimer. -# -# For authors and copyright check AUTHORS.TXT -# -# Reclaimer is free software under the GNU General Public License v3.0. -# See LICENSE for more information. -# - -class VorbisBitrateInfo: - ''' - Intermediary class for storing bitrate info to pass to vorbis compression - functions. - - Combinations of nominal, lower, upper values carry the following - implications: - all three set to the same value: - implies a fixed rate bitstream - only nominal set: - implies a VBR stream that nominals the nominal bitrate. No hard - upper/lower limit - upper and or lower set: - implies a VBR bitstream that obeys the bitrate limits. nominal - may also be set to give a nominal rate. - none set: - the coder does not care to speculate. - ''' - lower = -1 - upper = -1 - nominal = -1 - - # compression base quality [-0.1, 1.0] - quality = None - - use_quality = False - - def __init__(self, nominal=-1, lower=-1, upper=-1, quality=0.5): - ''' - See class documentation for variable descriptions. - ''' - if quality is not None: - self.set_bitrate_quality(quality) - else: - self.set_bitrate_variable(nominal, upper, lower) - - def set_bitrate_fixed(self, bitrate): - ''' - Sets a fixed bitrate to the requested number. - ''' - self.upper = self.nominal = self.lower = bitrate - self.use_quality = False - - def set_bitrate_variable(self, nominal, upper=-1, lower=-1): - ''' - Sets a variable bitrate based on the nominal and - optionally upper and lower numbers. - Check class docstring for further detail. - ''' - self.nominal = nominal - self.upper = upper - self.lower = lower - self.use_quality = False - - def set_bitrate_quality(self, quality): - ''' - Sets a bitrate using the quality float. - ''' - self.quality = min(1.0, max(-0.1, float(quality))) - self.use_quality = True +import ctypes +import io +import itertools +import pathlib +import random +import threading + +import reclaimer +from reclaimer.sounds import constants, util + +try: + import pyogg + from pyogg import vorbis as pyogg_vorbis, ogg as pyogg_ogg +except ImportError: + pyogg = pyogg_vorbis = pyogg_ogg = None + + +class VorbisEncoderSetupError(Exception): + pass + +class VorbisAnalysisError(Exception): + pass + + +# ogg does analysis and compression on 1024 float samples at a time +OGG_ANALYSIS_BUFFER_SIZE = 1024 + + +def pyogg_audiofile_from_filepath(filepath, ext=None, streaming=False): + cls = _get_pyogg_class(filepath, ext, streaming) + return cls(str(filepath)) + + +def pyogg_audiofile_from_data_stream(data_stream, ext, streaming=False): + # look, it's WAY easier to just dump the file to a temp folder and + # have VorbisFile decode the entire thing than to try and hook into + # calling the ogg parsing and vorbis decoder functions on a stream. + # TODO: linux supports memory-only files, so look into doing that + # so we don't have to dump to a temp directory on linux. + filepath = constants.TEMP_ROOT.joinpath( + constants.OGGVORBIS_TMPNAME_FORMAT % ( + threading.get_native_id(), ext + ) + ) + cls = _get_pyogg_class(filepath, ext, streaming) + + filepath.parent.mkdir(parents=True, exist_ok=True) + with open(filepath, "wb") as f: + f.write(data_stream) + + return cls(str(filepath)) + + +def get_ogg_pcm_sample_count(data_stream): + vorbis_file = pyogg_audiofile_from_data_stream( + data_stream, constants.CONTAINER_EXT_OGG, streaming=True + ) + return pyogg_vorbis.libvorbisfile.ov_pcm_total( + ctypes.byref(vorbis_file.vf), 0 + ) + + +def decode_oggvorbis(data_stream): + vorbis_file = pyogg_audiofile_from_data_stream( + data_stream, constants.CONTAINER_EXT_OGG, streaming=False + ) + sample_data = vorbis_file.buffer + sample_rate = vorbis_file.frequency + compression = constants.OGG_DECOMPRESSED_FORMAT + encoding = ( + constants.ENCODING_STEREO if vorbis_file.channels == 2 else + constants.ENCODING_MONO if vorbis_file.channels == 1 else + constants.ENCODING_UNKNOWN + ) + + return sample_data, compression, encoding, sample_rate + + +def encode_oggvorbis( + sample_data, sample_rate, sample_width, channels, is_big_endian=False, + bitrate_lower=-1, bitrate_nominal=-1, bitrate_upper=-1, + quality_setting=1.0, use_quality_value=True, bitrate_managed=True + ): + if not (pyogg and constants.OGGVORBIS_ENCODING_AVAILABLE): + raise NotImplementedError( + "OggVorbis encoder not available. Cannot compress." + ) + elif channels not in (1, 2): + raise NotImplementedError( + "Cannot encode %s channel audio to OggVorbis." % channels + ) + elif sample_width not in (1, 2, 4): + raise NotImplementedError( + "Cannot encode %s-byte width samples to OggVorbis." % width + ) + + # one Ogg bitstream page. Vorbis packets inside + opg = pyogg_ogg.ogg_page() + opg_p = ctypes.pointer(opg) + + # one raw packet of data for decode + opk_p = ctypes.pointer(pyogg_ogg.ogg_packet()) + # takes pages, weld into a logical packet stream + oss_p = ctypes.pointer(pyogg_ogg.ogg_stream_state()) + + # local working space for packet->PCM decode + vbl_p = ctypes.pointer(pyogg_vorbis.vorbis_block()) + # struct that stores all user comments + vco_p = ctypes.pointer(pyogg_vorbis.vorbis_comment()) + # central working state for packet->PCM decoder + vds_p = ctypes.pointer(pyogg_vorbis.vorbis_dsp_state()) + # struct storing all static vorbis bitstream settings + vin_p = ctypes.pointer(pyogg_vorbis.vorbis_info()) + + pyogg_vorbis.vorbis_info_init(vin_p) + + if use_quality_value: + err = pyogg_vorbis.vorbis_encode_init_vbr(vin_p, + ctypes.c_int(channels), + ctypes.c_int(sample_rate), + ctypes.c_float(quality_setting) + ) + else: + err = pyogg_vorbis.vorbis_encode_init(vin_p, + ctypes.c_int(channels), + ctypes.c_int(sample_rate), + ctypes.c_int(bitrate_upper), + ctypes.c_int(bitrate_nominal), + ctypes.c_int(bitrate_lower) + ) + + if err: + raise VorbisEncoderSetupError( + "Vorbis encoder returned error code during setup. " + "Encoder settings are likely incorrect." + ) + + # create an output buffer + ogg_data = io.BytesIO() + + # add comments + pyogg_vorbis.vorbis_comment_init(vco_p) + for string in ( + str.encode("RECLAIMERVER=%s.%s.%s" % reclaimer.__version__), + str.encode("PYOGGVER=%s" % pyogg.__version__), + ): + pyogg_vorbis.vorbis_comment_add(vco_p, + ctypes.create_string_buffer(string) + ) + + # set up the analysis state and auxiliary encoding storage + pyogg_vorbis.vorbis_analysis_init(vds_p, vin_p) + pyogg_vorbis.vorbis_block_init(vds_p, vbl_p) + + # initialise stream state + # pick a random serial number; that way we can more + # likely build chained streams just by concatenation + pyogg_ogg.ogg_stream_init(oss_p, _get_random_serial_no()) + + # Vorbis streams begin with three headers; the initial header + # (with most of the codec setup parameters) which is mandated + # by the Ogg bitstream spec. The second header holds any comment + # fields. The third header holds the bitstream codebook. + # We merely need to make the headers and pass them to libvorbis one + # by one; libvorbis handles the additional Ogg bitstream constraints + ovh_p = ctypes.pointer(pyogg_ogg.ogg_packet()) + ovh_comm_p = ctypes.pointer(pyogg_ogg.ogg_packet()) + ovh_code_p = ctypes.pointer(pyogg_ogg.ogg_packet()) + + pyogg_vorbis.vorbis_analysis_headerout( + vds_p, vco_p, ovh_p, ovh_comm_p, ovh_code_p + ) + pyogg_ogg.ogg_stream_packetin(oss_p, ovh_p) + pyogg_ogg.ogg_stream_packetin(oss_p, ovh_comm_p) + pyogg_ogg.ogg_stream_packetin(oss_p, ovh_code_p) + + # per spec, ensure vorbis audio data starts on a new page, so + # we need to flush the current page and start a new one + while pyogg_ogg.ogg_stream_flush(oss_p, opg_p): + _write_ogg_page(ogg_data, opg) + + # deinterleave the audio if it's stereo, and convert it to float + inp_buffers = [ + util.convert_pcm_int_to_pcm_float32(buffer, sample_width, True) + for buffer in ( + util.deinterleave_stereo(sample_data, sample_width, True) + if channels == 2 else [sample_data] + ) + ] + + i, sample_count = 0, min(len(b) for b in inp_buffers) + samples_per_chunk = OGG_ANALYSIS_BUFFER_SIZE // channels + vorbis_buf_size = ctypes.c_int(OGG_ANALYSIS_BUFFER_SIZE) + + # if using bitrate managed encoding, vorbis coded block is not + # dumped directly to a packet. instead, it must be flushed using + # vorbis_bitrate_flushpacket after calling vorbis_bitrate_addblock. + # when not using bitrate managed encoding, we pass the pointer to + # the packet the vorbis analysis should dump its results directly to. + vorbis_out_opk_p = None if bitrate_managed else opk_p + + # encode the pcm data to vorbis and dump it to the ogg_data buffer + while i <= sample_count: + # figure out how much data to pass to the analysis buffer + read = min(sample_count - i, samples_per_chunk) + + # transfer sample data to analysis buffer(if any) + if read: + vorbis_buffers = pyogg_vorbis.vorbis_analysis_buffer( + vds_p, vorbis_buf_size + ) + for c in range(channels): + tuple( + itertools.starmap( + vorbis_buffers[c].__setitem__, + enumerate(inp_buffers[c][i: i+read]) + )) + + i += read + + # tell the vorbis encoder how many samples to analyze + pyogg_vorbis.vorbis_analysis_wrote(vds_p, ctypes.c_int(read)) + + # get a single block for encoding + while pyogg_vorbis.vorbis_analysis_blockout(vds_p, vbl_p) == 1: + # do analysis + res = pyogg_vorbis.vorbis_analysis(vbl_p, vorbis_out_opk_p) + if res: + raise VorbisAnalysisError("Vorbis analysis returned error: %d" % res) + + # NOTE: we're ALWAYS going to use the bitrate management + # system, as there are no downsides, but i am keeping + # this code here as an example of how to not use it. + # see here for why: + # https://xiph.org/vorbis/doc/libvorbis/overview.html + # + if not bitrate_managed: + pyogg_ogg.ogg_stream_packetin(oss_p, opk_p) + while pyogg_ogg.ogg_stream_pageout(oss_p, opg_p): + _write_ogg_page(ogg_data, opg) + + continue + + pyogg_vorbis.vorbis_bitrate_addblock(vbl_p) + + # while there are packets to flush, weld them into + # the bitstream and write out any resulting pages. + while pyogg_vorbis.vorbis_bitrate_flushpacket(vds_p, opk_p) == 1: + pyogg_ogg.ogg_stream_packetin(oss_p, opk_p) + while pyogg_ogg.ogg_stream_pageout(oss_p, opg_p): + _write_ogg_page(ogg_data, opg) + + if pyogg_ogg.ogg_page_eos(opg_p): + i = sample_count + 1 + break + + # ensure everything is flushed + while pyogg_ogg.ogg_stream_flush(oss_p, opg_p): + _write_ogg_page(ogg_data, opg) + + # finish up by clearing everything + pyogg_ogg.ogg_stream_clear(oss_p) + pyogg_vorbis.vorbis_block_clear(vbl_p) + pyogg_vorbis.vorbis_comment_clear(vco_p) + pyogg_vorbis.vorbis_dsp_clear(vds_p) + pyogg_vorbis.vorbis_info_clear(vin_p) + + ogg_bitstream_bytes = ogg_data.getbuffer().tobytes() + return ogg_bitstream_bytes + + +def _get_random_serial_no(bit_count=(ctypes.sizeof(ctypes.c_int)*8)): + return ctypes.c_int(random.randint( + -(1<<(bit_count - 1)), + (1<<(bit_count - 1))-1 + )) + + +def _c_pointer_to_buffer(pointer, buf_len, buf_typ=ctypes.c_ubyte): + BufferPtr = ctypes.POINTER(buf_typ * buf_len) + return BufferPtr(pointer)[0] + + +def _write_ogg_page(buffer, ogg_page): + buffer.write(_c_pointer_to_buffer( + ogg_page.header.contents, ogg_page.header_len + )) + buffer.write(_c_pointer_to_buffer( + ogg_page.body.contents, ogg_page.body_len + )) + + +def _get_pyogg_class(filepath=None, ext=None, streaming=False): + if filepath and ext is None: + ext = pathlib.Path(filepath).suffix + + if not pyogg: + raise NotImplementedError("PyOgg not available. Cannot open files.") + elif ext not in constants.PYOGG_CONTAINER_EXTS: + raise ValueError("Unknown PyOgg extension '%s'" % ext) + elif ((ext == constants.CONTAINER_EXT_OPUS and not constants.OPUS_AVAILABLE) or + (ext == constants.CONTAINER_EXT_FLAC and not constants.FLAC_AVAILABLE) or + (ext == constants.CONTAINER_EXT_OGG and not constants.OGGVORBIS_AVAILABLE) + ): + raise NotImplementedError( + "PyOgg was improperly initialized. Cannot open '%s' files" % ext + ) + + return ( + (pyogg.OpusFileStream if streaming else pyogg.OpusFile) + if ext == constants.CONTAINER_EXT_OPUS else + (pyogg.FlacFileStream if streaming else pyogg.FlacFile) + if ext == constants.CONTAINER_EXT_FLAC else + (pyogg.VorbisFileStream if streaming else pyogg.VorbisFile) + # NOTE: not checking for ogg ext cause it was done above + ) \ No newline at end of file diff --git a/reclaimer/sounds/playback.py b/reclaimer/sounds/playback.py new file mode 100644 index 00000000..5a46b176 --- /dev/null +++ b/reclaimer/sounds/playback.py @@ -0,0 +1,415 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +import threading +import time +from traceback import format_exc + +try: + import simpleaudio +except ImportError: + simpleaudio = None + +from reclaimer.sounds import audioop, constants, util,\ + blam_sound_permutation, sound_decompilation as sound_decomp + + +def build_wave_object(sample_data, encoding, compression, sample_rate): + #print("CREATING WAVE OBJECT") + if not simpleaudio: + raise NotImplementedError( + "Could not detect simpleaudio. Cannot build WaveObject." + ) + + channel_count = constants.channel_counts.get(encoding, 'unknown') + sample_width = constants.sample_widths.get(compression, 'unknown') + if channel_count not in (1, 2): + raise ValueError( + "Cannot build WaveObject from audio with %s channels" % channel_count + ) + elif sample_width not in (1, 2, 3, 4): + raise ValueError( + "Cannot build WaveObject from samples of %s byte width" % sample_width + ) + + if sample_width > 1 and util.is_big_endian_pcm(compression): + sample_data = audioop.byteswap(sample_data, sample_width) + + return simpleaudio.WaveObject( + sample_data, channel_count, sample_width, sample_rate + ) + + +def wave_object_from_blam_sound_perm(blam_perm): + samples = getattr(blam_perm, "source_sample_data", None) + comp = getattr(blam_perm, "source_compression", None) + enc = getattr(blam_perm, "source_encoding", None) + sr = getattr(blam_perm, "source_sample_rate", None) + if blam_perm and (not samples or comp not in constants.PCM_FORMATS): + comp = constants.COMPRESSION_PCM_16_LE + if samples: + # sample data is compressed in a format we can't + # linearly read, so we need to decompress it. + samples = blam_perm.decompress_source_samples(comp, sr, enc) + else: + # if there is no source sample data, so we'll try + # to use the decompressed processed samples instead. + blam_perm.regenerate_source(comp) + samples = blam_perm.source_sample_data + + return build_wave_object(samples, enc, comp, sr) if samples else None + + +class SoundPlayerBase: + _wave_objects = () + + _play_objects_by_wave_ids = () + _player_locks_by_wave_ids = () + _force_stops_by_wave_ids = () + _merged_play_objects = () + _merged_player_lock = None + _merged_force_stop = None + + pitch_range_index = -1 + permutation_index = -1 + separate_wave_queues = True + concatenate_perm_chain = True + + class ForceStop: + _stop = False + def __bool__(self): return self._stop + def set(self, val): self._stop = bool(val) + + def __init__(self): + self._wave_objects = {} + self._play_objects_by_wave_ids = {} + self._player_locks_by_wave_ids = {} + self._force_stops_by_wave_ids = {} + self._merged_play_objects = {} + self._merged_player_lock = threading.Lock() + self._merged_force_stop = SoundPlayerBase.ForceStop() + + def get_compression(self, wave_id=None): + raise NotImplementedError("Must override this method") + def get_sample_rate(self, wave_id=None): + raise NotImplementedError("Must override this method") + def get_encoding(self, wave_id=None): + raise NotImplementedError("Must override this method") + + def get_play_objects(self, wave_id=None): + wave_id = wave_id or self.get_wave_id() + return ( + self._play_objects_by_wave_ids.setdefault(wave_id, {}) + if self.separate_wave_queues else + self._merged_play_objects + ) + + def get_player_lock(self, wave_id=None): + wave_id = wave_id or self.get_wave_id() + return ( + self._player_locks_by_wave_ids.setdefault(wave_id, threading.Lock()) + if self.separate_wave_queues else + self._merged_player_lock + ) + + def get_force_stop(self, wave_id=None): + wave_id = wave_id or self.get_wave_id() + return ( + self._force_stops_by_wave_ids.setdefault(wave_id, SoundPlayerBase.ForceStop()) + if self.separate_wave_queues else + self._merged_force_stop + ) + + @property + def pitch_ranges(self): + raise NotImplementedError("Must override this method") + @property + def permutations(self): + raise NotImplementedError("Must override this method") + + def get_wave_id(self, pr_index=None, perm_index=None): + if pr_index is None: + pr_index = self.pitch_range_index + if perm_index is None: + perm_index = self.permutation_index + + wave_id = ( + None if pr_index not in self.pitch_ranges else pr_index, + None if perm_index not in self.permutations else perm_index, + ) + return None if None in wave_id else wave_id + + def get_pitch_range(self, pr_index=None): + if pr_index is None: + pr_index = self.pitch_range_index + + return self.pitch_ranges.get(pr_index) + + def get_permutation(self, pr_index=None, perm_index=None): + if perm_index is None: + perm_index = self.permutation_index + + pr = self.get_pitch_range(pr_index) + return None if pr is None else pr.permutations.get(pr_index) + + def play_sound(self, wave_id=None, threaded=True): + try: + comp = self.get_compression(wave_id) + if not ( + (comp == constants.COMPRESSION_OGG and constants.OGGVORBIS_AVAILABLE) or + (comp == constants.COMPRESSION_OPUS and constants.OPUS_AVAILABLE) or + (comp == constants.COMPRESSION_FLAC and constants.FLAC_AVAILABLE) or + (comp == constants.COMPRESSION_XBOX_ADPCM) or + (comp in constants.PCM_FORMATS) + ): + print("Cannot play audio (unknown/unsupported compression: %s)." % comp) + return + + wave_id = wave_id or self.get_wave_id() + pr_index, perm_index = wave_id + if None in wave_id: + return + + wave_object = self._wave_objects.get(wave_id) + + # create the wave object if it doesnt exist yet + if wave_object is None: + blam_perm = self.get_permutation(pr_index, perm_index) + if blam_perm is None: + return + + if not isinstance(blam_perm, blam_sound_permutation.BlamSoundPermutation): + perms = self.permutations + if self.concatenate_perm_chain: + permlist, _ = sound_decomp.get_tag_perm_chain(perms, perm_index) + elif perm_index in range(len(perms)): + permlist = [perms[perm_index]] + else: + permlist = [] + + blam_perm = sound_decomp.tag_perm_chain_to_blam_perm( + permlist, self.get_sample_rate(wave_id), self.get_encoding(wave_id), + (self.get_compression(wave_id) == constants.COMPRESSION_PCM_16_BE) + ) + wave_object = wave_object_from_blam_sound_perm(blam_perm) + + # initiate a queued play of the sound + if wave_object is None: + return + + self._wave_objects[wave_id] = wave_object + # ensure the play_objects queue, lock, and force_stop all exist + self.get_play_objects(wave_id) + self.get_player_lock(wave_id) + self.get_force_stop(wave_id).set(False) + + if threaded: + self.play_wave_object_threaded(wave_id) + else: + self.play_wave_object(wave_id) + + except Exception: + print(format_exc()) + + def stop_sound(self, wave_id=None): + self._stop_play_objects( + self.get_play_objects(wave_id), + self.get_player_lock(wave_id), + self.get_force_stop(wave_id), + ) + + def stop_all_sounds(self): + self._stop_play_objects( + self._merged_play_objects, + self._merged_player_lock, + self._merged_force_stop, + ) + for wave_id in self._play_objects_by_wave_ids.keys(): + self._stop_play_objects( + self._play_objects_by_wave_ids.get(wave_id), + self._player_locks_by_wave_ids.get(wave_id), + self._force_stops_by_wave_ids.get(wave_id), + ) + + def play_wave_object_threaded(self, wave_id=None): + play_thread = threading.Thread( + target=self.play_wave_object, daemon=True, + args=(wave_id or self.get_wave_id(), ) + ) + play_thread.start() + return play_thread + + def play_wave_object(self, wave_id=None): + wave_id = wave_id or self.get_wave_id() + wave_object = self._wave_objects[wave_id] + if wave_object is None: + return + + play_objects = self.get_play_objects(wave_id) + player_lock = self.get_player_lock(wave_id) + force_stop = self.get_force_stop(wave_id) + + # create a spot at the end of the queue and wait our turn + queue_id = 1 + with player_lock: + queue_id = max(tuple(play_objects) + (0, )) + 1 + play_objects[queue_id] = None + #print("%s ENTERED QUEUE" % queue_id) + + # wait till its our turn to play + while play_objects: + curr_po_id, curr_po = queue_id, None + with player_lock: + curr_po_id = min(tuple(play_objects) + (queue_id, )) + curr_po = play_objects.get(curr_po_id) + + if curr_po_id != queue_id and curr_po is None: + # sound that's supposed to play hasn't yet. + # we'll wait a short time, but if it doesn't play + # then we'll need to remove it to remove deadlock + time.sleep(1) + with player_lock: + curr_po_id = min(tuple(play_objects) + (queue_id, )) + curr_po = play_objects.get(curr_po_id) + if curr_po is None: + #print("%s KICKING %s FROM QUEUE" % (queue_id, curr_po_id)) + play_objects.pop(curr_po_id, None) + + #print("%s SEES CURR PLAYING IS %s" % (queue_id, curr_po_id)) + if curr_po_id == queue_id: + break + elif curr_po_id > queue_id: + # our place in line got removed by another thread + # that decided we deadlocked. oh well, gotta return + #print("%s KICKED OUT OF QUEUE. RETURNING" % queue_id) + return + elif curr_po and curr_po.is_playing(): + #print("%s WAITING ON %s" % (queue_id, curr_po_id)) + curr_po.wait_done() + + if force_stop: + #print("%s FORCE STOPPED. RETURNING" % queue_id) + return + + # only play if we're still queued(sounds weren't stopped) + po = None + with player_lock: + #print("%s PLAYING" % queue_id) + # play the sound and add to the queue for tracking + po = play_objects[queue_id] = wave_object.play() + + # wait till sound is done, and then cleanup + try: + po.wait_done() + finally: + #print("%s DONE PLAYING" % queue_id) + play_objects.pop(queue_id, None) + + def _stop_play_objects(self, play_objects, player_lock, force_stop): + force_stop.set(True) + play_objects_copy = dict(play_objects) + play_objects.clear() + for queue_id in play_objects_copy: + try: + play_objects_copy[queue_id].stop() + except Exception: + pass + + +class SoundTagPlayer(SoundPlayerBase): + sound_data = None + big_endian_pcm = False + + @property + def pitch_ranges(self): + try: + return { + i: pr for i, pr in enumerate( + self.sound_data.pitch_ranges.STEPTREE + )} + except AttributeError: + return {} + @property + def permutations(self): + try: + return { + i: pr for i, pr in enumerate( + self.get_pitch_range().permutations.STEPTREE + )} + except AttributeError: + return {} + + def get_permutation(self, pr_index=None, perm_index=None): + if perm_index is None: + perm_index = self.permutation_index + + pr = self.get_pitch_range(pr_index) + perms = None if pr is None else pr.permutations.STEPTREE + return ( + perms[perm_index] + if pr and perm_index in range(len(perms)) + else None + ) + + def get_compression(self, wave_id=None): + block = getattr(self.get_permutation(), "compression", None) + comp = None if block is None else constants.halo_1_compressions[block.data] + return ( + comp if not comp in constants.PCM_FORMATS else + constants.COMPRESSION_PCM_16_BE if self.big_endian_pcm else + constants.COMPRESSION_PCM_16_LE + ) + + def get_sample_rate(self, wave_id=None): + block = getattr(self.sound_data, "sample_rate", None) + return None if block is None else constants.halo_1_sample_rates[block.data] + + def get_encoding(self, wave_id=None): + block = getattr(self.sound_data, "encoding", None) + return None if block is None else constants.halo_1_encodings[block.data] + + +class SoundSourcePlayer(SoundPlayerBase): + sound_bank = None + + @property + def pitch_ranges(self): + try: + prs = self.sound_bank.pitch_ranges + return {i: prs[name] for i, name in enumerate(sorted(prs))} + except AttributeError: + return {} + @property + def permutations(self): + try: + perms = self.get_pitch_range().permutations + return {i: perms[name] for i, name in enumerate(sorted(perms))} + except AttributeError: + return {} + + def get_permutation(self, pr_index=None, perm_index=None): + if perm_index is None: + perm_index = self.permutation_index + + perms = getattr(self.get_pitch_range(pr_index), "permutations", {}) + try: + perm_name = list(sorted(perms))[perm_index] + except IndexError: + perm_name = None + + return perms.get(perm_name) + + def get_compression(self, wave_id=None): + return getattr(self.get_permutation(), "compression", None) + + def get_sample_rate(self, wave_id=None): + return getattr(self.get_permutation(), "sample_rate", None) + + def get_encoding(self, wave_id=None): + return getattr(self.get_permutation(), "encoding", None) \ No newline at end of file diff --git a/reclaimer/sounds/sound_compilation.py b/reclaimer/sounds/sound_compilation.py index ade119f3..5ee52076 100644 --- a/reclaimer/sounds/sound_compilation.py +++ b/reclaimer/sounds/sound_compilation.py @@ -10,7 +10,7 @@ import math import traceback -from reclaimer.sounds import util, constants +from reclaimer.sounds import audioop, constants, ogg, util __all__ = ("compile_sound", "compile_pitch_range",) @@ -37,13 +37,16 @@ def compile_pitch_range(pitch_range, blam_pitch_range, if channel_count != blam_channel_count: errors.append('Cannot add %s channel sounds to %s channel tag.' % - (blam_channel_count, sample_rate)) - - if blam_sound_perm.compression not in (constants.COMPRESSION_PCM_16_LE, - constants.COMPRESSION_PCM_16_BE, - constants.COMPRESSION_XBOX_ADPCM, - constants.COMPRESSION_IMA_ADPCM, - constants.COMPRESSION_OGG): + (blam_channel_count, channel_count)) + + if blam_sound_perm.compression not in ( + constants.COMPRESSION_PCM_16_LE, + constants.COMPRESSION_PCM_16_BE, + constants.COMPRESSION_XBOX_ADPCM, + # not supported in-engine + #constants.COMPRESSION_IMA_ADPCM, + constants.COMPRESSION_OGG + ): errors.append('Unknown permutation compression "%s"' % blam_sound_perm.compression) @@ -57,8 +60,8 @@ def compile_pitch_range(pitch_range, blam_pitch_range, if errors: return errors - snd__perms = pitch_range.permutations.STEPTREE - snd__perm_names = set() + snd__perms = pitch_range.permutations.STEPTREE + snd__perm_names = set() # loop over the permutations blam_samples and string # them together into lists of permutation blocks for blam_perm_name in sorted(blam_pitch_range.permutations): @@ -73,22 +76,33 @@ def compile_pitch_range(pitch_range, blam_pitch_range, snd__perms.append() # create the new perm block snd__perm, _ = snd__perms.pop(-1) snd__perm.name = name - snd__perm.ogg_sample_count = ( - channel_count * 2 * blam_samples.sample_count) snd__perm.samples.data = blam_samples.sample_data snd__perm.mouth_data.data = blam_samples.mouth_data + comp = blam_samples.compression + + if comp == constants.COMPRESSION_PCM_16_LE: + # need to byteswap to big-endian + snd__perm.samples.data = audioop.byteswap(snd__perm.samples.data, 2) + + comp_name = ( + "xbox_adpcm" if comp == constants.COMPRESSION_XBOX_ADPCM else + # not supported in-engine + #"ima_adpcm" if comp == constants.COMPRESSION_IMA_ADPCM else + "ogg" if comp == constants.COMPRESSION_OGG else + "none" + ) - if blam_samples.compression == constants.COMPRESSION_XBOX_ADPCM: - snd__perm.compression.set_to("xbox_adpcm") - snd__perm.ogg_sample_count = 0 # adpcm has this as 0 always - elif blam_samples.compression == constants.COMPRESSION_IMA_ADPCM: - snd__perm.compression.set_to("ima_adpcm") - snd__perm.ogg_sample_count = 0 # adpcm has this as 0 always - elif blam_samples.compression == constants.COMPRESSION_OGG: - snd__perm.compression.set_to("ogg") - else: - snd__perm.compression.set_to("none") + # only ogg and 16bit pcm need the buffer size calculated + sample_count = ( + ogg.get_ogg_pcm_sample_count(blam_samples.sample_data) + if comp_name == "ogg" and constants.OGGVORBIS_AVAILABLE else + blam_samples.sample_count + if comp_name in ("ogg", "none") else + 0 + ) + snd__perm.compression.set_to(comp_name) + snd__perm.buffer_size = channel_count * 2 * sample_count snd__perm_chain.append(snd__perm) errors.extend( @@ -104,7 +118,7 @@ def compile_pitch_range(pitch_range, blam_pitch_range, def compile_sound(snd__tag, blam_sound_bank, ignore_size_limits=False, update_mode=constants.SOUND_COMPILE_MODE_PRESERVE, - force_sample_rate=False): + force_sample_rate=False, sapien_pcm_hack=False): ''' Compiles the given blam_sound_bank into the given (Halo 1 snd!) snd__tag. ''' @@ -128,26 +142,27 @@ def compile_sound(snd__tag, blam_sound_bank, ignore_size_limits=False, tagdata.modifiers_when_scale_is_zero[:] = (1.0, 0.0, 1.0) tagdata.modifiers_when_scale_is_one[:] = (1.0, 1.0, 1.0) - if update_mode != constants.SOUND_COMPILE_MODE_ADDITIVE: # update the flags, compression, encoding, and sample rate # of the tag to that of the samples being stored in it. tagdata.flags.fit_to_adpcm_blocksize = False tagdata.flags.split_long_sound_into_permutations = bool( blam_sound_bank.split_into_smaller_chunks) + tagdata.flags.fit_to_adpcm_blocksize = bool( + blam_sound_bank.adpcm_fit_to_blocksize) if blam_sound_bank.compression in (constants.COMPRESSION_PCM_16_LE, constants.COMPRESSION_PCM_16_BE): - # intentionally set this to something other than "none" + # set this to something other than "none" # as otherwise the sound won't play in sapien - tagdata.compression.set_to("ogg") - # tagdata.compression.set_to("none") + tagdata.compression.set_to("ogg" if sapien_pcm_hack else "none") elif blam_sound_bank.compression == constants.COMPRESSION_XBOX_ADPCM: tagdata.flags.fit_to_adpcm_blocksize = True tagdata.compression.set_to("xbox_adpcm") - elif blam_sound_bank.compression == constants.COMPRESSION_IMA_ADPCM: - tagdata.flags.fit_to_adpcm_blocksize = True - tagdata.compression.set_to("ima_adpcm") + # not supported in-engine + #elif blam_sound_bank.compression == constants.COMPRESSION_IMA_ADPCM: + # tagdata.flags.fit_to_adpcm_blocksize = True + # tagdata.compression.set_to("ima_adpcm") elif blam_sound_bank.compression == constants.COMPRESSION_OGG: tagdata.compression.set_to("ogg") else: diff --git a/reclaimer/sounds/sound_decompilation.py b/reclaimer/sounds/sound_decompilation.py index 9735146f..b3d4562d 100644 --- a/reclaimer/sounds/sound_decompilation.py +++ b/reclaimer/sounds/sound_decompilation.py @@ -17,7 +17,78 @@ from reclaimer.sounds.blam_sound_samples import BlamSoundSamples -__all__ = ("extract_h1_sounds", "extract_h2_sounds", ) +__all__ = ( + "extract_h1_sounds", "extract_h2_sounds", + "tag_perm_chain_to_blam_perm", "get_tag_perm_chain" + ) + + +def tag_perm_chain_to_blam_perm( + sound_perm_chain, sample_rate, encoding, pcm_is_big_endian=False + ): + compression = constants.COMPRESSION_PCM_16_LE + for perm in sound_perm_chain: + compression = constants.halo_1_compressions.get( + perm.compression.data, str(perm.compression.data) + ) + break + + if compression is None: + print("Unknown audio compression type: %s" % compression) + return + + blam_permutation = BlamSoundPermutation( + sample_rate=sample_rate, encoding=encoding, + compression=compression + ) + channels = constants.channel_counts.get(encoding, 1) + + for perm in sound_perm_chain: + sample_data = perm.samples.data + compression = constants.halo_1_compressions.get( + perm.compression.data, str(perm.compression.data) + ) + if compression == constants.COMPRESSION_OGG: + sample_count = perm.buffer_size // 2 + elif compression == constants.COMPRESSION_PCM_16_LE: + compression = (constants.COMPRESSION_PCM_16_BE + if pcm_is_big_endian else + constants.COMPRESSION_PCM_16_LE) + sample_count = len(sample_data) // 2 + elif compression == constants.COMPRESSION_XBOX_ADPCM: + sample_count = ( + (constants.XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE // 2) * + (len(sample_data) // constants.XBOX_ADPCM_COMPRESSED_BLOCKSIZE)) + elif compression == constants.COMPRESSION_IMA_ADPCM: + sample_count = ( + (constants.IMA_ADPCM_DECOMPRESSED_BLOCKSIZE // 2) * + (len(sample_data) // constants.IMA_ADPCM_COMPRESSED_BLOCKSIZE)) + else: + print("Unknown audio compression type: %s" % compression) + continue + + sample_count = sample_count // channels + # NOTE: do NOT modify sample rate by natural pitch. + # it doesn't seem this is how it's used. try + # with warthog_7_engine to hear the issue + blam_permutation.processed_samples.append( + BlamSoundSamples( + sample_data, sample_count, compression, + sample_rate, encoding, perm.mouth_data.data) + ) + + return blam_permutation + + +def get_tag_perm_chain(permutations, perm_index): + seen, permlist = set(), [] + while perm_index not in seen and perm_index in range(len(permutations)): + seen.add(perm_index) + perm = permutations[perm_index] + permlist.append(perm) + perm_index = perm.next_permutation_index + + return permlist, seen def extract_h1_sounds(tagdata, tag_path, **kw): @@ -27,8 +98,6 @@ def extract_h1_sounds(tagdata, tag_path, **kw): pcm_is_big_endian = kw.get('byteswap_pcm_samples', False) tagpath_base = Path(kw['out_dir']).joinpath(Path(tag_path).with_suffix("")) - encoding = tagdata.encoding.data - channels = constants.channel_counts.get(encoding, 1) sample_rate = constants.halo_1_sample_rates.get( tagdata.sample_rate.data, 0) compression = constants.halo_1_compressions.get( @@ -38,9 +107,9 @@ def extract_h1_sounds(tagdata, tag_path, **kw): sound_bank = BlamSoundBank() sound_bank.split_into_smaller_chunks = tagdata.flags.split_long_sound_into_permutations sound_bank.split_to_adpcm_blocksize = tagdata.flags.fit_to_adpcm_blocksize - sound_bank.sample_rate = sample_rate - sound_bank.compression = compression - sound_bank.encoding = encoding + sound_bank.sample_rate = sample_rate + sound_bank.compression = compression + sound_bank.encoding = tagdata.encoding.data same_pr_names = {} for i, pr in enumerate(tagdata.pitch_ranges.STEPTREE): @@ -59,75 +128,33 @@ def extract_h1_sounds(tagdata, tag_path, **kw): same_names_permlists = {} unchecked_perms = set(range(len(pr.permutations.STEPTREE))) - perm_indices = list( - range(min(pr.actual_permutation_count, len(unchecked_perms)))) + perm_indices = list(range( + min(pr.actual_permutation_count, len(unchecked_perms)) + )) if not perm_indices: perm_indices = set(unchecked_perms) - playback_speed = 1.0 - if pr.natural_pitch > 0: - playback_speed = 1 / pr.natural_pitch - while perm_indices: # loop over all of the actual permutation indices and combine # the permutations they point to into a list with a shared name. # we do this so we can combine the permutations together into one. for j in perm_indices: - perm = pr.permutations.STEPTREE[j] - compression = perm.compression.enum_name - name = perm.name if perm.name else str(j) - permlist = same_names_permlists.get(name, []) - same_names_permlists[name] = permlist - - while j in unchecked_perms: - perm = pr.permutations.STEPTREE[j] - if compression != perm.compression.enum_name: - # cant combine when compression is different - break - - unchecked_perms.remove(j) - permlist.append(perm) - - j = perm.next_permutation_index + name = pr.permutations.STEPTREE[j].name.strip() or str(j) + + perm_list, seen = get_tag_perm_chain(pr.permutations.STEPTREE, j) + same_names_permlists.setdefault(name, []).extend(perm_list) + + unchecked_perms.difference_update(seen) + perm_indices = set(unchecked_perms) for name, permlist in same_names_permlists.items(): - blam_permutation = BlamSoundPermutation( - sample_rate=sample_rate, encoding=encoding) - sound_bank.pitch_ranges[pr_name].permutations[name] = blam_permutation - - for perm in permlist: - sample_data = perm.samples.data - if perm.compression.enum_name == "ogg": - # not actually a sample count. fix struct field name - compression = constants.COMPRESSION_OGG - sample_count = perm.ogg_sample_count // 2 - elif perm.compression.enum_name == "none": - compression = (constants.COMPRESSION_PCM_16_BE - if pcm_is_big_endian else - constants.COMPRESSION_PCM_16_LE) - sample_count = len(sample_data) // 2 - elif perm.compression.enum_name == "xbox_adpcm": - compression = constants.COMPRESSION_XBOX_ADPCM - sample_count = ( - (constants.XBOX_ADPCM_DECOMPRESSED_BLOCKSIZE // 2) * - (len(sample_data) // constants.XBOX_ADPCM_COMPRESSED_BLOCKSIZE)) - elif perm.compression.enum_name == "ima_adpcm": - compression = constants.COMPRESSION_IMA_ADPCM - sample_count = ( - (constants.IMA_ADPCM_DECOMPRESSED_BLOCKSIZE // 2) * - (len(sample_data) // constants.IMA_ADPCM_COMPRESSED_BLOCKSIZE)) - else: - print("Unknown audio compression type:", perm.compression.data) - continue - - sample_count = sample_count // channels - blam_permutation.processed_samples.append( - BlamSoundSamples( - sample_data, sample_count, compression, - int(round(sample_rate * playback_speed)), - encoding, perm.mouth_data.data) - ) + blam_permutation = tag_perm_chain_to_blam_perm( + permlist, sample_rate, tagdata.encoding.data, + pcm_is_big_endian + ) + if blam_permutation: + sound_bank.pitch_ranges[pr_name].permutations[name] = blam_permutation if do_write_wav: sound_bank.export_to_directory( @@ -143,8 +170,6 @@ def get_sound_name(import_names, index): def extract_h2_sounds(tagdata, tag_path, **kw): - # TODO: Make this multiply the sample rate by the natural pitch - halo_map = kw.get('halo_map') if not halo_map: print("Cannot run this function on tags.") diff --git a/reclaimer/sounds/util.py b/reclaimer/sounds/util.py index 378729cd..d350e66f 100644 --- a/reclaimer/sounds/util.py +++ b/reclaimer/sounds/util.py @@ -8,12 +8,12 @@ # import array -import audioop import re import struct import sys +from types import MethodType -from reclaimer.sounds import constants +from reclaimer.sounds import audioop, constants BAD_PATH_CHAR_REMOVAL = re.compile(r'[<>:"|?*]{1, }') @@ -72,13 +72,6 @@ def get_sample_count(sample_data, compression, encoding): return chunk_count -def byteswap_pcm16_sample_data(samples): - return convert_pcm_to_pcm( - samples, - constants.COMPRESSION_PCM_16_LE, - constants.COMPRESSION_PCM_16_BE) - - def is_big_endian_pcm(compression): ''' Returns True if the endianness of the compression modes are different. @@ -144,11 +137,11 @@ def convert_pcm_to_pcm(samples, compression, target_compression, samples = samples[: len(samples) - (len(samples) % current_width)] if compression == constants.COMPRESSION_PCM_8_UNSIGNED: - # bias by 128 to shift unsigned into signed + # convert unsigned into signed samples = audioop.bias(samples, 1, 128) elif current_width > 1 and compression not in constants.NATIVE_ENDIANNESS_FORMATS: # byteswap samples to system endianness before processing - samples = audioop.byteswap(samples, current_width) + samples = audioop.byteswap(samples, target_width) compression = change_pcm_endianness(compression) if current_width != target_width: @@ -177,7 +170,7 @@ def convert_pcm_to_pcm(samples, compression, target_compression, sample_rate = target_sample_rate if target_compression == constants.COMPRESSION_PCM_8_UNSIGNED: - # bias by 128 to shift signed back into unsigned + # convert signed back into unsigned samples = audioop.bias(samples, 1, 128) elif target_width > 1 and (is_big_endian_pcm(compression) != is_big_endian_pcm(target_compression)): @@ -187,17 +180,70 @@ def convert_pcm_to_pcm(samples, compression, target_compression, return samples -def convert_pcm_float32_to_pcm_32(sample_data): - samples = array.array('f', sample_data) +def convert_pcm_float32_to_pcm_int(sample_data, width, wantarray=False): + typecode = audioop.SAMPLE_TYPECODES[width-1] + float_samples = array.array('f', sample_data) + if sys.byteorder == "big" and not isinstance(sample_data, array.array): + # data is expected to be passed in as + # little-endian unless it's an array + float_samples.byteswap() + + maxval = (1 << (8*width-1)) - 1 + minval = -(maxval + 1) + + out_data = array.array(typecode, + map(MethodType(max, minval), + map(MethodType(min, maxval), + map(round, + map(float(maxval+1).__rmul__, float_samples) + )))) + + if sys.byteorder == "big" and width > 1: + # return is expected to be little-endian + out_data.byteswap() + + return out_data if wantarray else out_data.tobytes() + + +def convert_pcm_int_to_pcm_float32(sample_data, width, wantarray=False): + typecode = audioop.SAMPLE_TYPECODES[width-1] + int_samples = array.array(typecode, sample_data) if sys.byteorder == "big": samples.byteswap() - samples = [-0x7fFFffFF if val <= -1.0 else - (0x7fFFffFF if val >= 1.0 else - int(val * 0x7fFFffFF)) - for val in samples] + scale = 1/(1 << (8*width-1)) + + out_data = array.array("f", + map(MethodType(max, -1.0), + map(MethodType(min, 1.0), + map(scale.__rmul__, int_samples) + ))) - return struct.pack("<%di" % len(samples), *samples) + if sys.byteorder == "big" and width > 1: + # return is expected to be little-endian + out_data.byteswap() + + return out_data if wantarray else out_data.tobytes() + + +def deinterleave_stereo(fragment, width, wantarray=False): + if width not in (1, 2, 4): + raise NotImplementedError( + "Cannot deinterleave %s-byte width samples." % width + ) + + typecode = audioop.SAMPLE_TYPECODES[width-1] + if not(isinstance(fragment, array.array) and + fragment.typecode == typecode): + fragment = array.array(typecode, fragment) + + left_channel_data = array.array(typecode, fragment[0::2]) + right_channel_data = array.array(typecode, fragment[1::2]) + + return ( + left_channel_data if wantarray else left_channel_data.tobytes(), + right_channel_data if wantarray else right_channel_data.tobytes() + ) def generate_mouth_data(sample_data, compression, sample_rate, encoding): @@ -207,18 +253,15 @@ def generate_mouth_data(sample_data, compression, sample_rate, encoding): sample_width = constants.sample_widths[compression] channel_count = constants.channel_counts[encoding] - if compression == constants.COMPRESSION_PCM_8_UNSIGNED: - # bias by 128 to shift unsigned into signed sample_data = audioop.bias(sample_data, 1, 128) elif sample_width > 1 and compression not in constants.NATIVE_ENDIANNESS_FORMATS: - # byteswap samples to system endianness before processing sample_data = audioop.byteswap(sample_data, sample_width) - if sample_width == 2: - sample_data = memoryview(sample_data).cast("h") - elif sample_width == 4: - sample_data = memoryview(sample_data).cast("i") + if sample_width != 1: + sample_data = memoryview(sample_data).cast( + "h" if sample_width == 2 else "i" + ) # mouth data is sampled at 30Hz, so we divide the audio # sample_rate by that to determine how many samples we must @@ -269,4 +312,4 @@ def generate_mouth_data(sample_data, compression, sample_rate, encoding): else: mouth_data[i] = int(255 * mouth_sample) - return bytes(mouth_data) + return bytes(mouth_data) \ No newline at end of file diff --git a/reclaimer/stubbs/common_descs.py b/reclaimer/stubbs/common_descs.py index 958dd631..0cb5343a 100644 --- a/reclaimer/stubbs/common_descs.py +++ b/reclaimer/stubbs/common_descs.py @@ -9,6 +9,7 @@ from reclaimer.common_descs import * +# TODO: move shared enumerators into separate enums.py module # ########################################################################### # The order of element in all the enumerators is important(DONT SHUFFLE THEM) # ########################################################################### @@ -121,11 +122,22 @@ "unknown5", ) +# TODO: update these WHEN the new stubbs tag types are found to +# be used in scripts, or if there are new builtin functions. +# there are definitely new object types inserted, as the player +# script types are set to return weapons now instead of units. +# NOTE: we're re-defining these here simply as a placeholder +script_types = tuple(script_types) +script_object_types = tuple(script_object_types) + + damage_modifiers = QStruct("damage_modifiers", - *(float_zero_to_inf(material_name) for material_name in materials_list) + *(float_zero_to_inf(material_name) for material_name in materials_list), + # NOTE: there's enough allocated for 40 materials. We're assuming + # the rest of the space is all for these damage modifiers + SIZE=4*40 ) - def tag_class_stubbs(*args, **kwargs): ''' A macro for creating a tag_class enum desc with the @@ -144,12 +156,12 @@ def dependency_stubbs(name='tag_ref', valid_ids=None, **kwargs): elif valid_ids is None: valid_ids = stubbs_valid_tags - return TagRef(name, + return desc_variant(tag_ref_struct, valid_ids, - INCLUDE=tag_ref_struct, STEPTREE=StrTagRef( - "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=234), - **kwargs + "filepath", SIZE=tag_ref_str_size, GUI_NAME="", MAX=254 + ), + NAME=name, **kwargs ) diff --git a/reclaimer/stubbs/defs/actr.py b/reclaimer/stubbs/defs/actr.py index be405d26..54ab8a23 100644 --- a/reclaimer/stubbs/defs/actr.py +++ b/reclaimer/stubbs/defs/actr.py @@ -10,20 +10,8 @@ from ...hek.defs.actr import * from ..common_descs import * -actr_body = dict(actr_body) -actr_body[3] = SEnum16("type", *actor_types) -actr_body[12] = dict(actr_body[12]) -actr_body[12][2] = SEnum16("leader_type", *actor_types) - -actr_body[14] = dict(actr_body[14]) -actr_body[14][6] = SEnum16("defensive_crouch_type", - "never", - "danger", - "low_shields", - "hide_behind_shield", - "any_target", - ) - +panic = desc_variant(panic, SEnum16("leader_type", *actor_types)) +actr_body = desc_variant(actr_body, SEnum16("type", *actor_types), panic) def get(): return actr_def diff --git a/reclaimer/stubbs/defs/antr.py b/reclaimer/stubbs/defs/antr.py index deb369cd..01e24765 100644 --- a/reclaimer/stubbs/defs/antr.py +++ b/reclaimer/stubbs/defs/antr.py @@ -10,7 +10,7 @@ from ...hek.defs.antr import * from ..common_descs import * -# As meta, it seems MOST arrays of anim_enum_desc have an extra extra on the end +# As meta, it seems MOST arrays of anim_enum_desc have an extra enum on the end animations_extended_desc = Struct("weapon_types", Pad(16), @@ -18,6 +18,8 @@ SIZE=28, ) +# NOTE: this requires further investigation. It doesn't seem like +# they actually moved the padding to before the yaw_per_frame. unit_weapon_desc = Struct("weapon", ascii_str32("name"), ascii_str32("grip_marker"), @@ -56,27 +58,10 @@ SIZE=64 ) -unit_desc = Struct("unit", - ascii_str32("label"), - #pitch and yaw are saved in radians. - - #Looking screen bounds - float_rad("right_yaw_per_frame"), - float_rad("left_yaw_per_frame"), - SInt16("right_frame_count"), - SInt16("left_frame_count"), - - float_rad("down_pitch_per_frame"), - float_rad("up_pitch_per_frame"), - SInt16("down_frame_count"), - SInt16("up_frame_count"), - - Pad(8), - reflexive("animations", anim_enum_desc), - reflexive("ik_points", ik_point_desc, 4, DYN_NAME_PATH=".marker"), - reflexive("weapons", unit_weapon_desc, DYN_NAME_PATH=".name"), - reflexive("unknown", unknown_unit_desc), +unit_desc = desc_variant(unit_desc, + ("pad_13", reflexive("unknown", unknown_unit_desc)), SIZE=128, + verify=False ) seat_desc = Struct("seat", @@ -86,27 +71,8 @@ SIZE=60 ) -vehicle_desc = Struct("vehicle", - #pitch and yaw are saved in radians. - - #Steering screen bounds - float_rad("right_yaw_per_frame"), - float_rad("left_yaw_per_frame"), - SInt16("right_frame_count"), - SInt16("left_frame_count"), - - float_rad("down_pitch_per_frame"), - float_rad("up_pitch_per_frame"), - SInt16("down_frame_count"), - SInt16("up_frame_count"), - - Pad(56), - reflexive("seats", seat_desc), - reflexive("animations", anim_enum_desc, 8, - 'steering','roll','throttle','velocity', - 'braking','ground-speed','occupied','unoccupied'), - reflexive("suspension_animations", suspension_desc, 8), - SIZE=116, +vehicle_desc = desc_variant(vehicle_desc, + ("pad_9", reflexive("seats", seat_desc)), ) effect_reference_desc = Struct("effect_reference", @@ -114,80 +80,25 @@ SIZE=20, ) -animation_desc = Struct("animation", - ascii_str32("name"), - SEnum16("type", *anim_types), - SInt16("frame_count"), - SInt16("frame_size"), - SEnum16("frame_info_type", *anim_frame_info_types), - SInt32("node_list_checksum"), - SInt16("node_count"), - SInt16("loop_frame_index"), - - Float("weight"), - SInt16("key_frame_index"), - SInt16("second_key_frame_index"), - Pad(8), - - dyn_senum16("next_animation", DYN_NAME_PATH="..[DYN_I].name"), - Bool16("flags", - "compressed_data", - "world_relative", - {NAME:"pal", GUI_NAME:"25Hz(PAL)"}, - ), - dyn_senum16("sound", +animation_desc = desc_variant(animation_desc, + ("pad_11", Pad(8)), + ("sound", dyn_senum16("effect", DYN_NAME_PATH="tagdata.effect_references." + - "effect_references_array[DYN_I].effect.filepath"), - SInt16("sound_frame_index"), - SInt8("left_foot_frame_index"), - SInt8("right_foot_frame_index"), - FlSInt16("first_permutation_index", VISIBLE=False, - TOOLTIP="The index of the first animation in the permutation chain."), - FlFloat("chance_to_play", VISIBLE=False, - MIN=0.0, MAX=1.0, SIDETIP="[0,1]", - TOOLTIP=("Seems to be the chance range to select this permutation.\n" - "Random number in the range [0,1] is rolled. The permutation\n" - "chain is looped until the number is higher than or equal\n" - "to that permutations chance to play. This chance to play\n" - "is likely influenced by the animations 'weight' field.\n" - "All permutation chains should have the last one end with\n" - "a chance to play of 1.0")), - - rawdata_ref("frame_info", max_size=32768), - UInt32("trans_flags0", EDITABLE=False), - UInt32("trans_flags1", EDITABLE=False), - Pad(8), - UInt32("rot_flags0", EDITABLE=False), - UInt32("rot_flags1", EDITABLE=False), - Pad(8), - UInt32("scale_flags0", EDITABLE=False), - UInt32("scale_flags1", EDITABLE=False), - Pad(4), - SInt32("offset_to_compressed_data", EDITABLE=False), - rawdata_ref("default_data", max_size=16384), - rawdata_ref("frame_data", max_size=1048576), + "effect_references_array[DYN_I].effect.filepath" + )), SIZE=188, + verify=False ) -antr_body = Struct("tagdata", - reflexive("objects", object_desc), +antr_body = desc_variant(antr_body, reflexive("units", unit_desc, DYN_NAME_PATH=".label"), reflexive("weapons", weapon_desc), reflexive("vehicles", vehicle_desc), - reflexive("devices", device_desc), reflexive("unit_damages", anim_enum_desc), - reflexive("fp_animations", fp_animation_desc), - - reflexive("effect_references", effect_reference_desc, - DYN_NAME_PATH=".effect.filepath"), - Float("limp_body_node_radius"), - Bool16("flags", - "compress_all_animations", - "force_idle_compression", + ("sound_references", reflexive("effect_references", + effect_reference_desc, DYN_NAME_PATH=".effect.filepath") ), - Pad(2), - reflexive("nodes", nodes_desc, DYN_NAME_PATH=".name"), - reflexive("animations", animation_desc, DYN_NAME_PATH=".name"), + reflexive("animations", animation_desc, DYN_NAME_PATH=".name", EXT_MAX=2048), SIZE=128, ) diff --git a/reclaimer/stubbs/defs/bipd.py b/reclaimer/stubbs/defs/bipd.py index 79b69b35..9a342c0d 100644 --- a/reclaimer/stubbs/defs/bipd.py +++ b/reclaimer/stubbs/defs/bipd.py @@ -14,20 +14,16 @@ from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=0) - -bipd_body = Struct("tagdata", +obje_attrs = obje_attrs_variant(obje_attrs, "bipd") +bipd_body = Struct("tagdata", obje_attrs, unit_attrs, bipd_attrs, SIZE=1268, ) -#def get(): -# return bipd_def +def get(): + return bipd_def del get bipd_def = TagDef("bipd", diff --git a/reclaimer/stubbs/defs/cdmg.py b/reclaimer/stubbs/defs/cdmg.py index 9383d5c3..4ed84e9a 100644 --- a/reclaimer/stubbs/defs/cdmg.py +++ b/reclaimer/stubbs/defs/cdmg.py @@ -10,27 +10,8 @@ from ...hek.defs.cdmg import * from ..common_descs import * -cdmg_body = dict(cdmg_body) -cdmg_body[5] = dict(cdmg_body[5]) -cdmg_body[6] = damage_modifiers -cdmg_body[5][1] = SEnum16("category", *damage_category) -cdmg_body[5][2] = Bool32("flags", - "does_not_hurt_owner", - {NAME: "headshot", GUI_NAME: "causes headshots"}, - "pings_resistant_units", - "does_not_hurt_friends", - "does_not_ping_shields", - "detonates_explosives", - "only_hurts_shields", - "causes_flaming_death", - {NAME: "indicator_points_down", - GUI_NAME: "damage indicator always points down"}, - "skips_shields", - "only_hurts_one_infection_form", - {NAME: "multiplayer_headshot", - GUI_NAME: "causes multiplayer headshots"}, - "infection_form_pop", - ) +damage = desc_variant(damage, SEnum16("category", *damage_category)) +cdmg_body = desc_variant(cdmg_body, damage, damage_modifiers) def get(): return cdmg_def diff --git a/reclaimer/stubbs/defs/coll.py b/reclaimer/stubbs/defs/coll.py index 11be60b3..4e666bfa 100644 --- a/reclaimer/stubbs/defs/coll.py +++ b/reclaimer/stubbs/defs/coll.py @@ -10,8 +10,7 @@ from ...hek.defs.coll import * from ..common_descs import * -shield = dict(shield) -shield[2] = SEnum16("shield_material_type", *materials_list) +shield = desc_variant(shield, SEnum16("shield_material_type", *materials_list)) permutation = Struct("permutation", ascii_str32("name"), @@ -60,43 +59,14 @@ SIZE=144 ) -coll_body = Struct("tagdata", - Bool32("flags", - "takes_shield_damage_for_children", - "takes_body_damage_for_children", - "always_shields_friendly_damage", - "passes_area_damage_to_children", - "parent_never_takes_body_damage_for_us", - "only_damaged_by_explosives", - "only_damaged_while_occupied", - ), - dyn_senum16("indirect_damage_material", - DYN_NAME_PATH=".materials.materials_array[DYN_I].name"), - Pad(2), - - body, +coll_body = desc_variant(coll_body, shield, - - Pad(112), reflexive("materials", material, 32, DYN_NAME_PATH='.name'), reflexive("regions", region, 8, DYN_NAME_PATH='.name'), - reflexive("modifiers", modifier, 0, VISIBLE=False), - - Pad(16), - Struct("pathfinding_box", - QStruct("x", INCLUDE=from_to), - QStruct("y", INCLUDE=from_to), - QStruct("z", INCLUDE=from_to), - ), - - reflexive("pathfinding_spheres", pathfinding_sphere, 32), - reflexive("nodes", node, 64, DYN_NAME_PATH='.name'), - - SIZE=664, ) - -fast_coll_body = dict(coll_body) -fast_coll_body[12] = reflexive("nodes", fast_node, 64, DYN_NAME_PATH='.name') +fast_coll_body = desc_variant(coll_body, + reflexive("nodes", fast_node, 64, DYN_NAME_PATH='.name'), + ) def get(): diff --git a/reclaimer/stubbs/defs/ctrl.py b/reclaimer/stubbs/defs/ctrl.py index 3c607e20..09288c39 100644 --- a/reclaimer/stubbs/defs/ctrl.py +++ b/reclaimer/stubbs/defs/ctrl.py @@ -11,10 +11,7 @@ from .obje import * from .devi import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=8) +obje_attrs = obje_attrs_variant(obje_attrs, "ctrl") ctrl_body = Struct("tagdata", obje_attrs, diff --git a/reclaimer/stubbs/defs/eqip.py b/reclaimer/stubbs/defs/eqip.py index 61c4a9da..941121fe 100644 --- a/reclaimer/stubbs/defs/eqip.py +++ b/reclaimer/stubbs/defs/eqip.py @@ -8,22 +8,16 @@ # from ...hek.defs.eqip import * +from .item import * from .obje import * -from ..common_descs import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=3) - -eqip_attrs = dict(eqip_attrs) -eqip_attrs[1] = SEnum16('grenade_type', *grenade_types) +obje_attrs = obje_attrs_variant(obje_attrs, "eqip") +eqip_attrs = desc_variant(eqip_attrs, SEnum16('grenade_type', *grenade_types)) eqip_body = Struct("tagdata", obje_attrs, item_attrs, eqip_attrs, - SIZE=944, ) diff --git a/reclaimer/stubbs/defs/foot.py b/reclaimer/stubbs/defs/foot.py index b7857977..7f07b01b 100644 --- a/reclaimer/stubbs/defs/foot.py +++ b/reclaimer/stubbs/defs/foot.py @@ -31,7 +31,6 @@ ) - def get(): return foot_def diff --git a/reclaimer/stubbs/defs/garb.py b/reclaimer/stubbs/defs/garb.py index a8fc3994..28a994b7 100644 --- a/reclaimer/stubbs/defs/garb.py +++ b/reclaimer/stubbs/defs/garb.py @@ -8,18 +8,10 @@ # from ...hek.defs.garb import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=4) - -garb_body = dict(garb_body) -garb_body[0] = obje_attrs - +obje_attrs = obje_attrs_variant(obje_attrs, "garb") +garb_body = desc_variant(garb_body, obje_attrs) def get(): return garb_def diff --git a/reclaimer/stubbs/defs/jpt_.py b/reclaimer/stubbs/defs/jpt_.py index 5f758e17..425e78af 100644 --- a/reclaimer/stubbs/defs/jpt_.py +++ b/reclaimer/stubbs/defs/jpt_.py @@ -10,28 +10,8 @@ from ...hek.defs.jpt_ import * from ..common_descs import * -jpt__body = dict(jpt__body) -jpt__body[16] = dict(jpt__body[16]) -jpt__body[17] = damage_modifiers - -jpt__body[16][1] = SEnum16("category", *damage_category) -jpt__body[16][2] = Bool32("flags", - "does_not_hurt_owner", - {NAME: "headshot", GUI_NAME: "causes headshots"}, - "pings_resistant_units", - "does_not_hurt_friends", - "does_not_ping_units", - "detonates_explosives", - "only_hurts_shields", - "causes_flaming_death", - {NAME: "indicator_points_down", - GUI_NAME: "damage indicators always points down"}, - "skips_shields", - "only_hurts_one_infection_form", - {NAME: "multiplayer_headshot", - GUI_NAME: "causes multiplayer headshots"}, - "infection_form_pop", - ) +damage = desc_variant(damage, SEnum16("category", *damage_category)) +jpt__body = desc_variant(jpt__body, damage, damage_modifiers) def get(): diff --git a/reclaimer/stubbs/defs/lifi.py b/reclaimer/stubbs/defs/lifi.py index 8250a3f2..516b1618 100644 --- a/reclaimer/stubbs/defs/lifi.py +++ b/reclaimer/stubbs/defs/lifi.py @@ -7,21 +7,12 @@ # See LICENSE for more information. # +from ...hek.defs.lifi import * from .obje import * from .devi import * -from ...hek.defs.lifi import * - -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=9) -lifi_body = Struct("tagdata", - obje_attrs, - devi_attrs, - - SIZE=720, - ) +obje_attrs = obje_attrs_variant(obje_attrs, "lifi") +lifi_body = desc_variant(lifi_body, obje_attrs) def get(): return lifi_def diff --git a/reclaimer/stubbs/defs/mach.py b/reclaimer/stubbs/defs/mach.py index ecf73697..1c33d92c 100644 --- a/reclaimer/stubbs/defs/mach.py +++ b/reclaimer/stubbs/defs/mach.py @@ -10,20 +10,9 @@ from ...hek.defs.mach import * from .obje import * from .devi import * -from supyr_struct.defs.tag_def import TagDef -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=7) - -mach_body = Struct("tagdata", - obje_attrs, - devi_attrs, - mach_attrs, - - SIZE=804, - ) +obje_attrs = obje_attrs_variant(obje_attrs, "mach") +mach_body = desc_variant(mach_body, obje_attrs, devi_attrs) def get(): return mach_def diff --git a/reclaimer/stubbs/defs/matg.py b/reclaimer/stubbs/defs/matg.py index 74d920c6..be5daed8 100644 --- a/reclaimer/stubbs/defs/matg.py +++ b/reclaimer/stubbs/defs/matg.py @@ -14,29 +14,9 @@ def get(): return matg_def -matg_body = Struct('tagdata', - Pad(248), - reflexive("sounds", sound, 2, - "enter water", "exit water"), - reflexive("cameras", camera, 1), - reflexive("player_controls", player_control, 1), - reflexive("difficulties", difficulty, 1), +matg_body = desc_variant(matg_body, reflexive("grenades", grenade, len(grenade_types), *grenade_types), - reflexive("rasterizer_datas", rasterizer_data, 1), - reflexive("interface_bitmaps", interface_bitmaps, 1), - reflexive("cheat_weapons", cheat_weapon, 20, - DYN_NAME_PATH='.weapon.filepath'), - reflexive("cheat_powerups", cheat_powerup, 20, - DYN_NAME_PATH='.powerup.filepath'), - reflexive("multiplayer_informations", multiplayer_information, 1), - reflexive("player_informations", player_information, 1), - reflexive("first_person_interfaces", first_person_interface, 1), - reflexive("falling_damages", falling_damage, 1), reflexive("materials", material, len(materials_list), *materials_list), - reflexive("playlist_members", playlist_member, 20, - DYN_NAME_PATH='.map_name'), - - SIZE=428 ) matg_def = TagDef("matg", diff --git a/reclaimer/stubbs/defs/mode.py b/reclaimer/stubbs/defs/mode.py index cbf65558..7104e66d 100644 --- a/reclaimer/stubbs/defs/mode.py +++ b/reclaimer/stubbs/defs/mode.py @@ -16,8 +16,8 @@ def get(): # my guess at what the struct might be like unknown_struct = Struct("unknown", ascii_str32("name"), - LFloat('unknown1', ENDIAN='<', DEFAULT=1.0), - LFloat('unknown2', ENDIAN='<', DEFAULT=1.0), + Float('unknown1', DEFAULT=1.0), + Float('unknown2', DEFAULT=1.0), BytesRaw("unknown_data", SIZE=64-32-4*2), SIZE=64 ) @@ -27,134 +27,49 @@ def get(): # SIZE=64 # ) -pc_part = Struct('part', - Bool32('flags', - 'stripped', - ), - dyn_senum16('shader_index', - DYN_NAME_PATH="tagdata.shaders.shaders_array[DYN_I].shader.filepath"), - SInt8('previous_part_index'), - SInt8('next_part_index'), - - SInt16('centroid_primary_node'), - SInt16('centroid_secondary_node'), - Float('centroid_primary_weight'), - Float('centroid_secondary_weight'), - - QStruct('centroid_translation', INCLUDE=xyz_float), - - #reflexive("uncompressed_vertices", uncompressed_vertex_union, 65535), - #reflexive("compressed_vertices", compressed_vertex_union, 65535), - #reflexive("triangles", triangle_union, 65535), - reflexive("uncompressed_vertices", fast_uncompressed_vertex, 65535), - reflexive("compressed_vertices", fast_compressed_vertex, 65535), - reflexive("triangles", triangle, 65535), - - #Pad(36), - Struct("model_meta_info", - # the offset fields in model_meta_info struct are the only - # thing different from halo model tags. if they weren't, - # this whole new part definition wouldn't be necessary. - UEnum16("index_type", # name is a guess. always 1? - ("uncompressed", 1), - ), - Pad(2), - UInt32("index_count"), - # THESE VALUES ARE DIFFERENT THAN ON XBOX IT SEEMS - UInt32("indices_magic_offset"), - UInt32("indices_offset"), - - UEnum16("vertex_type", # name is a guess - ("uncompressed", 4), - ("compressed", 5), - ), - Pad(2), - UInt32("vertex_count"), - Pad(4), # always 0? - # THESE VALUES ARE DIFFERENT THAN ON XBOX IT SEEMS - UInt32("vertices_magic_offset"), - UInt32("vertices_offset"), - VISIBLE=False, SIZE=36 - ), - - SIZE=104 +pc_part = desc_variant(part, model_meta_info) +fast_part = desc_variant(part, + raw_reflexive("uncompressed_vertices", fast_uncompressed_vertex, SINT16_MAX), + raw_reflexive("compressed_vertices", fast_compressed_vertex, SINT16_MAX), + raw_reflexive("triangles", triangle, SINT16_MAX), ) +fast_pc_part = desc_variant(fast_part, model_meta_info) -fast_part = dict(part) -fast_part[9] = raw_reflexive("uncompressed_vertices", fast_uncompressed_vertex) -fast_part[10] = raw_reflexive("compressed_vertices", fast_compressed_vertex) -fast_part[11] = raw_reflexive("triangles", triangle) +pc_geometry = desc_variant(geometry, reflexive("parts", pc_part, 32, EXT_MAX=SINT16_MAX)) +fast_geometry = desc_variant(geometry, reflexive("parts", fast_part, 32, EXT_MAX=SINT16_MAX)) +fast_pc_geometry = desc_variant(geometry, reflexive("parts", fast_pc_part, 32, EXT_MAX=SINT16_MAX)) -pc_geometry = Struct('geometry', - Pad(36), - reflexive("parts", pc_part, 32), - SIZE=48 +mode_body = desc_variant(mode_body, + ("pad_16", reflexive("unknown", unknown_struct, DYN_NAME_PATH=".name")), ) +pc_mode_body = desc_variant(mode_body, reflexive("geometries", pc_geometry, 256, EXT_MAX=SINT16_MAX)) +fast_mode_body = desc_variant(mode_body, reflexive("geometries", fast_geometry, 256, EXT_MAX=SINT16_MAX)) +fast_pc_mode_body = desc_variant(mode_body, reflexive("geometries", fast_pc_geometry, 256, EXT_MAX=SINT16_MAX)) -fast_geometry = Struct('geometry', - Pad(36), - reflexive("parts", fast_part, 32), - SIZE=48 - ) - -mode_body = Struct('tagdata', - Bool32('flags', - 'blend_shared_normals', - ), - SInt32('node_list_checksum'), - - Float('superhigh_lod_cutoff', SIDETIP="pixels"), - Float('high_lod_cutoff', SIDETIP="pixels"), - Float('medium_lod_cutoff', SIDETIP="pixels"), - Float('low_lod_cutoff', SIDETIP="pixels"), - Float('superlow_lod_cutoff', SIDETIP="pixels"), - - SInt16('superhigh_lod_nodes', SIDETIP="nodes"), - SInt16('high_lod_nodes', SIDETIP="nodes"), - SInt16('medium_lod_nodes', SIDETIP="nodes"), - SInt16('low_lod_nodes', SIDETIP="nodes"), - SInt16('superlow_lod_nodes', SIDETIP="nodes"), - - Pad(10), - - Float('base_map_u_scale'), - Float('base_map_v_scale'), - - Pad(104), - reflexive("unknown", unknown_struct, DYN_NAME_PATH=".name"), - - reflexive("markers", marker, 256, DYN_NAME_PATH=".name"), - reflexive("nodes", node, 64, DYN_NAME_PATH=".name"), - reflexive("regions", region, 32, DYN_NAME_PATH=".name"), - reflexive("geometries", geometry, 256), - reflexive("shaders", shader, 256, DYN_NAME_PATH=".shader.filepath"), - - SIZE=232 - ) - -pc_mode_body = dict(mode_body) -pc_mode_body[20] = reflexive("geometries", pc_geometry, 256) - -fast_mode_body = dict(mode_body) -fast_mode_body[20] = reflexive("geometries", fast_geometry, 256) +# increment version to differentiate from halo models +stubbs_mode_header = blam_header_stubbs('mode', 6) +stubbs_mode_kwargs = dict(ext=".model", endian=">", tag_cls=ModeTag) mode_def = TagDef("mode", - blam_header_stubbs('mode', 6), # increment to differentiate it from mode and mod2 - mode_body, - - ext=".model", endian=">", tag_cls=ModeTag + stubbs_mode_header, + mode_body, + **stubbs_mode_kwargs ) fast_mode_def = TagDef("mode", - blam_header_stubbs('mode', 6), # increment to differentiate it from mode and mod2 - fast_mode_body, - - ext=".model", endian=">", tag_cls=ModeTag + stubbs_mode_header, + fast_mode_body, + **stubbs_mode_kwargs ) pc_mode_def = TagDef("mode", - blam_header_stubbs('mode', 6), # increment to differentiate it from mode and mod2 + stubbs_mode_header, pc_mode_body, + **stubbs_mode_kwargs + ) - ext=".model", endian=">", tag_cls=ModeTag +fast_pc_mode_def = TagDef("mode", + stubbs_mode_header, + fast_pc_mode_body, + **stubbs_mode_kwargs ) diff --git a/reclaimer/stubbs/defs/obje.py b/reclaimer/stubbs/defs/obje.py index 22b15007..b46386d0 100644 --- a/reclaimer/stubbs/defs/obje.py +++ b/reclaimer/stubbs/defs/obje.py @@ -13,8 +13,7 @@ def get(): return obje_def -obje_attrs = dict(obje_attrs) -obje_attrs[7] = dependency_stubbs('model', 'mode') +obje_attrs = desc_variant(obje_attrs, dependency_stubbs('model', 'mode')) obje_body = Struct('tagdata', obje_attrs, @@ -27,6 +26,3 @@ def get(): ext=".object", endian=">", tag_cls=ObjeTag ) - -def get(): - return obje_def diff --git a/reclaimer/stubbs/defs/objs/scnr.py b/reclaimer/stubbs/defs/objs/scnr.py new file mode 100644 index 00000000..87db1491 --- /dev/null +++ b/reclaimer/stubbs/defs/objs/scnr.py @@ -0,0 +1,13 @@ +# +# This file is part of Reclaimer. +# +# For authors and copyright check AUTHORS.TXT +# +# Reclaimer is free software under the GNU General Public License v3.0. +# See LICENSE for more information. +# + +from reclaimer.hek.defs.objs.scnr import ScnrTag + +class StubbsScnrTag(ScnrTag): + engine = "stubbs" diff --git a/reclaimer/stubbs/defs/plac.py b/reclaimer/stubbs/defs/plac.py index 46d5b915..79214cb9 100644 --- a/reclaimer/stubbs/defs/plac.py +++ b/reclaimer/stubbs/defs/plac.py @@ -8,17 +8,10 @@ # from ...hek.defs.plac import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=10) - -plac_body = dict(plac_body) -plac_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "plac") +plac_body = desc_variant(plac_body, obje_attrs) def get(): return plac_def diff --git a/reclaimer/stubbs/defs/proj.py b/reclaimer/stubbs/defs/proj.py index 37ce3d6e..98051fd8 100644 --- a/reclaimer/stubbs/defs/proj.py +++ b/reclaimer/stubbs/defs/proj.py @@ -10,16 +10,13 @@ from ...hek.defs.proj import * from ..common_descs import * from .obje import * -from supyr_struct.defs.tag_def import TagDef -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=5) +obje_attrs = obje_attrs_variant(obje_attrs, "proj") -proj_attrs = dict(proj_attrs) -proj_attrs[13] = reflexive("material_responses", material_response, - len(materials_list), *materials_list) +material_responses = reflexive("material_responses", + material_response, len(materials_list), *materials_list + ) +proj_attrs = desc_variant(proj_attrs, material_responses) proj_body = Struct("tagdata", obje_attrs, diff --git a/reclaimer/stubbs/defs/sbsp.py b/reclaimer/stubbs/defs/sbsp.py index 39528dfb..6a657ffb 100644 --- a/reclaimer/stubbs/defs/sbsp.py +++ b/reclaimer/stubbs/defs/sbsp.py @@ -10,49 +10,23 @@ from ...hek.defs.sbsp import * from ..common_descs import * - -cluster = Struct("cluster", - SInt16('sky'), - SInt16('fog'), - dyn_senum16('background_sound', - DYN_NAME_PATH="tagdata.background_sounds_palette.STEPTREE[DYN_I].name"), - dyn_senum16('sound_environment', - DYN_NAME_PATH="tagdata.sound_environments_palette." + - "STEPTREE[DYN_I].name"), - dyn_senum16('weather', - DYN_NAME_PATH="tagdata.weather_palettes.STEPTREE[DYN_I].name"), - - UInt16("transition_structure_bsp", VISIBLE=False), - UInt16("first_decal_index", VISIBLE=False), - UInt16("decal_count", VISIBLE=False), - QStruct("unknown0", - Float('float_0'), - Float('float_1'), - Float('float_2'), - Float('float_3'), - Float('float_4'), - Float('float_5'), - SIZE=24 - ), - - reflexive("predicted_resources", predicted_resource, 1024), - reflexive("subclusters", subcluster, 4096), - SInt16("first_lens_flare_marker_index"), - SInt16("lens_flare_marker_count"), - - # stubbs seems to have different data here than surface indices - Pad(12), - reflexive("mirrors", mirror, 16, DYN_NAME_PATH=".shader.filepath"), - reflexive("portals", portal, 128), - SIZE=104 +cluster_unknown = QStruct("unknown", + Float('float_0'), + Float('float_1'), + Float('float_2'), + Float('float_3'), + Float('float_4'), + Float('float_5'), + SIZE=24 ) +surface_indices = desc_variant(reflexive_struct, NAME="surface_indices") + +cluster = desc_variant(cluster, ("pad_8", cluster_unknown), surface_indices) +clusters = reflexive("clusters", cluster, 8192) -sbsp_body = dict(sbsp_body) -sbsp_body[28] = reflexive("clusters", cluster, 8192) - -fast_sbsp_body = dict(fast_sbsp_body) -fast_sbsp_body[28] = reflexive("clusters", cluster, 8192) +sbsp_body = desc_variant(sbsp_body, clusters) +fast_sbsp_body = desc_variant(fast_sbsp_body, clusters) def get(): diff --git a/reclaimer/stubbs/defs/scen.py b/reclaimer/stubbs/defs/scen.py index e545302d..4c508784 100644 --- a/reclaimer/stubbs/defs/scen.py +++ b/reclaimer/stubbs/defs/scen.py @@ -8,17 +8,10 @@ # from ...hek.defs.scen import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=6) - -scen_body = dict(scen_body) -scen_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "scen") +scen_body = desc_variant(scen_body, obje_attrs) def get(): return scen_def diff --git a/reclaimer/stubbs/defs/scnr.py b/reclaimer/stubbs/defs/scnr.py index 5ccac64f..e2d2d19d 100644 --- a/reclaimer/stubbs/defs/scnr.py +++ b/reclaimer/stubbs/defs/scnr.py @@ -8,3 +8,22 @@ # from ...hek.defs.scnr import * +from ..common_descs import * +from .objs.scnr import StubbsScnrTag + +reference = desc_variant(reference, dependency_stubbs("reference")) + +scnr_body = desc_variant(scnr_body, + reflexive("references", reference, 256, DYN_NAME_PATH='.reference.filepath'), + ) + +def get(): + return scnr_def + +# TODO: update dependencies +scnr_def = TagDef("scnr", + blam_header('scnr', 2), + scnr_body, + + ext=".scenario", endian=">", tag_cls=StubbsScnrTag + ) diff --git a/reclaimer/stubbs/defs/shdr.py b/reclaimer/stubbs/defs/shdr.py index fb56c4a3..9350d1c4 100644 --- a/reclaimer/stubbs/defs/shdr.py +++ b/reclaimer/stubbs/defs/shdr.py @@ -9,11 +9,8 @@ from ...hek.defs.shdr import * from ..common_descs import * -from supyr_struct.defs.tag_def import TagDef - -shdr_attrs = dict(shdr_attrs) -shdr_attrs[6] = SEnum16("material_type", *materials_list) +shdr_attrs = desc_variant(shdr_attrs, SEnum16("material_type", *materials_list)) shader_body = Struct("tagdata", shdr_attrs, SIZE=40 diff --git a/reclaimer/stubbs/defs/soso.py b/reclaimer/stubbs/defs/soso.py index 52769fe8..29927f03 100644 --- a/reclaimer/stubbs/defs/soso.py +++ b/reclaimer/stubbs/defs/soso.py @@ -11,40 +11,9 @@ from .shdr import * from supyr_struct.defs.tag_def import TagDef -bump_properties = Struct("bump_properties", - Float("bump_scale"), - dependency_stubbs("bump_map", "bitm"), - ) - -soso_attrs = Struct("soso_attrs", - #Model Shader Properties - model_shader, - - Pad(16), - #Color-Change - SEnum16("color_change_source", *function_names), - - Pad(30), - #Self-Illumination - self_illumination, - - Pad(12), - #Diffuse, Multipurpose, and Detail Maps - maps, - - # this padding is the reflexive for the OS shader model extension - Pad(12), - - #Texture Scrolling Animation - texture_scrolling, - - Pad(8), - #Reflection Properties - reflection, - - Pad(16), - bump_properties, - SIZE=440 +soso_attrs = desc_variant(soso_attrs, + ("pad_11", Float("bump_scale")), + ("pad_12", dependency_stubbs("bump_map", "bitm")), ) soso_body = Struct("tagdata", diff --git a/reclaimer/stubbs/defs/sotr.py b/reclaimer/stubbs/defs/sotr.py index 6b004099..4276c520 100644 --- a/reclaimer/stubbs/defs/sotr.py +++ b/reclaimer/stubbs/defs/sotr.py @@ -14,7 +14,6 @@ sotr_body = Struct("tagdata", shdr_attrs, sotr_attrs, - SIZE=108, ) diff --git a/reclaimer/stubbs/defs/spla.py b/reclaimer/stubbs/defs/spla.py index 11bce9ac..00a7c9db 100644 --- a/reclaimer/stubbs/defs/spla.py +++ b/reclaimer/stubbs/defs/spla.py @@ -17,7 +17,6 @@ SIZE=332, ) - def get(): return spla_def diff --git a/reclaimer/stubbs/defs/ssce.py b/reclaimer/stubbs/defs/ssce.py index 0a640b41..5426a514 100644 --- a/reclaimer/stubbs/defs/ssce.py +++ b/reclaimer/stubbs/defs/ssce.py @@ -8,17 +8,10 @@ # from ...hek.defs.ssce import * - -#import and use the open saucified obje attrs from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=11) - -ssce_body = dict(ssce_body) -ssce_body[0] = obje_attrs +obje_attrs = obje_attrs_variant(obje_attrs, "ssce") +ssce_body = desc_variant(ssce_body, obje_attrs) def get(): return ssce_def diff --git a/reclaimer/stubbs/defs/tagc.py b/reclaimer/stubbs/defs/tagc.py index 6e3f05af..90522e11 100644 --- a/reclaimer/stubbs/defs/tagc.py +++ b/reclaimer/stubbs/defs/tagc.py @@ -8,3 +8,23 @@ # from ...hek.defs.tagc import * +from ..common_descs import dependency_stubbs + +tag_reference = desc_variant(tag_reference, dependency_stubbs("tag")) + +tagc_body = desc_variant(tagc_body, + reflexive("tag_references", tag_reference, 200, + DYN_NAME_PATH='.tag.filepath' + ) + ) + + +def get(): + return tagc_def + +tagc_def = TagDef("tagc", + blam_header('tagc'), + tagc_body, + + ext=".tag_collection", endian=">", tag_cls=HekTag + ) diff --git a/reclaimer/stubbs/defs/unhi.py b/reclaimer/stubbs/defs/unhi.py index e9d79e9b..bcf694cd 100644 --- a/reclaimer/stubbs/defs/unhi.py +++ b/reclaimer/stubbs/defs/unhi.py @@ -8,3 +8,36 @@ # from ...hek.defs.unhi import * +from ..common_descs import dependency_stubbs + + +health_panel_meter = desc_variant(health_panel_meter, + SIZE=148, verify=False + ) + +auxilary_meter = desc_variant(auxilary_meter, + SEnum16("type", + "integrated_light", + "unknown", + VISIBLE=False + ), + ) + +unhi_body = desc_variant(unhi_body, + health_panel_meter, + # yeah, they're swapped around + ("warning_sounds", reflexive("auxilary_meters", auxilary_meter, 16)), + ("auxilary_meters", dependency_stubbs("screen_effect", "imef")), + verify=False + ) + + +def get(): + return unhi_def + +unhi_def = TagDef("unhi", + blam_header("unhi"), + unhi_body, + + ext=".unit_hud_interface", endian=">", tag_cls=HekTag, + ) \ No newline at end of file diff --git a/reclaimer/stubbs/defs/unit.py b/reclaimer/stubbs/defs/unit.py index 8a929067..6ee50fca 100644 --- a/reclaimer/stubbs/defs/unit.py +++ b/reclaimer/stubbs/defs/unit.py @@ -118,7 +118,8 @@ SEnum16('grenade_type', *grenade_types), SInt16('grenade_count', MIN=0), - Pad(4), + FlUInt16("soft_ping_stun_ticks", VISIBLE=False), # set to soft_ping_interrupt_time * 30 + FlUInt16("hard_ping_stun_ticks", VISIBLE=False), # set to hard_ping_interrupt_time * 30 reflexive("powered_seats", powered_seat, 2, "driver", "gunner"), reflexive("weapons", weapon, 4, DYN_NAME_PATH='.weapon.filepath'), diff --git a/reclaimer/stubbs/defs/vehi.py b/reclaimer/stubbs/defs/vehi.py index e782e05b..b23d4602 100644 --- a/reclaimer/stubbs/defs/vehi.py +++ b/reclaimer/stubbs/defs/vehi.py @@ -14,13 +14,8 @@ from .obje import * from .unit import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=1) - -vehi_attrs = dict(vehi_attrs) -vehi_attrs[1] = SEnum16('type', *vehicle_types) +obje_attrs = obje_attrs_variant(obje_attrs, "vehi") +vehi_attrs = desc_variant(vehi_attrs, SEnum16('type', *vehicle_types)) vehi_body = Struct("tagdata", obje_attrs, diff --git a/reclaimer/stubbs/defs/weap.py b/reclaimer/stubbs/defs/weap.py index 8ef2b974..a240b3fb 100644 --- a/reclaimer/stubbs/defs/weap.py +++ b/reclaimer/stubbs/defs/weap.py @@ -8,18 +8,11 @@ # from ...hek.defs.weap import * -from .obje import * from .item import * +from .obje import * -# replace the object_type enum one that uses -# the correct default value for this object -obje_attrs = dict(obje_attrs) -obje_attrs[0] = dict(obje_attrs[0], DEFAULT=2) - -# replace the object_type enum one that uses -# the correct default value for this object -weap_attrs = dict(weap_attrs) -weap_attrs[24] = SEnum16('weapon_type', *weapon_types) +obje_attrs = obje_attrs_variant(obje_attrs, "weap") +weap_attrs = desc_variant(weap_attrs, SEnum16('weapon_type', *weapon_types)) weap_body = Struct("tagdata", obje_attrs, diff --git a/reclaimer/stubbs/defs/wphi.py b/reclaimer/stubbs/defs/wphi.py index 6e303145..ae1a3206 100644 --- a/reclaimer/stubbs/defs/wphi.py +++ b/reclaimer/stubbs/defs/wphi.py @@ -8,3 +8,22 @@ # from ...hek.defs.wphi import * +from ..common_descs import dependency_stubbs + + +wphi_body = desc_variant(wphi_body, + ("pad_11", Pad(16)), + ("screen_effect", dependency_stubbs("screen_effect", "imef")), + verify=False + ) + + +def get(): + return wphi_def + +wphi_def = TagDef("wphi", + blam_header("wphi", 2), + wphi_body, + + ext=".weapon_hud_interface", endian=">", tag_cls=WphiTag, + ) \ No newline at end of file diff --git a/reclaimer/util/__init__.py b/reclaimer/util/__init__.py index e92c7e96..f7064a62 100644 --- a/reclaimer/util/__init__.py +++ b/reclaimer/util/__init__.py @@ -4,6 +4,7 @@ from math import log from supyr_struct.util import * +from supyr_struct.exceptions import DescKeyError from reclaimer.util import compression from reclaimer.util import geometry from reclaimer.util import matrices @@ -19,7 +20,7 @@ VALID_NUMERIC_CHARS = frozenset("0123456789") for name in ('CON', 'PRN', 'AUX', 'NUL'): RESERVED_WINDOWS_FILENAME_MAP[name] = '_' + name -for i in range(1, 9): +for i in VALID_NUMERIC_CHARS: RESERVED_WINDOWS_FILENAME_MAP['COM%s' % i] = '_COM%s' % i RESERVED_WINDOWS_FILENAME_MAP['LPT%s' % i] = '_LPT%s' % i INVALID_PATH_CHARS.update('<>:"|?*') @@ -48,6 +49,13 @@ def get_is_xbox_map(engine): return "xbox" in engine or engine in ("stubbs", "shadowrun_proto") +def get_block_max(block, default=0xFFffFFff): + try: + return block.get_desc('MAX', 'size') + except DescKeyError: + return default + + def float_to_str(f, max_sig_figs=7): if f == POS_INF: return "1000000000000000000000000000000000000000" @@ -126,10 +134,15 @@ def is_valid_ascii_name_str(string): return True def calc_halo_crc32(buffer, offset=None, size=None, crc=0xFFffFFff): - if offset is not None: + offset = 0 if offset is None else offset + if isinstance(buffer, (bytes, bytearray)): + size = len(buffer) if size is None else size + tag_bytes = buffer[offset: offset + size] + else: buffer.seek(offset) + tag_bytes = buffer.read(size) - return zlib.crc32(buffer.read(size), crc ^ 0xFFffFFff) ^ 0xFFffFFff + return zlib.crc32(tag_bytes, crc ^ 0xFFffFFff) ^ 0xFFffFFff NEWLINE_MATCHER = re.compile(r'\r\n|\n\r|\r|\n') diff --git a/reclaimer/util/compression.py b/reclaimer/util/compression.py index 026445a1..0b1316ff 100644 --- a/reclaimer/util/compression.py +++ b/reclaimer/util/compression.py @@ -58,17 +58,16 @@ def decompress_normal32(n): def compress_normal32(i, j, k): - i = min(max(i, -1.0), 1.0) - j = min(max(j, -1.0), 1.0) - k = min(max(k, -1.0), 1.0) + # logical comparisons like this are faster than chaining 2(more) function calls + ci = 1024 if i <= -1 else 1023 if i >= 1 else int(round(i*1023)) % 2047 + cj = 1024 if j <= -1 else 1023 if j >= 1 else int(round(j*1023)) % 2047 + ck = 512 if k <= -1 else 511 if k >= 1 else int(round(k*511)) % 1023 # original algorithm before shelly's optimization, kept for clarity #if i < 0: i += 2047 #if j < 0: j += 2047 #if k < 0: k += 1023 #return i | (j << 11) | (k << 22) - return ((int(round(i*1023)) % 2047) | - ((int(round(j*1023)) % 2047) << 11) | - ((int(round(k*511)) % 1023) << 22)) + return ci | (cj << 11) | (ck << 22) #uncomp_norm = [.333, -.75, 1] diff --git a/reclaimer/util/geometry.py b/reclaimer/util/geometry.py index 30cf386f..f495e940 100644 --- a/reclaimer/util/geometry.py +++ b/reclaimer/util/geometry.py @@ -345,9 +345,8 @@ def depth(self): return depth -def planes_to_verts_and_edge_loops(planes, center, plane_dir=True, max_plane_ct=32, +def planes_to_verts_and_edge_loops(planes, plane_dir=True, max_plane_ct=32, use_double_rounding=False, round_adjust=0): - assert len(center) == 3 # make a set out of the planes to remove duplicates planes = list(set(tuple(Plane(p).normalized) for p in planes)) indices_by_planes = {p: set() for p in planes} diff --git a/requirements.txt b/requirements.txt index 00ba78f9..39ef0e4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ supyr_struct binilla +simpleaudio +pyogg \ No newline at end of file diff --git a/setup.py b/setup.py index fcde4aec..4d77dfa6 100644 --- a/setup.py +++ b/setup.py @@ -39,9 +39,13 @@ 'reclaimer.h3.defs', 'reclaimer.h3.defs.objs', 'reclaimer.halo_script', + 'reclaimer.halo_script.defs', 'reclaimer.hek', 'reclaimer.hek.defs', 'reclaimer.hek.defs.objs', + 'reclaimer.mcc_hek', + 'reclaimer.mcc_hek.defs', + 'reclaimer.mcc_hek.defs.objs', 'reclaimer.meta', 'reclaimer.meta.gen3_resources', 'reclaimer.meta.objs', @@ -64,6 +68,7 @@ 'reclaimer.sounds.ext', 'reclaimer.shadowrun_prototype', 'reclaimer.shadowrun_prototype.defs', + 'reclaimer.shadowrun_prototype.defs.objs', 'reclaimer.strings', 'reclaimer.stubbs', 'reclaimer.stubbs.defs', @@ -89,8 +94,8 @@ }, platforms=["POSIX", "Windows"], keywords=["reclaimer", "halo"], - install_requires=['supyr_struct>=1.5.0', 'binilla', 'arbytmap'], - requires=['supyr_struct', 'binilla', 'arbytmap'], + install_requires=['supyr_struct>=1.5.0', 'binilla', 'arbytmap', 'simpleaudio', 'pyogg'], + requires=['supyr_struct', 'binilla', 'arbytmap', 'simpleaudio', 'pyogg'], provides=['reclaimer'], python_requires=">=3.5", classifiers=[