Skip to content

Commit 410bb90

Browse files
sveinsesveinse
committed
Improve odg diff mechanism
* Add text diff by default, with options for data and internal variants
1 parent c0c29ca commit 410bb90

File tree

6 files changed

+283
-103
lines changed

6 files changed

+283
-103
lines changed

‎setup.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ python_requires = >=3.10, <4
3131
install_requires =
3232
jsonschema
3333
colorama
34-
deepdiff<8.0.0
34+
deepdiff
3535

3636
[options.packages.find]
3737
where = src

‎src/objdictgen/__main__.py

+15-48
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,15 @@
2323
import logging
2424
import sys
2525
from dataclasses import dataclass, field
26-
from pprint import pformat
27-
from typing import TYPE_CHECKING, Callable, Generator, Sequence, TypeVar
26+
from typing import Callable, Sequence, TypeVar
2827

2928
from colorama import Fore, Style, init
3029

3130
import objdictgen
3231
from objdictgen import jsonod
3332
from objdictgen.node import Node
34-
from objdictgen.printing import format_node
35-
from objdictgen.typing import TDiffEntries, TDiffNodes, TPath
33+
from objdictgen.printing import format_diff_nodes, format_node
34+
from objdictgen.typing import TPath
3635

3736
T = TypeVar('T')
3837

@@ -88,38 +87,6 @@ def open_od(fname: TPath|str, validate=True, fix=False) -> "Node":
8887
raise
8988

9089

91-
def print_diffs(diffs: TDiffNodes, show=False):
92-
""" Print the differences between two object dictionaries"""
93-
94-
def _pprint(text: str):
95-
for line in pformat(text).splitlines():
96-
print(" ", line)
97-
98-
def _printlines(entries: TDiffEntries):
99-
for chtype, change, path in entries:
100-
if 'removed' in chtype:
101-
print(f"<<< {path} only in LEFT")
102-
if show:
103-
_pprint(change.t1)
104-
elif 'added' in chtype:
105-
print(f" >>> {path} only in RIGHT")
106-
if show:
107-
_pprint(change.t2)
108-
elif 'changed' in chtype:
109-
print(f"<< - >> {path} value changed from '{change.t1}' to '{change.t2}'")
110-
else:
111-
print(f"{Fore.RED}{chtype} {path} {change}{Style.RESET_ALL}")
112-
113-
rest = diffs.pop('', None)
114-
if rest:
115-
print(f"{Fore.GREEN}Changes:{Style.RESET_ALL}")
116-
_printlines(rest)
117-
118-
for index in sorted(diffs):
119-
print(f"{Fore.GREEN}Index 0x{index:04x} ({index}){Style.RESET_ALL}")
120-
_printlines(diffs[index])
121-
122-
12390
@debug_wrapper()
12491
def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
12592
""" Main command dispatcher """
@@ -144,6 +111,7 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
144111
opt_debug = dict(action='store_true', help="Debug: enable tracebacks on errors")
145112
opt_od = dict(metavar='od', default=None, help="Object dictionary")
146113
opt_novalidate = dict(action='store_true', help="Don't validate input files")
114+
opt_nocolor = dict(action='store_true', help="Disable colored output")
147115

148116
parser.add_argument('--version', action='version', version='%(prog)s ' + objdictgen.__version__)
149117
parser.add_argument('--no-color', action='store_true', help="Disable colored output")
@@ -183,11 +151,13 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
183151
""", aliases=['compare'])
184152
subp.add_argument('od1', **opt_od) # type: ignore[arg-type]
185153
subp.add_argument('od2', **opt_od) # type: ignore[arg-type]
154+
subp.add_argument('--show', action="store_true", help="Show difference data")
186155
subp.add_argument('--internal', action="store_true", help="Diff internal object")
156+
subp.add_argument('--data', action="store_true", help="Show difference as data")
157+
subp.add_argument('--raw', action="store_true", help="Show raw difference")
158+
subp.add_argument('--no-color', **opt_nocolor) # type: ignore[arg-type]
187159
subp.add_argument('--novalidate', **opt_novalidate) # type: ignore[arg-type]
188-
subp.add_argument('--show', action="store_true", help="Show difference data")
189160
subp.add_argument('-D', '--debug', **opt_debug) # type: ignore[arg-type]
190-
subp.add_argument('--no-color', action='store_true', help="Disable colored output")
191161

192162
# -- EDIT --
193163
subp = subparser.add_parser('edit', help="""
@@ -210,9 +180,9 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
210180
subp.add_argument('--short', action="store_true", help="Do not list sub-index")
211181
subp.add_argument('--unused', action="store_true", help="Include unused profile parameters")
212182
subp.add_argument('--internal', action="store_true", help="Show internal data")
213-
subp.add_argument('-D', '--debug', **opt_debug) # type: ignore[arg-type]
183+
subp.add_argument('--no-color', **opt_nocolor) # type: ignore[arg-type]
214184
subp.add_argument('--novalidate', **opt_novalidate) # type: ignore[arg-type]
215-
subp.add_argument('--no-color', action='store_true', help="Disable colored output")
185+
subp.add_argument('-D', '--debug', **opt_debug) # type: ignore[arg-type]
216186

217187
# -- NETWORK --
218188
subp = subparser.add_parser('network', help="""
@@ -306,22 +276,19 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
306276

307277
# -- DIFF command --
308278
elif opts.command in ("diff", "compare"):
279+
309280
od1 = open_od(opts.od1, validate=not opts.novalidate)
310281
od2 = open_od(opts.od2, validate=not opts.novalidate)
311282

312-
diffs = jsonod.diff_nodes(
313-
od1, od2, asdict=not opts.internal,
314-
validate=not opts.novalidate,
315-
)
283+
lines = list(format_diff_nodes(od1, od2, data=opts.data, raw=opts.raw,
284+
internal=opts.internal, show=opts.show))
316285

317-
if diffs:
318-
errcode = 1
286+
errcode = 1 if lines else 0
287+
if errcode:
319288
print(f"{objdictgen.ODG_PROGRAM}: '{opts.od1}' and '{opts.od2}' differ")
320289
else:
321-
errcode = 0
322290
print(f"{objdictgen.ODG_PROGRAM}: '{opts.od1}' and '{opts.od2}' are equal")
323291

324-
print_diffs(diffs, show=opts.show)
325292
if errcode:
326293
parser.exit(errcode)
327294

‎src/objdictgen/jsonod.py

+47-51
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
1818
# USA
1919

20+
from __future__ import annotations
21+
2022
import copy
2123
import json
2224
import logging
@@ -36,7 +38,7 @@
3638
from objdictgen.typing import (TDiffNodes, TIndexEntry, TODJson, TODObjJson,
3739
TODObj, TODSubObj, TODSubObjJson, TODValue, TParamEntry, TPath, TProfileMenu)
3840
from objdictgen.utils import (copy_in_order, exc_amend, maybe_number,
39-
str_to_int)
41+
str_to_int, strip_brackets)
4042

4143
T = TypeVar('T')
4244
M = TypeVar('M', bound=Mapping)
@@ -173,6 +175,10 @@ class ValidationError(Exception):
173175
# Copied from https://github.com/NickolaiBeloguzov/jsonc-parser/blob/master/jsonc_parser/parser.py#L11-L39
174176
RE_JSONC = re.compile(r"(\".*?(?<!\\)\"|\'.*?\')|(\s*/\*.*?\*/\s*|\s*//[^\r\n]*$)", re.MULTILINE | re.DOTALL)
175177

178+
# Regexs to handle parsing of diffing the JSON
179+
RE_DIFF_ROOT = re.compile(r"^(root(\[.*?\]))(.*)")
180+
RE_DIFF_INDEX = re.compile(r"\['dictionary'\]\[(\d+)\](.*)")
181+
176182

177183
def remove_jsonc(text: str) -> str:
178184
""" Remove jsonc annotations """
@@ -1394,67 +1400,57 @@ def _validate_dictionary(index, obj):
13941400
raise
13951401

13961402

1397-
def diff_nodes(node1: "Node", node2: "Node", asdict=True, validate=True) -> TDiffNodes:
1403+
def diff(node1: Node, node2: Node, internal=False) -> TDiffNodes:
13981404
"""Compare two nodes and return the differences."""
13991405

14001406
diffs: dict[int|str, list] = {}
14011407

1402-
if asdict:
1403-
jd1 = node_todict(node1, sort=True, validate=validate)
1404-
jd2 = node_todict(node2, sort=True, validate=validate)
1408+
if internal:
14051409

1406-
dt = datetime.isoformat(datetime.now())
1407-
jd1['$date'] = jd2['$date'] = dt
1410+
# Simply diff the python data structure for the nodes
1411+
diff = deepdiff.DeepDiff(node1.__dict__, node2.__dict__, exclude_paths=[
1412+
"root.IndexOrder"
1413+
], view='tree')
14081414

1409-
# DeepDiff does not have typing, but the diff object is a dict-like object
1410-
# DeepDiff[str, deepdiff.model.PrettyOrderedSet].
1411-
# PrettyOrderedSet is a list-like object
1412-
# PrettyOrderedSet[deepdiff.model.DiffLevel]
1415+
else:
14131416

1414-
diff = deepdiff.DeepDiff(jd1, jd2, exclude_paths=[
1415-
"root['dictionary']"
1416-
], view='tree')
1417+
# Don't use rich format for diffing, as it will contain comments which confuse the output
1418+
jd1 = node_todict(node1, sort=True, rich=False, internal=True)
1419+
jd2 = node_todict(node2, sort=True, rich=False, internal=True)
14171420

1418-
chtype: str
1419-
for chtype, changes in diff.items():
1420-
change: deepdiff.model.DiffLevel
1421-
for change in changes:
1422-
path: str = change.path(force='fake') # pyright: ignore[reportAssignmentType]
1423-
entries = diffs.setdefault('', [])
1424-
entries.append((chtype, change, path.replace('root', '')))
1425-
1426-
diff = deepdiff.DeepDiff(jd1['dictionary'], jd2['dictionary'], view='tree', group_by='index')
1427-
1428-
res = re.compile(r"root\[('0x[0-9a-fA-F]+'|\d+)\]")
1429-
1430-
for chtype, changes in diff.items():
1431-
for change in changes:
1432-
path = change.path(force='fake') # pyright: ignore[reportAssignmentType]
1433-
m = res.search(path)
1434-
if m:
1435-
num = str_to_int(m.group(1).strip("'"))
1436-
entries = diffs.setdefault(num, [])
1437-
entries.append((chtype, change, path.replace(m.group(0), '')))
1438-
else:
1439-
entries = diffs.setdefault('', [])
1440-
entries.append((chtype, change, path.replace('root', '')))
1421+
# Convert the dictionary list to a dict to ensure the order of the objects
1422+
jd1["dictionary"] = {obj["index"]: obj for obj in jd1["dictionary"]}
1423+
jd2["dictionary"] = {obj["index"]: obj for obj in jd2["dictionary"]}
14411424

1442-
else:
1443-
diff = deepdiff.DeepDiff(node1.__dict__, node2.__dict__, exclude_paths=[
1444-
"root.IndexOrder"
1445-
], view='tree')
1425+
# Diff the two nodes in json object format
1426+
diff = deepdiff.DeepDiff(jd1, jd2, view='tree')
1427+
1428+
# Iterate over the changes
1429+
for chtype, changes in diff.items():
1430+
for change in changes:
1431+
path = change.path()
14461432

1447-
res = re.compile(r"root\.(Profile|Dictionary|ParamsDictionary|UserMapping|DS302)\[(\d+)\]")
1433+
# Match the root[<obj>]... part of the path
1434+
m = RE_DIFF_ROOT.match(path)
1435+
if not m:
1436+
raise ValueError(f"Unexpected path '{path}' in compare")
1437+
1438+
# Path is the display path, root the categorization
1439+
path = m[2] + m[3]
1440+
root = m[2]
1441+
1442+
if not internal:
1443+
if m[1] == "root['dictionary']":
1444+
# Extract the index from the path
1445+
m = RE_DIFF_INDEX.match(path)
1446+
root = f"Index {m[1]}"
1447+
path = m[2]
14481448

1449-
for chtype, changes in diff.items():
1450-
for change in changes:
1451-
path = change.path(force='fake') # pyright: ignore[reportAssignmentType]
1452-
m = res.search(path)
1453-
if m:
1454-
entries = diffs.setdefault(int(m.group(2)), [])
1455-
entries.append((chtype, change, path.replace(m.group(0), m.group(1))))
14561449
else:
1457-
entries = diffs.setdefault('', [])
1458-
entries.append((chtype, change, path.replace('root.', '')))
1450+
root = "Header fields"
1451+
1452+
# Append the change to the list of changes
1453+
entries = diffs.setdefault(strip_brackets(root), [])
1454+
entries.append((chtype, change, strip_brackets(path)))
14591455

14601456
return diffs

‎src/objdictgen/printing.py

+112-2
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@
1010
from objdictgen import jsonod, maps
1111
from objdictgen.maps import OD
1212
from objdictgen.node import Node
13-
from objdictgen.typing import TIndexEntry
14-
from objdictgen.utils import TERM_COLS, str_to_int
13+
from objdictgen.typing import TDiffNodes, TIndexEntry
14+
from objdictgen.utils import (TERM_COLS, diff_colored_lines, highlight_changes,
15+
remove_color, str_to_int)
1516

1617

1718
@dataclass
@@ -313,3 +314,112 @@ def format_od_object(
313314
if not compact and infos:
314315
yield ""
315316

317+
318+
def format_diff_nodes(
319+
od1: Node, od2: Node, *, data=False, raw=False,
320+
internal=False, show=False
321+
) -> Generator[str, None, None]:
322+
""" Compare two object dictionaries and return the formatted differences. """
323+
324+
if internal or data:
325+
diffs = jsonod.diff(od1, od2, internal=internal)
326+
else:
327+
diffs = text_diff(od1, od2, data_mode=raw)
328+
329+
rst = Style.RESET_ALL
330+
331+
def _pprint(text: str, prefix: str = ' '):
332+
for line in pformat(text, width=TERM_COLS).splitlines():
333+
yield prefix + line
334+
335+
for index in sorted(diffs):
336+
yield f"{Fore.LIGHTYELLOW_EX}{index}{rst}"
337+
entries = diffs[index]
338+
for chtype, change, path in entries:
339+
340+
# Prepare the path for printing
341+
ppath = path
342+
if ppath:
343+
if ppath[0] != "'":
344+
ppath = "'" + ppath + "'"
345+
ppath = ppath + ' '
346+
ppath = f"{Fore.CYAN}{ppath}{rst}"
347+
348+
if 'removed' in chtype:
349+
yield f"<< {ppath}only in {Fore.MAGENTA}LEFT{rst}"
350+
if show:
351+
yield from _pprint(change.t1, " < ")
352+
elif 'added' in chtype:
353+
yield f" >> {ppath}only in {Fore.BLUE}RIGHT{rst}"
354+
if show:
355+
yield from _pprint(change.t2, " > ")
356+
elif 'changed' in chtype:
357+
yield f"<< - >> {ppath}changed value from '{Fore.GREEN}{change.t1}{rst}' to '{Fore.GREEN}{change.t2}{rst}'"
358+
if show:
359+
yield from _pprint(change.t1, " < ")
360+
yield from _pprint(change.t2, " > ")
361+
elif 'type_changes' in chtype:
362+
yield f"<< - >> {ppath}changed type and value from '{Fore.GREEN}{change.t1}{rst}' to '{Fore.GREEN}{change.t2}{rst}'"
363+
if show:
364+
yield from _pprint(change.t1, " < ")
365+
yield from _pprint(change.t2, " > ")
366+
elif 'diff' in chtype:
367+
start = path[0:2]
368+
if start == ' ':
369+
ppath = ' ' + path
370+
elif start == '+ ':
371+
ppath = path.replace('+ ', ' >> ')
372+
if ppath == ' >> ':
373+
ppath = ''
374+
elif start == '- ':
375+
ppath = path.replace('- ', '<< ')
376+
if ppath == '<< ':
377+
ppath = ''
378+
elif start == '? ':
379+
ppath = path.replace('? ', ' ')
380+
ppath = f"{Fore.RED}{ppath}{rst}"
381+
else:
382+
ppath = f"{Fore.RED}{chtype} {ppath} {change}{rst}"
383+
yield ppath
384+
else:
385+
yield f"{Fore.RED}{chtype} {ppath} {change}{rst}"
386+
387+
388+
def text_diff(od1: Node, od2: Node, data_mode: bool=False) -> TDiffNodes:
389+
""" Compare two object dictionaries as text and return the differences. """
390+
391+
# Get all indices for the nodes
392+
keys1 = set(od1.GetAllIndices())
393+
keys2 = set(od2.GetAllIndices())
394+
395+
diffs: dict[int|str, list] = {}
396+
397+
for index in sorted(keys1 | keys2):
398+
changes = []
399+
400+
# Get the object print entries
401+
text1 = text2 = []
402+
entry1: TIndexEntry = {}
403+
entry2: TIndexEntry = {}
404+
if index in keys1:
405+
text1 = list(format_od_object(od1, index))
406+
entry1 = od1.GetIndexEntry(index)
407+
if index in keys2:
408+
text2 = list(format_od_object(od2, index))
409+
entry2 = od2.GetIndexEntry(index)
410+
411+
if data_mode:
412+
text1 = text2 = []
413+
if entry1:
414+
text1 = pformat(entry1, width=TERM_COLS-10, indent=2).splitlines()
415+
if entry2:
416+
text2 = pformat(entry2, width=TERM_COLS-10, indent=2).splitlines()
417+
418+
if entry1 == entry2:
419+
continue
420+
421+
for line in highlight_changes(diff_colored_lines(text1, text2)):
422+
changes.append(('diff', '', line))
423+
diffs[f"Index {index}"] = changes
424+
425+
return diffs

0 commit comments

Comments
 (0)