diff --git a/can/__init__.py b/can/__init__.py index 1d4b7f0cf..9a1e0c715 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -44,6 +44,8 @@ "MessageSync", "ModifiableCyclicTaskABC", "Notifier", + "PcapngReader", + "PcapngWriter", "Printer", "RedirectReader", "RestartableCyclicTaskABC", @@ -111,6 +113,8 @@ MessageSync, MF4Reader, MF4Writer, + PcapngReader, + PcapngWriter, Printer, SizedRotatingLogger, SqliteReader, diff --git a/can/interfaces/socketcan/__init__.py b/can/interfaces/socketcan/__init__.py index 6e279c2fe..9cc034711 100644 --- a/can/interfaces/socketcan/__init__.py +++ b/can/interfaces/socketcan/__init__.py @@ -6,7 +6,6 @@ "CyclicSendTask", "MultiRateCyclicSendTask", "SocketcanBus", - "constants", "socketcan", "utils", ] diff --git a/can/interfaces/socketcan/constants.py b/can/interfaces/socketcan/constants.py deleted file mode 100644 index 941d52573..000000000 --- a/can/interfaces/socketcan/constants.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -Defines shared CAN constants. -""" - -# Generic socket constants -SO_TIMESTAMPNS = 35 - -CAN_ERR_FLAG = 0x20000000 -CAN_RTR_FLAG = 0x40000000 -CAN_EFF_FLAG = 0x80000000 - -# BCM opcodes -CAN_BCM_TX_SETUP = 1 -CAN_BCM_TX_DELETE = 2 -CAN_BCM_TX_READ = 3 - -# BCM flags -SETTIMER = 0x0001 -STARTTIMER = 0x0002 -TX_COUNTEVT = 0x0004 -TX_ANNOUNCE = 0x0008 -TX_CP_CAN_ID = 0x0010 -RX_FILTER_ID = 0x0020 -RX_CHECK_DLC = 0x0040 -RX_NO_AUTOTIMER = 0x0080 -RX_ANNOUNCE_RESUME = 0x0100 -TX_RESET_MULTI_IDX = 0x0200 -RX_RTR_FRAME = 0x0400 -CAN_FD_FRAME = 0x0800 - -CAN_RAW = 1 -CAN_BCM = 2 - -SOL_CAN_BASE = 100 -SOL_CAN_RAW = SOL_CAN_BASE + CAN_RAW - -CAN_RAW_FILTER = 1 -CAN_RAW_ERR_FILTER = 2 -CAN_RAW_LOOPBACK = 3 -CAN_RAW_RECV_OWN_MSGS = 4 -CAN_RAW_FD_FRAMES = 5 - -MSK_ARBID = 0x1FFFFFFF -MSK_FLAGS = 0xE0000000 - -PF_CAN = 29 -SOCK_RAW = 3 -SOCK_DGRAM = 2 -AF_CAN = PF_CAN - -SIOCGIFNAME = 0x8910 -SIOCGIFINDEX = 0x8933 -SIOCGSTAMP = 0x8906 -EXTFLG = 0x0004 - -CANFD_BRS = 0x01 # bit rate switch (second bitrate for payload data) -CANFD_ESI = 0x02 # error state indicator of the transmitting node -CANFD_FDF = 0x04 # mark CAN FD for dual use of struct canfd_frame - -# CAN payload length and DLC definitions according to ISO 11898-1 -CAN_MAX_DLC = 8 -CAN_MAX_RAW_DLC = 15 -CAN_MAX_DLEN = 8 - -# CAN FD payload length and DLC definitions according to ISO 11898-7 -CANFD_MAX_DLC = 15 -CANFD_MAX_DLEN = 64 - -CANFD_MTU = 72 - -STD_ACCEPTANCE_MASK_ALL_BITS = 2**11 - 1 -MAX_11_BIT_ID = STD_ACCEPTANCE_MASK_ALL_BITS - -EXT_ACCEPTANCE_MASK_ALL_BITS = 2**29 - 1 -MAX_29_BIT_ID = EXT_ACCEPTANCE_MASK_ALL_BITS diff --git a/can/interfaces/socketcan/socketcan.py b/can/interfaces/socketcan/socketcan.py index a5819dcb5..f38d14969 100644 --- a/can/interfaces/socketcan/socketcan.py +++ b/can/interfaces/socketcan/socketcan.py @@ -18,13 +18,13 @@ from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union import can +import can.socketcan_common as common from can import BusABC, CanProtocol, Message from can.broadcastmanager import ( LimitedDurationCyclicSendTaskABC, ModifiableCyclicTaskABC, RestartableCyclicTaskABC, ) -from can.interfaces.socketcan import constants from can.interfaces.socketcan.utils import find_available_interfaces, pack_filters from can.typechecking import CanFilters @@ -134,118 +134,6 @@ def bcm_header_factory( ) -# struct module defines a binary packing format: -# https://docs.python.org/3/library/struct.html#struct-format-strings -# The 32bit can id is directly followed by the 8bit data link count -# The data field is aligned on an 8 byte boundary, hence we add padding -# which aligns the data field to an 8 byte boundary. -CAN_FRAME_HEADER_STRUCT = struct.Struct("=IBB1xB") - - -def build_can_frame(msg: Message) -> bytes: - """CAN frame packing/unpacking (see 'struct can_frame' in ) - /** - * struct can_frame - Classical CAN frame structure (aka CAN 2.0B) - * @can_id: CAN ID of the frame and CAN_*_FLAG flags, see canid_t definition - * @len: CAN frame payload length in byte (0 .. 8) - * @can_dlc: deprecated name for CAN frame payload length in byte (0 .. 8) - * @__pad: padding - * @__res0: reserved / padding - * @len8_dlc: optional DLC value (9 .. 15) at 8 byte payload length - * len8_dlc contains values from 9 .. 15 when the payload length is - * 8 bytes but the DLC value (see ISO 11898-1) is greater then 8. - * CAN_CTRLMODE_CC_LEN8_DLC flag has to be enabled in CAN driver. - * @data: CAN frame payload (up to 8 byte) - */ - struct can_frame { - canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ - union { - /* CAN frame payload length in byte (0 .. CAN_MAX_DLEN) - * was previously named can_dlc so we need to carry that - * name for legacy support - */ - __u8 len; - __u8 can_dlc; /* deprecated */ - } __attribute__((packed)); /* disable padding added in some ABIs */ - __u8 __pad; /* padding */ - __u8 __res0; /* reserved / padding */ - __u8 len8_dlc; /* optional DLC for 8 byte payload length (9 .. 15) */ - __u8 data[CAN_MAX_DLEN] __attribute__((aligned(8))); - }; - - /* - * defined bits for canfd_frame.flags - * - * The use of struct canfd_frame implies the FD Frame (FDF) bit to - * be set in the CAN frame bitstream on the wire. The FDF bit switch turns - * the CAN controllers bitstream processor into the CAN FD mode which creates - * two new options within the CAN FD frame specification: - * - * Bit Rate Switch - to indicate a second bitrate is/was used for the payload - * Error State Indicator - represents the error state of the transmitting node - * - * As the CANFD_ESI bit is internally generated by the transmitting CAN - * controller only the CANFD_BRS bit is relevant for real CAN controllers when - * building a CAN FD frame for transmission. Setting the CANFD_ESI bit can make - * sense for virtual CAN interfaces to test applications with echoed frames. - * - * The struct can_frame and struct canfd_frame intentionally share the same - * layout to be able to write CAN frame content into a CAN FD frame structure. - * When this is done the former differentiation via CAN_MTU / CANFD_MTU gets - * lost. CANFD_FDF allows programmers to mark CAN FD frames in the case of - * using struct canfd_frame for mixed CAN / CAN FD content (dual use). - * Since the introduction of CAN XL the CANFD_FDF flag is set in all CAN FD - * frame structures provided by the CAN subsystem of the Linux kernel. - */ - #define CANFD_BRS 0x01 /* bit rate switch (second bitrate for payload data) */ - #define CANFD_ESI 0x02 /* error state indicator of the transmitting node */ - #define CANFD_FDF 0x04 /* mark CAN FD for dual use of struct canfd_frame */ - - /** - * struct canfd_frame - CAN flexible data rate frame structure - * @can_id: CAN ID of the frame and CAN_*_FLAG flags, see canid_t definition - * @len: frame payload length in byte (0 .. CANFD_MAX_DLEN) - * @flags: additional flags for CAN FD - * @__res0: reserved / padding - * @__res1: reserved / padding - * @data: CAN FD frame payload (up to CANFD_MAX_DLEN byte) - */ - struct canfd_frame { - canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ - __u8 len; /* frame payload length in byte */ - __u8 flags; /* additional flags for CAN FD */ - __u8 __res0; /* reserved / padding */ - __u8 __res1; /* reserved / padding */ - __u8 data[CANFD_MAX_DLEN] __attribute__((aligned(8))); - }; - """ - can_id = _compose_arbitration_id(msg) - - flags = 0 - - # The socketcan code identify the received FD frame by the packet length. - # So, padding to the data length is performed according to the message type (Classic / FD) - if msg.is_fd: - flags |= constants.CANFD_FDF - max_len = constants.CANFD_MAX_DLEN - else: - max_len = constants.CAN_MAX_DLEN - - if msg.bitrate_switch: - flags |= constants.CANFD_BRS - if msg.error_state_indicator: - flags |= constants.CANFD_ESI - - data = bytes(msg.data).ljust(max_len, b"\x00") - - if msg.is_remote_frame: - data_len = msg.dlc - else: - data_len = min(i for i in can.util.CAN_FD_DLC if i >= len(msg.data)) - header = CAN_FRAME_HEADER_STRUCT.pack(can_id, data_len, flags, msg.dlc) - return header + data - - def build_bcm_header( opcode: int, flags: int, @@ -272,7 +160,7 @@ def build_bcm_header( def build_bcm_tx_delete_header(can_id: int, flags: int) -> bytes: - opcode = constants.CAN_BCM_TX_DELETE + opcode = common.CAN_BCM_TX_DELETE return build_bcm_header(opcode, flags, 0, 0, 0, 0, 0, can_id, 1) @@ -284,13 +172,13 @@ def build_bcm_transmit_header( msg_flags: int, nframes: int = 1, ) -> bytes: - opcode = constants.CAN_BCM_TX_SETUP + opcode = common.CAN_BCM_TX_SETUP - flags = msg_flags | constants.SETTIMER | constants.STARTTIMER + flags = msg_flags | common.SETTIMER | common.STARTTIMER if initial_period > 0: # Note `TX_COUNTEVT` creates the message TX_EXPIRED when count expires - flags |= constants.TX_COUNTEVT + flags |= common.TX_COUNTEVT def split_time(value: float) -> Tuple[int, int]: """Given seconds as a float, return whole seconds and microseconds""" @@ -316,40 +204,13 @@ def split_time(value: float) -> Tuple[int, int]: def build_bcm_update_header(can_id: int, msg_flags: int, nframes: int = 1) -> bytes: return build_bcm_header( - constants.CAN_BCM_TX_SETUP, msg_flags, 0, 0, 0, 0, 0, can_id, nframes + common.CAN_BCM_TX_SETUP, msg_flags, 0, 0, 0, 0, 0, can_id, nframes ) -def is_frame_fd(frame: bytes): - # According to the SocketCAN implementation the frame length - # should indicate if the message is FD or not (not the flag value) - return len(frame) == constants.CANFD_MTU - - -def dissect_can_frame(frame: bytes) -> Tuple[int, int, int, bytes]: - can_id, data_len, flags, len8_dlc = CAN_FRAME_HEADER_STRUCT.unpack_from(frame) - - if data_len not in can.util.CAN_FD_DLC: - data_len = min(i for i in can.util.CAN_FD_DLC if i >= data_len) - - can_dlc = data_len - - if not is_frame_fd(frame): - # Flags not valid in non-FD frames - flags = 0 - - if ( - data_len == constants.CAN_MAX_DLEN - and constants.CAN_MAX_DLEN < len8_dlc <= constants.CAN_MAX_RAW_DLC - ): - can_dlc = len8_dlc - - return can_id, can_dlc, flags, frame[8 : 8 + data_len] - - def create_bcm_socket(channel: str) -> socket.socket: """create a broadcast manager socket and connect to the given interface""" - s = socket.socket(constants.PF_CAN, socket.SOCK_DGRAM, constants.CAN_BCM) + s = socket.socket(common.PF_CAN, socket.SOCK_DGRAM, common.CAN_BCM) s.connect((channel,)) return s @@ -375,20 +236,6 @@ def send_bcm(bcm_socket: socket.socket, data: bytes) -> int: raise can.CanOperationError(base + specific_message, error.errno) from error -def _compose_arbitration_id(message: Message) -> int: - can_id = message.arbitration_id - if message.is_extended_id: - log.debug("sending an extended id type message") - can_id |= constants.CAN_EFF_FLAG - if message.is_remote_frame: - log.debug("requesting a remote frame") - can_id |= constants.CAN_RTR_FLAG - if message.is_error_frame: - log.debug("sending error frame") - can_id |= constants.CAN_ERR_FLAG - return can_id - - class CyclicSendTask( LimitedDurationCyclicSendTaskABC, ModifiableCyclicTaskABC, RestartableCyclicTaskABC ): @@ -440,7 +287,7 @@ def _tx_setup( ) -> None: # Create a low level packed frame to pass to the kernel body = bytearray() - self.flags = constants.CAN_FD_FRAME if messages[0].is_fd else 0 + self.flags = common.CAN_FD_FRAME if messages[0].is_fd else 0 if self.duration: count = int(self.duration / self.period) @@ -458,7 +305,7 @@ def _tx_setup( self.task_id, count, ival1, ival2, self.flags, nframes=len(messages) ) for message in messages: - body += build_can_frame(message) + body += common.build_can_frame(message) log.debug("Sending BCM command") send_bcm(self.bcm_socket, header + body) @@ -466,7 +313,7 @@ def _check_bcm_task(self) -> None: # Do a TX_READ on a task ID, and check if we get EINVAL. If so, # then we are referring to a CAN message with an existing ID check_header = build_bcm_header( - opcode=constants.CAN_BCM_TX_READ, + opcode=common.CAN_BCM_TX_READ, flags=0, count=0, ival1_seconds=0, @@ -529,7 +376,7 @@ def modify_data(self, messages: Union[Sequence[Message], Message]) -> None: can_id=self.task_id, msg_flags=self.flags, nframes=len(messages) ) for message in messages: - body += build_can_frame(message) + body += common.build_can_frame(message) log.debug("Sending BCM command") send_bcm(self.bcm_socket, header + body) @@ -571,7 +418,7 @@ def __init__( body = bytearray() for message in messages: - body += build_can_frame(message) + body += common.build_can_frame(message) log.info("Sending BCM TX_SETUP command") send_bcm(self.bcm_socket, header + body) @@ -581,7 +428,7 @@ def create_socket() -> socket.socket: """Creates a raw CAN socket. The socket will be returned unbound to any interface. """ - sock = socket.socket(constants.PF_CAN, socket.SOCK_RAW, constants.CAN_RAW) + sock = socket.socket(common.PF_CAN, socket.SOCK_RAW, common.CAN_RAW) log.info("Created a socket") @@ -620,7 +467,7 @@ def capture_message( # Fetching the Arb ID, DLC and Data try: cf, ancillary_data, msg_flags, addr = sock.recvmsg( - constants.CANFD_MTU, RECEIVED_ANCILLARY_BUFFER_SIZE + common.CANFD_MTU, RECEIVED_ANCILLARY_BUFFER_SIZE ) if get_channel: channel = addr[0] if isinstance(addr, tuple) else addr @@ -631,13 +478,11 @@ def capture_message( f"Error receiving: {error.strerror}", error.errno ) from error - can_id, can_dlc, flags, data = dissect_can_frame(cf) - # Fetching the timestamp assert len(ancillary_data) == 1, "only requested a single extra field" cmsg_level, cmsg_type, cmsg_data = ancillary_data[0] assert ( - cmsg_level == socket.SOL_SOCKET and cmsg_type == constants.SO_TIMESTAMPNS + cmsg_level == socket.SOL_SOCKET and cmsg_type == common.SO_TIMESTAMPNS ), "received control message type that was not requested" # see https://man7.org/linux/man-pages/man3/timespec.3.html -> struct timespec for details seconds, nanoseconds = RECEIVED_TIMESTAMP_STRUCT.unpack_from(cmsg_data) @@ -647,43 +492,13 @@ def capture_message( ) timestamp = seconds + nanoseconds * 1e-9 - # EXT, RTR, ERR flags -> boolean attributes - # /* special address description flags for the CAN_ID */ - # #define CAN_EFF_FLAG 0x80000000U /* EFF/SFF is set in the MSB */ - # #define CAN_RTR_FLAG 0x40000000U /* remote transmission request */ - # #define CAN_ERR_FLAG 0x20000000U /* error frame */ - is_extended_frame_format = bool(can_id & constants.CAN_EFF_FLAG) - is_remote_transmission_request = bool(can_id & constants.CAN_RTR_FLAG) - is_error_frame = bool(can_id & constants.CAN_ERR_FLAG) - is_fd = len(cf) == constants.CANFD_MTU - bitrate_switch = bool(flags & constants.CANFD_BRS) - error_state_indicator = bool(flags & constants.CANFD_ESI) - # Section 4.7.1: MSG_DONTROUTE: set when the received frame was created on the local host. is_rx = not bool(msg_flags & socket.MSG_DONTROUTE) - if is_extended_frame_format: - # log.debug("CAN: Extended") - # TODO does this depend on SFF or EFF? - arbitration_id = can_id & 0x1FFFFFFF - else: - # log.debug("CAN: Standard") - arbitration_id = can_id & 0x000007FF - - msg = Message( - timestamp=timestamp, - channel=channel, - arbitration_id=arbitration_id, - is_extended_id=is_extended_frame_format, - is_remote_frame=is_remote_transmission_request, - is_error_frame=is_error_frame, - is_fd=is_fd, - is_rx=is_rx, - bitrate_switch=bitrate_switch, - error_state_indicator=error_state_indicator, - dlc=can_dlc, - data=data, - ) + msg = common.parse_can_frame(cf) + msg.timestamp = timestamp + msg.channel = channel + msg.is_rx = is_rx return msg @@ -748,8 +563,8 @@ def __init__( # set the local_loopback parameter try: self.socket.setsockopt( - constants.SOL_CAN_RAW, - constants.CAN_RAW_LOOPBACK, + common.SOL_CAN_RAW, + common.CAN_RAW_LOOPBACK, 1 if local_loopback else 0, ) except OSError as error: @@ -758,8 +573,8 @@ def __init__( # set the receive_own_messages parameter try: self.socket.setsockopt( - constants.SOL_CAN_RAW, - constants.CAN_RAW_RECV_OWN_MSGS, + common.SOL_CAN_RAW, + common.CAN_RAW_RECV_OWN_MSGS, 1 if receive_own_messages else 0, ) except OSError as error: @@ -768,9 +583,7 @@ def __init__( # enable CAN-FD frames if desired if fd: try: - self.socket.setsockopt( - constants.SOL_CAN_RAW, constants.CAN_RAW_FD_FRAMES, 1 - ) + self.socket.setsockopt(common.SOL_CAN_RAW, common.CAN_RAW_FD_FRAMES, 1) except OSError as error: log.error("Could not enable CAN-FD frames (%s)", error) @@ -778,7 +591,7 @@ def __init__( # enable error frames try: self.socket.setsockopt( - constants.SOL_CAN_RAW, constants.CAN_RAW_ERR_FILTER, 0x1FFFFFFF + common.SOL_CAN_RAW, common.CAN_RAW_ERR_FILTER, 0x1FFFFFFF ) except OSError as error: log.error("Could not enable error frames (%s)", error) @@ -788,7 +601,7 @@ def __init__( # 1) it is guaranteed to be at least as precise as without # 2) it is available since Linux 2.6.22, and CAN support was only added afterward # so this is always supported by the kernel - self.socket.setsockopt(socket.SOL_SOCKET, constants.SO_TIMESTAMPNS, 1) + self.socket.setsockopt(socket.SOL_SOCKET, common.SO_TIMESTAMPNS, 1) try: bind_socket(self.socket, channel) @@ -861,7 +674,7 @@ def send(self, msg: Message, timeout: Optional[float] = None) -> None: if timeout is None: timeout = 0 time_left = timeout - data = build_can_frame(msg) + data = common.build_can_frame(msg) while time_left >= 0: # Wait for write availability @@ -976,7 +789,7 @@ def _get_bcm_socket(self, channel: str) -> socket.socket: def _apply_filters(self, filters: Optional[can.typechecking.CanFilters]) -> None: try: self.socket.setsockopt( - constants.SOL_CAN_RAW, constants.CAN_RAW_FILTER, pack_filters(filters) + common.SOL_CAN_RAW, common.CAN_RAW_FILTER, pack_filters(filters) ) except OSError as error: # fall back to "software filtering" (= not in kernel) diff --git a/can/interfaces/socketcan/utils.py b/can/interfaces/socketcan/utils.py index 1505c6cf8..7d15500ee 100644 --- a/can/interfaces/socketcan/utils.py +++ b/can/interfaces/socketcan/utils.py @@ -12,7 +12,8 @@ from typing import List, Optional, cast from can import typechecking -from can.interfaces.socketcan.constants import CAN_EFF_FLAG + +from ...socketcan_common import CAN_EFF_FLAG log = logging.getLogger(__name__) diff --git a/can/io/__init__.py b/can/io/__init__.py index 69894c3d0..f04aca7c0 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -20,6 +20,8 @@ "MF4Reader", "MF4Writer", "MessageSync", + "PcapngReader", + "PcapngWriter", "Printer", "SizedRotatingLogger", "SqliteReader", @@ -52,6 +54,7 @@ from .canutils import CanutilsLogReader, CanutilsLogWriter from .csv import CSVReader, CSVWriter from .mf4 import MF4Reader, MF4Writer +from .pcapng import PcapngReader, PcapngWriter from .printer import Printer from .sqlite import SqliteReader, SqliteWriter from .trc import TRCFileVersion, TRCReader, TRCWriter diff --git a/can/io/logger.py b/can/io/logger.py index 359aae4ac..fc9258154 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -37,10 +37,17 @@ MessageWriter, ) from .mf4 import MF4Writer +from .pcapng import PcapngWriter from .printer import Printer from .sqlite import SqliteWriter from .trc import TRCWriter +try: + import pyzstd +except ImportError: + pyzstd = None + + #: A map of file suffixes to their corresponding #: :class:`can.io.generic.MessageWriter` class MESSAGE_WRITERS: Final[Dict[str, Type[MessageWriter]]] = { @@ -50,6 +57,7 @@ ".db": SqliteWriter, ".log": CanutilsLogWriter, ".mf4": MF4Writer, + ".pcapng": PcapngWriter, ".trc": TRCWriter, ".txt": Printer, } @@ -101,7 +109,16 @@ def _compress( else: mode = "at" if append else "wt" - return logger_type, gzip.open(filename, mode) + if suffixes[-1] == ".gz": + compressor = gzip.open(filename, mode) + elif suffixes[-1] == ".zst" and pyzstd is not None: + compressor = pyzstd.open(filename, mode) + else: + raise ValueError( + f"Unknown compression type {suffixes[-1]} in {filename}, maybe a dependency is missing?" + ) + + return logger_type, compressor def Logger( # noqa: N802 @@ -123,7 +140,8 @@ def Logger( # noqa: N802 Any of these formats can be used with gzip compression by appending the suffix .gz (e.g. filename.asc.gz). However, third-party tools might not - be able to read these files. + be able to read these files. Zstandard (.zst) compression is also supported + when an optional dependency is installed. The **filename** may also be *None*, to fall back to :class:`can.Printer`. @@ -148,7 +166,7 @@ def Logger( # noqa: N802 suffix = pathlib.PurePath(filename).suffix.lower() file_or_filename: AcceptedIOType = filename - if suffix == ".gz": + if suffix in (".gz", ".zst"): logger_type, file_or_filename = _compress(filename, **kwargs) else: logger_type = _get_logger_for_suffix(suffix) diff --git a/can/io/pcapng.py b/can/io/pcapng.py new file mode 100644 index 000000000..ad8b91264 --- /dev/null +++ b/can/io/pcapng.py @@ -0,0 +1,167 @@ +""" +Contains handling of pcapng logging files. + +pcapng file is a binary file format used for packet capture files. +Spec: https://www.ietf.org/archive/id/draft-tuexen-opsawg-pcapng-03.html +""" + +import logging +from typing import Any, BinaryIO, Dict, Generator, Optional, Union + +from ..message import Message +from ..socketcan_common import ( + CAN_FRAME_HEADER_STRUCT_BE, + build_can_frame, + parse_can_frame, +) +from ..typechecking import Channel, StringPathLike +from .generic import BinaryIOMessageReader, BinaryIOMessageWriter + +logger = logging.getLogger("can.io.pcapng") + +try: + import pcapng + from pcapng import blocks +except ImportError: + pcapng = None + + +# https://www.tcpdump.org/linktypes.html +# https://www.tcpdump.org/linktypes/LINKTYPE_CAN_SOCKETCAN.html +LINKTYPE_CAN_SOCKETCAN = 227 + + +class PcapngWriter(BinaryIOMessageWriter): + """ + Logs CAN data to an pcapng file supported by Wireshark and other tools. + """ + + def __init__( + self, + file: Union[StringPathLike, BinaryIO], + append: bool = False, + tsresol: int = 9, + **kwargs: Any, + ) -> None: + """ + :param file: + A path-like object or as file-like object to write to. + If this is a file-like object, is has to be opened in + binary write mode, not text write mode. + + :param append: + If True, the file will be opened in append mode. Otherwise, + it will be opened in write mode. The default is False. + + :param tsresol: + The time resolution of the timestamps in the pcapng file, + expressed as -log10(unit in seconds), + e.g. 9 for nanoseconds, 6 for microseconds. + The default is 9, which corresponds to nanoseconds. + . + """ + if pcapng is None: + raise NotImplementedError( + "The python-pcapng package was not found. Install python-can with " + "the optional dependency [pcapng] to use the PcapngWriter." + ) + + mode = "wb+" + if append: + mode = "ab+" + + # pcapng supports concatenation, and thus append + super().__init__(file, mode=mode) + self._header_block = blocks.SectionHeader(endianness=">") + self._writer = pcapng.FileWriter(self.file, self._header_block) + self._idbs: Dict[Channel, blocks.InterfaceDescription] = {} + self.tsresol = tsresol + + def _resolve_idb(self, channel: Optional[Channel]) -> Any: + channel_name = str(channel) + if channel is None: + channel_name = "can0" + + if channel_name not in self._idbs: + idb = blocks.InterfaceDescription( + section=self._header_block.section, + link_type=LINKTYPE_CAN_SOCKETCAN, + options={ + "if_name": channel_name, + "if_tsresol": bytes([self.tsresol]), # nanoseconds + }, + endianness=">", # big + ) + self._header_block.register_interface(idb) + self._writer.write_block(idb) + self._idbs[channel_name] = idb + + return self._idbs[channel_name] + + def on_message_received(self, msg: Message) -> None: + idb: blocks.InterfaceDescription = self._resolve_idb(msg.channel) + timestamp_units = int(msg.timestamp * 10**self.tsresol) + self._writer.write_block( + blocks.EnhancedPacket( + self._header_block.section, + interface_id=idb.interface_id, + packet_data=build_can_frame(msg, structure=CAN_FRAME_HEADER_STRUCT_BE), + # timestamp (in tsresol units) = timestamp_high << 32 + timestamp_low + timestamp_high=timestamp_units >> 32, + timestamp_low=timestamp_units & 0xFFFFFFFF, + endianness=">", # big + ) + ) + + +class PcapngReader(BinaryIOMessageReader): + """ + Iterator of CAN messages from a Pcapng File. + """ + + file: BinaryIO + + def __init__( + self, + file: Union[StringPathLike, BinaryIO], + **kwargs: Any, + ) -> None: + """ + :param file: a path-like object or as file-like object to read from + If this is a file-like object, is has to opened in binary + read mode, not text read mode. + """ + + if pcapng is None: + raise NotImplementedError( + "The python-pcapng package was not found. Install python-can with " + "the optional dependency [pcapng] to use the PcapngReader." + ) + + super().__init__(file, mode="rb") + self._scanner = pcapng.FileScanner(self.file) + + def __iter__(self) -> Generator[Message, None, None]: + for block in self._scanner: + if isinstance(block, blocks.EnhancedPacket): + idn: blocks.InterfaceDescription = block.interface + # We only care about the CAN packets + if idn.link_type != LINKTYPE_CAN_SOCKETCAN: + logger.debug( + "Skipping non-CAN packet, link type: %s", idn.link_type + ) + continue + + msg = parse_can_frame( + block.packet_data, structure=CAN_FRAME_HEADER_STRUCT_BE + ) + + timestamp64 = (block.timestamp_high << 32) + block.timestamp_low + msg.timestamp = timestamp64 * idn.timestamp_resolution + + if "if_name" in idn.options: + msg.channel = idn.options["if_name"] + else: + msg.channel = block.interface_id + + yield msg diff --git a/can/io/player.py b/can/io/player.py index 214112164..c0100c99f 100644 --- a/can/io/player.py +++ b/can/io/player.py @@ -27,9 +27,16 @@ from .csv import CSVReader from .generic import BinaryIOMessageReader, MessageReader from .mf4 import MF4Reader +from .pcapng import PcapngReader from .sqlite import SqliteReader from .trc import TRCReader +try: + import pyzstd +except ImportError: + pyzstd = None + + #: A map of file suffixes to their corresponding #: :class:`can.io.generic.MessageReader` class MESSAGE_READERS: Final[Dict[str, Type[MessageReader]]] = { @@ -39,6 +46,7 @@ ".db": SqliteReader, ".log": CanutilsLogReader, ".mf4": MF4Reader, + ".pcapng": PcapngReader, ".trc": TRCReader, } @@ -79,7 +87,16 @@ def _decompress( mode = "rb" if issubclass(reader_type, BinaryIOMessageReader) else "rt" - return reader_type, gzip.open(filename, mode) + if suffixes[-1] == ".gz": + decompressor = gzip.open(filename, mode) + elif suffixes[-1] == ".zst" and pyzstd is not None: + decompressor = pyzstd.open(filename, mode) + else: + raise ValueError( + f"Unknown compression type {suffixes[-1]} in {filename}, maybe a dependency is missing?" + ) + + return reader_type, decompressor def LogReader(filename: StringPathLike, **kwargs: Any) -> MessageReader: # noqa: N802 @@ -96,7 +113,7 @@ def LogReader(filename: StringPathLike, **kwargs: Any) -> MessageReader: # noqa (optional, depends on `asammdf `_) * .trc :class:`can.TRCReader` - Gzip compressed files can be used as long as the original + Gzip and Zstd compressed files can be used as long as the original files suffix is one of the above (e.g. filename.asc.gz). @@ -123,7 +140,7 @@ def LogReader(filename: StringPathLike, **kwargs: Any) -> MessageReader: # noqa suffix = pathlib.PurePath(filename).suffix.lower() file_or_filename: AcceptedIOType = filename - if suffix == ".gz": + if suffix in (".gz", ".zst"): reader_type, file_or_filename = _decompress(filename) else: reader_type = _get_logger_for_suffix(suffix) diff --git a/can/socketcan_common.py b/can/socketcan_common.py new file mode 100644 index 000000000..1fc4b9f69 --- /dev/null +++ b/can/socketcan_common.py @@ -0,0 +1,222 @@ +""" +Defines shared SocketCAN methods and constants. +""" + +import logging +import struct +from typing import Tuple + +import can.util + +from .message import Message + +# Generic socket constants +SO_TIMESTAMPNS = 35 + +CAN_ERR_FLAG = 0x20000000 +CAN_RTR_FLAG = 0x40000000 +CAN_EFF_FLAG = 0x80000000 + +# BCM opcodes +CAN_BCM_TX_SETUP = 1 +CAN_BCM_TX_DELETE = 2 +CAN_BCM_TX_READ = 3 + +# BCM flags +SETTIMER = 0x0001 +STARTTIMER = 0x0002 +TX_COUNTEVT = 0x0004 +TX_ANNOUNCE = 0x0008 +TX_CP_CAN_ID = 0x0010 +RX_FILTER_ID = 0x0020 +RX_CHECK_DLC = 0x0040 +RX_NO_AUTOTIMER = 0x0080 +RX_ANNOUNCE_RESUME = 0x0100 +TX_RESET_MULTI_IDX = 0x0200 +RX_RTR_FRAME = 0x0400 +CAN_FD_FRAME = 0x0800 + +CAN_RAW = 1 +CAN_BCM = 2 + +SOL_CAN_BASE = 100 +SOL_CAN_RAW = SOL_CAN_BASE + CAN_RAW + +CAN_RAW_FILTER = 1 +CAN_RAW_ERR_FILTER = 2 +CAN_RAW_LOOPBACK = 3 +CAN_RAW_RECV_OWN_MSGS = 4 +CAN_RAW_FD_FRAMES = 5 + +MSK_ARBID = 0x1FFFFFFF +MSK_FLAGS = 0xE0000000 + +PF_CAN = 29 +SOCK_RAW = 3 +SOCK_DGRAM = 2 +AF_CAN = PF_CAN + +SIOCGIFNAME = 0x8910 +SIOCGIFINDEX = 0x8933 +SIOCGSTAMP = 0x8906 +EXTFLG = 0x0004 + +CANFD_BRS = 0x01 # bit rate switch (second bitrate for payload data) +CANFD_ESI = 0x02 # error state indicator of the transmitting node +CANFD_FDF = 0x04 # mark CAN FD for dual use of struct canfd_frame + +# CAN payload length and DLC definitions according to ISO 11898-1 +CAN_MAX_DLC = 8 +CAN_MAX_RAW_DLC = 15 +CAN_MAX_DLEN = 8 + +# CAN FD payload length and DLC definitions according to ISO 11898-7 +CANFD_MAX_DLC = 15 +CANFD_MAX_DLEN = 64 + +CANFD_MTU = 72 + +STD_ACCEPTANCE_MASK_ALL_BITS = 2**11 - 1 +MAX_11_BIT_ID = STD_ACCEPTANCE_MASK_ALL_BITS + +EXT_ACCEPTANCE_MASK_ALL_BITS = 2**29 - 1 +MAX_29_BIT_ID = EXT_ACCEPTANCE_MASK_ALL_BITS + +# struct module defines a binary packing format: +# https://docs.python.org/3/library/struct.html#struct-format-strings +# The 32bit can id is directly followed by the 8bit data link count +# The data field is aligned on an 8 byte boundary, hence we add padding +# which aligns the data field to an 8 byte boundary. + +# host-endian for communication with kernel +CAN_FRAME_HEADER_STRUCT = struct.Struct("=IBB1xB") +# big-endian for pcapng +CAN_FRAME_HEADER_STRUCT_BE = struct.Struct(">IBB1xB") + + +log = logging.getLogger(__name__) + + +def parse_can_frame( + cf: bytes, structure: struct.Struct = CAN_FRAME_HEADER_STRUCT +) -> Message: + """Parse a CAN frame. + + :param cf: A CAN frame in socketcan format + :return: A :class:`~can.Message` object with the parsed data + """ + can_id, can_dlc, flags, data = dissect_can_frame(cf, structure) + + # EXT, RTR, ERR flags -> boolean attributes + # /* special address description flags for the CAN_ID */ + # #define CAN_EFF_FLAG 0x80000000U /* EFF/SFF is set in the MSB */ + # #define CAN_RTR_FLAG 0x40000000U /* remote transmission request */ + # #define CAN_ERR_FLAG 0x20000000U /* error frame */ + is_extended_frame_format = bool(can_id & CAN_EFF_FLAG) + is_remote_transmission_request = bool(can_id & CAN_RTR_FLAG) + is_error_frame = bool(can_id & CAN_ERR_FLAG) + is_fd = len(cf) == CANFD_MTU + bitrate_switch = bool(flags & CANFD_BRS) + error_state_indicator = bool(flags & CANFD_ESI) + + if is_extended_frame_format: + # log.debug("CAN: Extended") + # TODO does this depend on SFF or EFF? + arbitration_id = can_id & 0x1FFFFFFF + else: + # log.debug("CAN: Standard") + arbitration_id = can_id & 0x000007FF + + return Message( + arbitration_id=arbitration_id, + is_extended_id=is_extended_frame_format, + is_remote_frame=is_remote_transmission_request, + is_error_frame=is_error_frame, + is_fd=is_fd, + bitrate_switch=bitrate_switch, + error_state_indicator=error_state_indicator, + dlc=can_dlc, + data=data, + ) + + +def _compose_arbitration_id(message: Message) -> int: + can_id = message.arbitration_id + if message.is_extended_id: + log.debug("sending an extended id type message") + can_id |= CAN_EFF_FLAG + if message.is_remote_frame: + log.debug("requesting a remote frame") + can_id |= CAN_RTR_FLAG + if message.is_error_frame: + log.debug("sending error frame") + can_id |= CAN_ERR_FLAG + return can_id + + +def build_can_frame( + msg: Message, structure: struct.Struct = CAN_FRAME_HEADER_STRUCT +) -> bytes: + """CAN frame packing (see 'struct can_frame' in ) + + :param msg: A :class:`~can.Message` object to convert to a CAN frame + :return: A CAN frame in socketcan format + """ + + can_id = _compose_arbitration_id(msg) + + flags = 0 + + # The socketcan code identify the received FD frame by the packet length. + # So, padding to the data length is performed according to the message type (Classic / FD) + if msg.is_fd: + flags |= CANFD_FDF + max_len = CANFD_MAX_DLEN + else: + max_len = CAN_MAX_DLEN + + if msg.bitrate_switch: + flags |= CANFD_BRS + if msg.error_state_indicator: + flags |= CANFD_ESI + + data = bytes(msg.data).ljust(max_len, b"\x00") + + if msg.is_remote_frame: + data_len = msg.dlc + else: + data_len = min(i for i in can.util.CAN_FD_DLC if i >= len(msg.data)) + header = structure.pack(can_id, data_len, flags, msg.dlc) + return header + data + + +def is_frame_fd(frame: bytes) -> bool: + # According to the SocketCAN implementation the frame length + # should indicate if the message is FD or not (not the flag value) + return len(frame) == CANFD_MTU + + +def dissect_can_frame( + frame: bytes, structure: struct.Struct = CAN_FRAME_HEADER_STRUCT +) -> Tuple[int, int, int, bytes]: + """Dissect a CAN frame into its components. + + :param frame: A CAN frame in socketcan format + :return: Tuple of (CAN ID, CAN DLC, flags, data) + """ + + can_id, data_len, flags, len8_dlc = structure.unpack_from(frame) + + if data_len not in can.util.CAN_FD_DLC: + data_len = min(i for i in can.util.CAN_FD_DLC if i >= data_len) + + can_dlc = data_len + + if not is_frame_fd(frame): + # Flags not valid in non-FD frames + flags = 0 + + if data_len == CAN_MAX_DLEN and CAN_MAX_DLEN < len8_dlc <= CAN_MAX_RAW_DLC: + can_dlc = len8_dlc + + return can_id, can_dlc, flags, frame[8 : 8 + data_len] diff --git a/pyproject.toml b/pyproject.toml index c65275559..601a41a97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,8 @@ viewer = [ "windows-curses; platform_system == 'Windows' and platform_python_implementation=='CPython'" ] mf4 = ["asammdf>=6.0.0"] +pcapng = ["python-pcapng>=2.1.1"] +zstd = ["pyzstd>=0.16.2"] [tool.setuptools.dynamic] readme = { file = "README.rst" } diff --git a/test/logformats_test.py b/test/logformats_test.py index f3fe485b2..61015026b 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -44,6 +44,11 @@ except ModuleNotFoundError: asammdf = None +try: + import pcapng +except ModuleNotFoundError: + pcapng = None + @contextmanager def override_locale(category: int, locale_str: str) -> None: @@ -103,6 +108,13 @@ def test_extension_matching_mf4(self): if asammdf is not None: raise + def test_extension_matching_pcapng(self): + try: + self._test_extension(".pcapng") + except NotImplementedError: + if pcapng is not None: + raise + class ReaderWriterTest(unittest.TestCase, ComparingMessagesTestCase, metaclass=ABCMeta): """Tests a pair of writer and reader by writing all data first and @@ -859,6 +871,28 @@ def _setup_instance(self): ) +@unittest.skipIf(pcapng is None, "pcapng is unavailable") +class TestPcapngFileFormat(ReaderWriterTest): + """Tests can.PcapngWriter and can.PcapngReader""" + + def _setup_instance(self): + super()._setup_instance_helper( + can.PcapngWriter, + can.PcapngReader, + binary_file=True, + check_remote_frames=True, + check_error_frames=True, + check_fd=True, + check_comments=False, + test_append=True, + # default unit is nanoseconds, yet float causes smaller tolerance to fail + allowed_timestamp_delta=1e-6, + # It preserves channels, however adds_default_channel doesn't work properly + preserves_channel=False, + adds_default_channel="can0", + ) + + class TestSqliteDatabaseFormat(ReaderWriterTest): """Tests can.SqliteWriter and can.SqliteReader""" diff --git a/test/test_socketcan.py b/test/test_socketcan.py index af06b8169..6b1da311c 100644 --- a/test/test_socketcan.py +++ b/test/test_socketcan.py @@ -10,7 +10,7 @@ from unittest.mock import patch import can -from can.interfaces.socketcan.constants import ( +from can.socketcan_common import ( CAN_BCM_TX_DELETE, CAN_BCM_TX_SETUP, SETTIMER, diff --git a/tox.ini b/tox.ini index e112c22b4..4b915f60c 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ deps = parameterized~=0.8 asammdf>=6.0; platform_python_implementation=="CPython" and python_version<"3.13" pywin32>=305; platform_system=="Windows" and platform_python_implementation=="CPython" and python_version<"3.14" + python-pcapng>=2.1.1 commands = pytest {posargs}