Files

298 lines
11 KiB
Python

import dpkt
import time
import socket
from kaitaistruct import KaitaiStream, BytesIO
import sys
import time
from datetime import datetime
import random
import argparse
import os
import logging
import mmap
from tqdm import tqdm
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from multiprocessing import get_context, Pool, freeze_support, current_process
from collections import defaultdict, Counter, deque
from parser.utils.pcaplib import Writer
from scapy.all import *
load_layer("tls")
from parser.config import PAYLOAD_LEVEL_NUM, HEADER_LEVEL_NUM
from parser.config import BBHEADER_LEN, MPEG_TS_SYNC_BYTE, IP_HEADER_MIN_SIZE, GSE_HEADER_MIN_LEN
from parser.config import write_dir, plot_dir, logs_dir
def initialize_custom_logging_levels():
"""
Defines custom logging levels (PAYLOAD, HEADER) and adds convenience methods
to the logging.Logger class. This function should be called ONCE at
application startup to ensure custom levels are registered.
"""
# Check if levels are already added to prevent re-adding warnings/errors
# (though typically called once, this makes it robust)
if not hasattr(logging, 'PAYLOAD'): # Check for convenience attribute
logging.addLevelName(PAYLOAD_LEVEL_NUM, 'PAYLOAD')
# Add convenience method for PAYLOAD level
def payload(self, message, *args, **kws):
if self.isEnabledFor(PAYLOAD_LEVEL_NUM):
self._log(PAYLOAD_LEVEL_NUM, message, args, **kws)
logging.Logger.payload = payload
if not hasattr(logging, 'HEADER'): # Check for convenience attribute
logging.addLevelName(HEADER_LEVEL_NUM, 'HEADER')
# Add convenience method for HEADER level
def header(self, message, *args, **kws):
if self.isEnabledFor(HEADER_LEVEL_NUM):
self._log(HEADER_LEVEL_NUM, message, args, **kws)
logging.Logger.header = header
def parse_tls_cert(cert_bytes):
"""
Parse and print details from a DER-encoded TLS certificate.
:param cert_bytes: bytes - DER-encoded certificate data
"""
try:
cert = x509.load_der_x509_certificate(cert_bytes, backend=default_backend())
print("=== Certificate Details ===")
print("Subject: ", cert.subject.rfc4514_string())
print("Issuer: ", cert.issuer.rfc4514_string())
print("Serial Number: ", hex(cert.serial_number))
print("Valid From: ", cert.not_valid_before.isoformat())
print("Valid To: ", cert.not_valid_after.isoformat())
print("Signature Algo: ", cert.signature_algorithm_oid._name)
print("Public Key Type: ", cert.public_key().__class__.__name__)
print("Extensions:")
for ext in cert.extensions:
print(f" - {ext.oid._name if ext.oid._name else ext.oid.dotted_string}: {ext.value}")
except Exception as e:
print("Failed to parse certificate:", str(e))
def create_pcap_handler(filename):
pcap_file_writer = open(filename, 'wb')
pcap_writer = Writer()
pcap_writer.create_header(pcap_file_writer)
return pcap_file_writer, pcap_writer
def write_ip_packet_to_pcap(pcap_file_writer, pcap_writer, ip_packet_bytes):
# Capture the current time and packet length for PCAP writing
seconds_time = time.time()
dt = datetime.now()
packet_length = len(ip_packet_bytes)
# Create a packet tuple as expected by the PCAP writer
simple_packet = (
int(seconds_time), # timestamp seconds
dt.microsecond, # timestamp microseconds
packet_length, # captured length
packet_length, # original length
ip_packet_bytes # actual IP packet data
)
# Write the IP packet to the PCAP file
pcap_writer.write([simple_packet], pcap_file_writer)
def close_pcap_handler(pcap_file_writer):
if pcap_file_writer is not None:
pcap_file_writer.close()
def flip_bytes(bytearray_input):
bytearray_copy = bytearray_input.copy() # Create a shallow copy of the input
for i in range(0, len(bytearray_copy) - 1, 2):
bytearray_copy[i], bytearray_copy[i + 1] = bytearray_copy[i + 1], bytearray_copy[i]
return bytearray_copy
def crc32mpeg2(buf, crc=0xffffffff):
# https://stackoverflow.com/questions/69332500/how-can-calculate-mpeg2-crc32-in-python
for val in buf:
crc ^= val << 24
for _ in range(8):
crc = crc << 1 if (crc & 0x80000000) == 0 else (crc << 1) ^ 0x104c11db7
return crc
def create_file_logger(filename, level=logging.INFO):
"""
Creates a logger that writes log messages to the given filename.
"""
logger = logging.getLogger(filename)
logger.setLevel(level) # Set the desired log level
# Create a file handler that logs messages to the specified file
file_handler = logging.FileHandler(filename, mode='w') # Open in binary write mode
file_handler.setLevel(level) # Set the level for the handler
# Create a formatter and set it for the handler
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
# formatter = logging.Formatter('%(asctime)s - %(levelname)-7s - %(message)s')
file_handler.setFormatter(formatter)
# Add the handler to the logger
logger.addHandler(file_handler)
return logger
def open_file_writer(filename):
return open(filename, 'wb')
def close_file_writer(file_writer):
file_writer.close()
import matplotlib.pyplot as plt
def plot_skips(file_name, skips):
max_index = max(skips.values(), default=0)
plt.figure(figsize=(20, 3))
current_pos = 0
for start, end in sorted(skips.items()):
if start > current_pos:
plt.fill_between([current_pos, start], 0, 1, color='g')
plt.fill_between([start, end], 0, 1, color='r')
current_pos = end + 1
if current_pos <= max_index:
plt.fill_between([current_pos, max_index], 0, 1, color='g')
plt.title(f"Skipped Ranges in {os.path.basename(file_name)}")
plt.xlabel("Index in File")
plt.yticks([])
plt.xlim(0, max_index)
plt.ylim(0, 1)
plot_filename = f"{file_name}.png"
plt.savefig(plot_filename, bbox_inches='tight')
plt.close()
class ParserBase:
def __init__(self, read_file, protocol, show_pbar=False, log_level=logging.INFO):
self.read_file = os.path.basename(read_file)
self.protocol = protocol
self.show_pbar = show_pbar
self.log_level = log_level
self.plot_file = plot_dir + self.read_file + f".{protocol}"
self.protocol_file = write_dir + self.read_file + f".{protocol}"
self.skips_file = write_dir + self.read_file + f".not_{protocol}"
self.log_file = logs_dir + self.read_file + f".{protocol}.log"
self.write_protocol = open_file_writer(self.protocol_file)
self.write_skips = open_file_writer(self.skips_file)
self.logger = create_file_logger(self.log_file, level=log_level)
self.bytes_searched = 0
self.bytes_skipped = 0
# self.num_protocol = 0
self.skips = {}
self.parsers = []
pass
def reset(self):
self.bytes_searched = 0
self.bytes_skipped = 0
self.skips = {}
self.parsers = []
self.write_protocol = open_file_writer(self.protocol_file)
self.write_skips = open_file_writer(self.skips_file)
self.logger = create_file_logger(self.log_file, level=self.log_level)
def add_parser(self, parser):
self.parsers.append(parser)
def process_capture(self, capture):
pass
def process_capture_file(self, capture_file, preview_len=None):
try:
with open(capture_file, 'rb') as f:
if preview_len is not None:
file_size = f.seek(0, 2) # Seek to the end to get the file size
preview_len = min(preview_len, file_size)
f.seek(0) # Reset to the beginning of the file
with mmap.mmap(f.fileno(), preview_len, access=mmap.ACCESS_READ) as capture:
self.process_capture(capture)
else:
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as capture:
self.process_capture(capture)
except FileNotFoundError:
self.logger.info(f"Error: The file '{capture_file}' does not exist.")
return None # Return None if the file does not exist
except Exception as e:
self.logger.info(f"An unexpected error occurred: {e}")
return None # Return None in case of other errors
def done_processing(self):
close_file_writer(self.write_protocol)
close_file_writer(self.write_skips)
def plot_skips(self):
plot_skips(self.plot_file, self.skips)
def log_status(self, logger=None):
logger = self.logger if logger == None else logger
logger.info(f"After running the {self.protocol.upper()} Parser on {self.read_file}")
logger.info(f"{self.bytes_skipped} bytes were skipped in a {self.bytes_searched} byte search space")
if self.bytes_searched == 0:
logger.info(f"{self.protocol.upper()} Parser was initialized, but never called")
else:
logger.info(f"{((self.bytes_searched-self.bytes_skipped)/self.bytes_searched)*100}% of the capture is {self.protocol.upper()} compliant")
logger.debug(f"num_bytes_searched {self.bytes_searched}")
logger.debug(f"num_bytes_skipped {self.bytes_skipped}")
def get_compliance(self):
if self.bytes_searched == 0: return 0
return (self.bytes_searched-self.bytes_skipped)/self.bytes_searched
def can_parse(self):
if self.bytes_searched == 0:
return False
if self.num_gse_packets > 0:
if self.bytes_skipped == 0:
return True
self.logger.info(f"{(self.bytes_searched-self.bytes_skipped)/self.bytes_searched}")
if (self.bytes_searched - self.bytes_skipped)/self.bytes_searched >= .9999:
return True
return False
else:
return False
import os
import json
class ParserResults:
def __init__(self):
self.summary_file = "summary.json"
self.current_file = None
self.current_chain = []
def start_file(self, capture_file):
"""Begin recording a new file's parser chain."""
self.current_file = capture_file
self.current_chain = []
def add_parser(self, parser_name, compliance):
"""Append a parser stage to the current chain."""
if self.current_file is None:
raise RuntimeError("Call start_file() before adding parsers.")
self.current_chain.append({
"parser": parser_name,
"compliance": float(compliance)
})
def finalize_file(self):
"""Write the current file's chain as a JSON line to summary.json."""
if self.current_file is None:
return
record = {
"file": self.current_file,
"chain": self.current_chain
}
# Append as a single line (JSONL)
with open(self.summary_file, "a") as f:
f.write(json.dumps(record) + "\n")
self.current_file = None
self.current_chain = []