| import itertools |
| import io |
| import json |
| import os |
| import platform |
| import sys |
| import tempfile |
| import threading |
| import time |
| |
| from .collector import Collector, filter_internal_frames |
| from .opcode_utils import get_opcode_info, format_opcode |
| try: |
| from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION, THREAD_STATUS_MAIN_THREAD |
| except ImportError: |
| # Fallback if module not available (shouldn't happen in normal use) |
| THREAD_STATUS_HAS_GIL = (1 << 0) |
| THREAD_STATUS_ON_CPU = (1 << 1) |
| THREAD_STATUS_UNKNOWN = (1 << 2) |
| THREAD_STATUS_GIL_REQUESTED = (1 << 3) |
| THREAD_STATUS_HAS_EXCEPTION = (1 << 4) |
| THREAD_STATUS_MAIN_THREAD = (1 << 5) |
| |
| |
| # Categories matching Firefox Profiler expectations |
| GECKO_CATEGORIES = [ |
| {"name": "Other", "color": "grey", "subcategories": ["Other"]}, |
| {"name": "Python", "color": "yellow", "subcategories": ["Other"]}, |
| {"name": "Native", "color": "blue", "subcategories": ["Other"]}, |
| {"name": "GC", "color": "orange", "subcategories": ["Other"]}, |
| {"name": "GIL", "color": "green", "subcategories": ["Other"]}, |
| {"name": "CPU", "color": "purple", "subcategories": ["Other"]}, |
| {"name": "Code Type", "color": "red", "subcategories": ["Other"]}, |
| {"name": "Opcodes", "color": "magenta", "subcategories": ["Other"]}, |
| {"name": "Exception", "color": "lightblue", "subcategories": ["Other"]}, |
| ] |
| |
| # Category indices |
| CATEGORY_OTHER = 0 |
| CATEGORY_PYTHON = 1 |
| CATEGORY_NATIVE = 2 |
| CATEGORY_GC = 3 |
| CATEGORY_GIL = 4 |
| CATEGORY_CPU = 5 |
| CATEGORY_CODE_TYPE = 6 |
| CATEGORY_OPCODES = 7 |
| CATEGORY_EXCEPTION = 8 |
| |
| # Subcategory indices |
| DEFAULT_SUBCATEGORY = 0 |
| |
| GECKO_FORMAT_VERSION = 32 |
| GECKO_PREPROCESSED_VERSION = 57 |
| |
| # Resource type constants |
| RESOURCE_TYPE_LIBRARY = 1 |
| |
| # Frame constants |
| FRAME_ADDRESS_NONE = -1 |
| FRAME_INLINE_DEPTH_ROOT = 0 |
| |
| # Process constants |
| PROCESS_TYPE_MAIN = 0 |
| STACKWALK_DISABLED = 0 |
| |
| # In-memory buffer before spilling to disk |
| DEFAULT_SPILL_BUFFER_BYTES = 128 * 1024 |
| _JSON_SEPARATORS = (",", ":") |
| _JSON_ENCODER = json.JSONEncoder( |
| separators=_JSON_SEPARATORS, allow_nan=False |
| ) |
| |
| |
| class SpillColumn: |
| def __init__(self, directory, basename, *, |
| buffer_bytes=None): |
| self.path = os.path.join(directory, basename) |
| self.buffer = bytearray() |
| self._buffer_bytes = ( |
| DEFAULT_SPILL_BUFFER_BYTES if buffer_bytes is None |
| else buffer_bytes |
| ) |
| |
| def append(self, value): |
| self.buffer += (_JSON_ENCODER.encode(value) + "\n").encode("utf-8") |
| if len(self.buffer) >= self._buffer_bytes: |
| self.flush() |
| |
| def flush(self): |
| with open(self.path, "ab") as file: |
| file.write(self.buffer) |
| self.buffer.clear() |
| |
| def iter_tokens(self): |
| with open(self.path, encoding="utf-8") as file: |
| for line in file: |
| yield line.rstrip("\n") |
| |
| |
| class GeckoThreadSpill: |
| _COLUMNS = ( |
| ("samples_stack", "samples-stack.json"), |
| ("samples_time", "samples-time.json"), |
| ("markers_name", "markers-name.json"), |
| ("markers_start_time", "markers-start-time.json"), |
| ("markers_end_time", "markers-end-time.json"), |
| ("markers_phase", "markers-phase.json"), |
| ("markers_category", "markers-category.json"), |
| ("markers_data", "markers-data.json"), |
| ) |
| |
| def __init__(self, directory, tid): |
| prefix = f"thread-{tid}-" |
| for attr, basename in self._COLUMNS: |
| setattr(self, attr, SpillColumn(directory, prefix + basename)) |
| self.sample_count = 0 |
| self.marker_count = 0 |
| |
| def append_sample(self, stack_index, time_ms): |
| self.samples_stack.append(stack_index) |
| self.samples_time.append(time_ms) |
| self.sample_count += 1 |
| |
| def append_marker(self, name_idx, start_time, end_time, phase, category, data): |
| self.markers_name.append(name_idx) |
| self.markers_start_time.append(start_time) |
| self.markers_end_time.append(end_time) |
| self.markers_phase.append(phase) |
| self.markers_category.append(category) |
| self.markers_data.append(data) |
| self.marker_count += 1 |
| |
| def prepare_read(self): |
| for attr, _basename in self._COLUMNS: |
| getattr(self, attr).flush() |
| |
| |
| class GeckoCollector(Collector): |
| aggregating = True |
| |
| def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): |
| self.sample_interval_usec = sample_interval_usec |
| self.skip_idle = skip_idle |
| self.opcodes_enabled = opcodes |
| self.start_time = time.monotonic() * 1000 # milliseconds since start |
| |
| # Global string table (shared across all threads) |
| self.global_strings = ["(root)"] # Start with root |
| self.global_string_map = {"(root)": 0} |
| |
| # Per-thread data structures |
| self.threads = {} # tid -> thread data |
| self.spill_dir = None |
| self.exported = False |
| |
| # Global tables |
| self.libs = [] |
| |
| # Sampling interval tracking |
| self.sample_count = 0 |
| self.last_sample_time = 0 |
| self.interval = 1.0 # Will be calculated from actual sampling |
| |
| # State tracking for interval markers (tid -> start_time) |
| self.has_gil_start = {} # Thread has the GIL |
| self.no_gil_start = {} # Thread doesn't have the GIL |
| self.on_cpu_start = {} # Thread is running on CPU |
| self.off_cpu_start = {} # Thread is off CPU |
| self.python_code_start = {} # Thread running Python code (has GIL) |
| self.native_code_start = {} # Thread running native code (on CPU without GIL) |
| self.gil_wait_start = {} # Thread waiting for GIL |
| self.exception_start = {} # Thread has an exception set |
| self.no_exception_start = {} # Thread has no exception set |
| |
| # GC event tracking: track GC start time per thread |
| self.gc_start_per_thread = {} # tid -> start_time |
| |
| # Track which threads have been initialized for state tracking |
| self.initialized_threads = set() |
| |
| # Opcode state tracking per thread: tid -> (opcode, lineno, col_offset, funcname, filename, start_time) |
| self.opcode_state = {} |
| |
| # For binary replay: track base timestamp (first sample's timestamp) |
| self._replay_base_timestamp_us = None |
| |
| def _track_state_transition(self, tid, condition, active_dict, inactive_dict, |
| active_name, inactive_name, category, current_time): |
| """Track binary state transitions and emit markers. |
| |
| Args: |
| tid: Thread ID |
| condition: Whether the active state is true |
| active_dict: Dict tracking start time of active state |
| inactive_dict: Dict tracking start time of inactive state |
| active_name: Name for active state marker |
| inactive_name: Name for inactive state marker |
| category: Gecko category for the markers |
| current_time: Current timestamp |
| """ |
| # On first observation of a thread, just record the current state |
| # without creating a marker (we don't know what the previous state was) |
| if tid not in self.initialized_threads: |
| if condition: |
| active_dict[tid] = current_time |
| else: |
| inactive_dict[tid] = current_time |
| return |
| |
| # For already-initialized threads, track transitions |
| if condition: |
| active_dict.setdefault(tid, current_time) |
| if tid in inactive_dict: |
| self._add_marker(tid, inactive_name, inactive_dict.pop(tid), |
| current_time, category) |
| else: |
| inactive_dict.setdefault(tid, current_time) |
| if tid in active_dict: |
| self._add_marker(tid, active_name, active_dict.pop(tid), |
| current_time, category) |
| |
| def collect(self, stack_frames, timestamps_us=None): |
| """Collect samples from stack frames. |
| |
| Args: |
| stack_frames: List of interpreter/thread frame info |
| timestamps_us: List of timestamps in microseconds (None for live sampling) |
| """ |
| if self.exported: |
| raise RuntimeError("cannot append to GeckoCollector after export") |
| |
| # Handle live sampling (no timestamps provided) |
| if timestamps_us is None: |
| current_time = (time.monotonic() * 1000) - self.start_time |
| times = [current_time] |
| else: |
| if not timestamps_us: |
| return |
| # Initialize base timestamp if needed |
| if self._replay_base_timestamp_us is None: |
| self._replay_base_timestamp_us = timestamps_us[0] |
| # Convert all timestamps to times (ms relative to first sample) |
| base = self._replay_base_timestamp_us |
| times = [(ts - base) / 1000 for ts in timestamps_us] |
| |
| first_time = times[0] |
| |
| # Update interval calculation |
| if self.sample_count > 0 and self.last_sample_time > 0: |
| self.interval = (times[-1] - self.last_sample_time) / self.sample_count |
| self.last_sample_time = times[-1] |
| |
| # Process threads |
| for interpreter_info in stack_frames: |
| for thread_info in interpreter_info.threads: |
| frames = filter_internal_frames(thread_info.frame_info) |
| tid = thread_info.thread_id |
| status_flags = thread_info.status |
| is_main_thread = bool(status_flags & THREAD_STATUS_MAIN_THREAD) |
| |
| # Initialize thread if needed |
| if tid not in self.threads: |
| self.threads[tid] = self._create_thread(tid, is_main_thread) |
| |
| thread_data = self.threads[tid] |
| |
| # Decode status flags |
| has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) |
| on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) |
| gil_requested = bool(status_flags & THREAD_STATUS_GIL_REQUESTED) |
| |
| # Track state transitions using first timestamp |
| self._track_state_transition( |
| tid, has_gil, self.has_gil_start, self.no_gil_start, |
| "Has GIL", "No GIL", CATEGORY_GIL, first_time |
| ) |
| self._track_state_transition( |
| tid, on_cpu, self.on_cpu_start, self.off_cpu_start, |
| "On CPU", "Off CPU", CATEGORY_CPU, first_time |
| ) |
| |
| # Track code type |
| if has_gil: |
| self._track_state_transition( |
| tid, True, self.python_code_start, self.native_code_start, |
| "Python Code", "Native Code", CATEGORY_CODE_TYPE, first_time |
| ) |
| elif on_cpu: |
| self._track_state_transition( |
| tid, True, self.native_code_start, self.python_code_start, |
| "Native Code", "Python Code", CATEGORY_CODE_TYPE, first_time |
| ) |
| else: |
| if tid in self.initialized_threads: |
| if tid in self.python_code_start: |
| self._add_marker(tid, "Python Code", self.python_code_start.pop(tid), |
| first_time, CATEGORY_CODE_TYPE) |
| if tid in self.native_code_start: |
| self._add_marker(tid, "Native Code", self.native_code_start.pop(tid), |
| first_time, CATEGORY_CODE_TYPE) |
| |
| # Track GIL wait |
| if gil_requested: |
| self.gil_wait_start.setdefault(tid, first_time) |
| elif tid in self.gil_wait_start: |
| self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), |
| first_time, CATEGORY_GIL) |
| |
| # Track exception state |
| has_exception = bool(status_flags & THREAD_STATUS_HAS_EXCEPTION) |
| self._track_state_transition( |
| tid, has_exception, self.exception_start, self.no_exception_start, |
| "Has Exception", "No Exception", CATEGORY_EXCEPTION, first_time |
| ) |
| |
| # Track GC events |
| has_gc_frame = any(frame[2] == "<GC>" for frame in frames) |
| if has_gc_frame: |
| if tid not in self.gc_start_per_thread: |
| self.gc_start_per_thread[tid] = first_time |
| elif tid in self.gc_start_per_thread: |
| self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), |
| first_time, CATEGORY_GC) |
| |
| # Mark thread as initialized |
| self.initialized_threads.add(tid) |
| |
| # Skip idle threads if requested |
| is_idle = not has_gil and not on_cpu |
| if self.skip_idle and is_idle: |
| continue |
| |
| if not frames: |
| continue |
| |
| # Process stack once to get stack_index |
| stack_index = self._process_stack(thread_data, frames) |
| |
| # Add samples with timestamps |
| thread_spill = thread_data["_spill"] |
| for t in times: |
| thread_spill.append_sample(stack_index, t) |
| |
| # Handle opcodes |
| if self.opcodes_enabled and frames: |
| leaf_frame = frames[0] |
| filename, location, funcname, opcode = leaf_frame |
| if isinstance(location, tuple): |
| lineno, _, col_offset, _ = location |
| else: |
| lineno = location |
| col_offset = -1 |
| |
| current_state = (opcode, lineno, col_offset, funcname, filename) |
| |
| if tid not in self.opcode_state: |
| self.opcode_state[tid] = (*current_state, first_time) |
| elif self.opcode_state[tid][:5] != current_state: |
| prev_opcode, prev_lineno, prev_col, prev_funcname, prev_filename, prev_start = self.opcode_state[tid] |
| self._add_opcode_interval_marker( |
| tid, prev_opcode, prev_lineno, prev_col, prev_funcname, prev_start, first_time |
| ) |
| self.opcode_state[tid] = (*current_state, first_time) |
| |
| self.sample_count += len(times) |
| |
| def _create_thread(self, tid, is_main_thread): |
| """Create a new thread structure with processed profile format.""" |
| if self.spill_dir is None: |
| self.spill_dir = tempfile.TemporaryDirectory() |
| |
| thread = { |
| "name": f"Thread-{tid}", |
| "isMainThread": is_main_thread, |
| "processStartupTime": 0, |
| "processShutdownTime": None, |
| "registerTime": 0, |
| "unregisterTime": None, |
| "pausedRanges": [], |
| "pid": str(os.getpid()), |
| "tid": tid, |
| "processType": "default", |
| "processName": "Python Process", |
| # Stack table - processed format |
| "stackTable": { |
| "frame": [], |
| "category": [], |
| "subcategory": [], |
| "prefix": [], |
| "length": 0, # Will be updated on export |
| }, |
| # Frame table - processed format |
| "frameTable": { |
| "address": [], |
| "category": [], |
| "subcategory": [], |
| "func": [], |
| "innerWindowID": [], |
| "implementation": [], |
| "optimizations": [], |
| "line": [], |
| "column": [], |
| "inlineDepth": [], |
| "nativeSymbol": [], |
| "length": 0, # Will be updated on export |
| }, |
| # Function table - processed format |
| "funcTable": { |
| "name": [], |
| "isJS": [], |
| "relevantForJS": [], |
| "resource": [], |
| "fileName": [], |
| "lineNumber": [], |
| "columnNumber": [], |
| "length": 0, # Will be updated on export |
| }, |
| # Resource table - processed format |
| "resourceTable": { |
| "lib": [], |
| "name": [], |
| "host": [], |
| "type": [], |
| "length": 0, # Will be updated on export |
| }, |
| # Native symbols table (empty for Python) |
| "nativeSymbols": { |
| "libIndex": [], |
| "address": [], |
| "name": [], |
| "functionSize": [], |
| "length": 0, |
| }, |
| # Caches for deduplication |
| "_stackCache": {}, |
| "_frameCache": {}, |
| "_funcCache": {}, |
| "_resourceCache": {}, |
| "_spill": GeckoThreadSpill(self.spill_dir.name, tid), |
| } |
| |
| return thread |
| |
| def _is_python(self, filename: str) -> bool: |
| return not filename.startswith("<") or filename in ["<stdin>", "<string>"] |
| |
| def _get_category(self, filename: str) -> int: |
| return CATEGORY_PYTHON if self._is_python(filename) else CATEGORY_NATIVE |
| |
| def _intern_string(self, s): |
| """Intern a string in the global string table.""" |
| if s in self.global_string_map: |
| return self.global_string_map[s] |
| idx = len(self.global_strings) |
| self.global_strings.append(s) |
| self.global_string_map[s] = idx |
| return idx |
| |
| def _add_marker(self, tid, name, start_time, end_time, category): |
| """Add an interval marker for a specific thread.""" |
| if tid not in self.threads: |
| return |
| |
| duration = end_time - start_time |
| |
| name_idx = self._intern_string(name) |
| self.threads[tid]["_spill"].append_marker( |
| name_idx, start_time, end_time, 1, category, { |
| "type": name.replace(" ", ""), |
| "duration": duration, |
| "tid": tid, |
| } |
| ) |
| |
| def _add_opcode_interval_marker(self, tid, opcode, lineno, col_offset, |
| funcname, start_time, end_time): |
| """Add an interval marker for opcode execution span.""" |
| if tid not in self.threads or opcode is None: |
| return |
| |
| opcode_info = get_opcode_info(opcode) |
| # Use formatted opcode name (with base opcode for specialized ones) |
| formatted_opname = format_opcode(opcode) |
| |
| name_idx = self._intern_string(formatted_opname) |
| |
| self.threads[tid]["_spill"].append_marker( |
| name_idx, start_time, end_time, 1, CATEGORY_OPCODES, { |
| "type": "Opcode", |
| "opcode": opcode, |
| "opname": formatted_opname, |
| "base_opname": opcode_info["base_opname"], |
| "is_specialized": opcode_info["is_specialized"], |
| "line": lineno, |
| "column": col_offset if col_offset >= 0 else None, |
| "function": funcname, |
| "duration": end_time - start_time, |
| } |
| ) |
| |
| def _process_stack(self, thread_data, frames): |
| """Process a stack and return the stack index.""" |
| if not frames: |
| return None |
| |
| # Cache references to avoid repeated dictionary lookups |
| stack_cache = thread_data["_stackCache"] |
| stack_table = thread_data["stackTable"] |
| stack_frames = stack_table["frame"] |
| stack_prefix = stack_table["prefix"] |
| stack_category = stack_table["category"] |
| stack_subcategory = stack_table["subcategory"] |
| |
| # Build stack bottom-up (from root to leaf) |
| prefix_stack_idx = None |
| |
| for frame_tuple in reversed(frames): |
| # frame_tuple is (filename, location, funcname, opcode) |
| # location is (lineno, end_lineno, col_offset, end_col_offset) or just lineno |
| filename, location, funcname, opcode = frame_tuple |
| if isinstance(location, tuple): |
| lineno, end_lineno, col_offset, end_col_offset = location |
| else: |
| # Legacy format: location is just lineno |
| lineno = location |
| col_offset = -1 |
| end_col_offset = -1 |
| |
| # Get or create function |
| func_idx = self._get_or_create_func( |
| thread_data, filename, funcname, lineno |
| ) |
| |
| # Get or create frame (include column for precise source location) |
| frame_idx = self._get_or_create_frame( |
| thread_data, func_idx, lineno, col_offset |
| ) |
| |
| # Check stack cache |
| stack_key = (frame_idx, prefix_stack_idx) |
| if stack_key in stack_cache: |
| prefix_stack_idx = stack_cache[stack_key] |
| else: |
| # Create new stack entry |
| stack_idx = len(stack_frames) |
| stack_frames.append(frame_idx) |
| stack_prefix.append(prefix_stack_idx) |
| |
| # Determine category |
| category = self._get_category(filename) |
| stack_category.append(category) |
| stack_subcategory.append(DEFAULT_SUBCATEGORY) |
| |
| stack_cache[stack_key] = stack_idx |
| prefix_stack_idx = stack_idx |
| |
| return prefix_stack_idx |
| |
| def _get_or_create_func(self, thread_data, filename, funcname, lineno): |
| """Get or create a function entry.""" |
| func_cache = thread_data["_funcCache"] |
| func_key = (filename, funcname) |
| |
| if func_key in func_cache: |
| return func_cache[func_key] |
| |
| # Cache references for func table |
| func_table = thread_data["funcTable"] |
| func_names = func_table["name"] |
| func_is_js = func_table["isJS"] |
| func_relevant = func_table["relevantForJS"] |
| func_resources = func_table["resource"] |
| func_filenames = func_table["fileName"] |
| func_line_numbers = func_table["lineNumber"] |
| func_column_numbers = func_table["columnNumber"] |
| |
| func_idx = len(func_names) |
| |
| # Intern strings in global table |
| name_idx = self._intern_string(funcname) |
| |
| # Determine if Python |
| is_python = self._is_python(filename) |
| |
| # Create resource |
| resource_idx = self._get_or_create_resource(thread_data, filename) |
| |
| # Add function |
| func_names.append(name_idx) |
| func_is_js.append(is_python) |
| func_relevant.append(is_python) |
| func_resources.append(resource_idx) |
| |
| if is_python: |
| filename_idx = self._intern_string(os.path.basename(filename)) |
| func_filenames.append(filename_idx) |
| func_line_numbers.append(lineno) |
| else: |
| func_filenames.append(None) |
| func_line_numbers.append(None) |
| func_column_numbers.append(None) |
| |
| func_cache[func_key] = func_idx |
| return func_idx |
| |
| def _get_or_create_resource(self, thread_data, filename): |
| """Get or create a resource entry.""" |
| resource_cache = thread_data["_resourceCache"] |
| |
| if filename in resource_cache: |
| return resource_cache[filename] |
| |
| # Cache references for resource table |
| resource_table = thread_data["resourceTable"] |
| resource_libs = resource_table["lib"] |
| resource_names = resource_table["name"] |
| resource_hosts = resource_table["host"] |
| resource_types = resource_table["type"] |
| |
| resource_idx = len(resource_names) |
| resource_name = ( |
| os.path.basename(filename) if "/" in filename else filename |
| ) |
| name_idx = self._intern_string(resource_name) |
| |
| resource_libs.append(None) |
| resource_names.append(name_idx) |
| resource_hosts.append(None) |
| resource_types.append(RESOURCE_TYPE_LIBRARY) |
| |
| resource_cache[filename] = resource_idx |
| return resource_idx |
| |
| def _get_or_create_frame(self, thread_data, func_idx, lineno, col_offset=-1): |
| """Get or create a frame entry.""" |
| frame_cache = thread_data["_frameCache"] |
| # Include column in cache key for precise frame identification |
| frame_key = (func_idx, lineno, col_offset if col_offset >= 0 else None) |
| |
| if frame_key in frame_cache: |
| return frame_cache[frame_key] |
| |
| # Cache references for frame table |
| frame_table = thread_data["frameTable"] |
| frame_addresses = frame_table["address"] |
| frame_inline_depths = frame_table["inlineDepth"] |
| frame_categories = frame_table["category"] |
| frame_subcategories = frame_table["subcategory"] |
| frame_funcs = frame_table["func"] |
| frame_native_symbols = frame_table["nativeSymbol"] |
| frame_inner_window_ids = frame_table["innerWindowID"] |
| frame_implementations = frame_table["implementation"] |
| frame_lines = frame_table["line"] |
| frame_columns = frame_table["column"] |
| frame_optimizations = frame_table["optimizations"] |
| |
| frame_idx = len(frame_funcs) |
| |
| # Determine category based on function - use cached func table reference |
| is_python = thread_data["funcTable"]["isJS"][func_idx] |
| category = CATEGORY_PYTHON if is_python else CATEGORY_NATIVE |
| |
| frame_addresses.append(FRAME_ADDRESS_NONE) |
| frame_inline_depths.append(FRAME_INLINE_DEPTH_ROOT) |
| frame_categories.append(category) |
| frame_subcategories.append(DEFAULT_SUBCATEGORY) |
| frame_funcs.append(func_idx) |
| frame_native_symbols.append(None) |
| frame_inner_window_ids.append(None) |
| frame_implementations.append(None) |
| frame_lines.append(lineno if lineno else None) |
| # Store column offset if available (>= 0), otherwise None |
| frame_columns.append(col_offset if col_offset >= 0 else None) |
| frame_optimizations.append(None) |
| |
| frame_cache[frame_key] = frame_idx |
| return frame_idx |
| |
| def _finalize_markers(self): |
| """Close any open markers at the end of profiling.""" |
| end_time = self.last_sample_time |
| |
| # Close all open markers for each thread using a generic approach |
| marker_states = [ |
| (self.has_gil_start, "Has GIL", CATEGORY_GIL), |
| (self.no_gil_start, "No GIL", CATEGORY_GIL), |
| (self.on_cpu_start, "On CPU", CATEGORY_CPU), |
| (self.off_cpu_start, "Off CPU", CATEGORY_CPU), |
| (self.python_code_start, "Python Code", CATEGORY_CODE_TYPE), |
| (self.native_code_start, "Native Code", CATEGORY_CODE_TYPE), |
| (self.gil_wait_start, "Waiting for GIL", CATEGORY_GIL), |
| (self.gc_start_per_thread, "GC Collecting", CATEGORY_GC), |
| (self.exception_start, "Has Exception", CATEGORY_EXCEPTION), |
| (self.no_exception_start, "No Exception", CATEGORY_EXCEPTION), |
| ] |
| |
| for state_dict, marker_name, category in marker_states: |
| for tid in list(state_dict.keys()): |
| self._add_marker(tid, marker_name, state_dict[tid], end_time, category) |
| del state_dict[tid] |
| |
| # Close any open opcode markers |
| for tid, state in list(self.opcode_state.items()): |
| opcode, lineno, col_offset, funcname, filename, start_time = state |
| self._add_opcode_interval_marker(tid, opcode, lineno, col_offset, funcname, start_time, end_time) |
| self.opcode_state.clear() |
| |
| def export(self, filename): |
| """Export the profile to a Gecko JSON file.""" |
| if self.sample_count > 0 and self.last_sample_time > 0: |
| self.interval = self.last_sample_time / self.sample_count |
| |
| # Spinner for progress indication |
| spinner = itertools.cycle(['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']) |
| stop_spinner = threading.Event() |
| |
| def spin(): |
| message = 'Building Gecko profile...' |
| while not stop_spinner.is_set(): |
| sys.stderr.write(f'\r{next(spinner)} {message}') |
| sys.stderr.flush() |
| time.sleep(0.1) |
| # Clear the spinner line |
| sys.stderr.write('\r' + ' ' * (len(message) + 3) + '\r') |
| sys.stderr.flush() |
| |
| spinner_thread = threading.Thread(target=spin, daemon=True) |
| spinner_thread.start() |
| |
| temp_path = None |
| replaced = False |
| try: |
| self._prepare_for_serialization() |
| output_dir = os.path.dirname(os.path.abspath(filename)) or "." |
| with tempfile.NamedTemporaryFile( |
| "w", dir=output_dir, delete=False |
| ) as file: |
| temp_path = file.name |
| self._stream_profile(file) |
| os.replace(temp_path, filename) |
| replaced = True |
| finally: |
| self.exported = True |
| stop_spinner.set() |
| spinner_thread.join(timeout=1.0) |
| # Small delay to ensure the clear happens |
| time.sleep(0.01) |
| if temp_path is not None and not replaced: |
| try: |
| os.unlink(temp_path) |
| except FileNotFoundError: |
| pass |
| self._cleanup_spills() |
| |
| print(f"Gecko profile written to {filename}") |
| print( |
| f"Open in Firefox Profiler: https://profiler.firefox.com/" |
| ) |
| |
| def _build_marker_schema(self): |
| """Build marker schema definitions for Firefox Profiler.""" |
| schema = [] |
| |
| # Opcode marker schema (only if opcodes enabled) |
| if self.opcodes_enabled: |
| schema.append({ |
| "name": "Opcode", |
| "display": ["marker-table", "marker-chart"], |
| "tooltipLabel": "{marker.data.opname}", |
| "tableLabel": "{marker.data.opname} at line {marker.data.line}", |
| "chartLabel": "{marker.data.opname}", |
| "fields": [ |
| {"key": "opname", "label": "Opcode", "format": "string", "searchable": True}, |
| {"key": "base_opname", "label": "Base Opcode", "format": "string"}, |
| {"key": "is_specialized", "label": "Specialized", "format": "string"}, |
| {"key": "line", "label": "Line", "format": "integer"}, |
| {"key": "column", "label": "Column", "format": "integer"}, |
| {"key": "function", "label": "Function", "format": "string"}, |
| {"key": "duration", "label": "Duration", "format": "duration"}, |
| ], |
| }) |
| |
| return schema |
| |
| def _build_profile(self): |
| """Build the complete profile structure in processed format.""" |
| try: |
| self._prepare_for_serialization() |
| file = io.StringIO() |
| self._stream_profile(file) |
| return json.loads(file.getvalue()) |
| finally: |
| self.exported = True |
| self._cleanup_spills() |
| |
| def _profile_head(self): |
| return { |
| "meta": { |
| "interval": self.interval, |
| "startTime": self.start_time, |
| "abi": platform.machine(), |
| "misc": "Python profiler", |
| "oscpu": platform.machine(), |
| "platform": platform.system(), |
| "processType": PROCESS_TYPE_MAIN, |
| "categories": GECKO_CATEGORIES, |
| "stackwalk": STACKWALK_DISABLED, |
| "toolkit": "", |
| "version": GECKO_FORMAT_VERSION, |
| "preprocessedProfileVersion": GECKO_PREPROCESSED_VERSION, |
| "appBuildID": "", |
| "physicalCPUs": os.cpu_count() or 0, |
| "logicalCPUs": os.cpu_count() or 0, |
| "CPUName": "", |
| "product": "Python", |
| "symbolicated": True, |
| "markerSchema": self._build_marker_schema(), |
| "importedFrom": "Tachyon Sampling Profiler", |
| "extensions": { |
| "id": [], |
| "name": [], |
| "baseURL": [], |
| "length": 0, |
| }, |
| }, |
| "libs": self.libs, |
| } |
| |
| def _profile_tail(self): |
| return { |
| "pages": [], |
| "shared": { |
| "stringArray": self.global_strings, |
| "sources": {"length": 0, "uuid": [], "filename": []}, |
| }, |
| } |
| |
| def _prepare_for_serialization(self): |
| if self.exported: |
| raise RuntimeError("GeckoCollector has already been exported") |
| self._finalize_markers() |
| for thread_data in self.threads.values(): |
| thread_data["_spill"].prepare_read() |
| thread_data["stackTable"]["length"] = len(thread_data["stackTable"]["frame"]) |
| thread_data["frameTable"]["length"] = len(thread_data["frameTable"]["func"]) |
| thread_data["funcTable"]["length"] = len(thread_data["funcTable"]["name"]) |
| thread_data["resourceTable"]["length"] = len(thread_data["resourceTable"]["name"]) |
| |
| def _cleanup_spills(self): |
| if self.spill_dir is not None: |
| self.spill_dir.cleanup() |
| self.spill_dir = None |
| |
| def _stream_profile(self, file): |
| file.write("{") |
| first = True |
| for key, value in self._profile_head().items(): |
| first = _write_json_member(file, key, value, first) |
| |
| first = _write_member_name(file, "threads", first) |
| file.write("[") |
| for index, (tid, thread_data) in enumerate(self.threads.items()): |
| if index: |
| file.write(",") |
| self._stream_thread(file, tid, thread_data) |
| file.write("]") |
| |
| for key, value in self._profile_tail().items(): |
| first = _write_json_member(file, key, value, first) |
| file.write("}") |
| |
| def _stream_thread(self, file, tid, thread_data): |
| spill = thread_data["_spill"] |
| metadata = { |
| "name": thread_data["name"], |
| "isMainThread": thread_data["isMainThread"], |
| "processStartupTime": thread_data["processStartupTime"], |
| "processShutdownTime": thread_data["processShutdownTime"], |
| "registerTime": thread_data["registerTime"], |
| "unregisterTime": thread_data["unregisterTime"], |
| "pausedRanges": thread_data["pausedRanges"], |
| "pid": thread_data["pid"], |
| "tid": thread_data["tid"], |
| "processType": thread_data["processType"], |
| "processName": thread_data["processName"], |
| } |
| file.write("{") |
| first = True |
| for key, value in metadata.items(): |
| first = _write_json_member(file, key, value, first) |
| |
| first = _write_member_name(file, "samples", first) |
| self._stream_samples(file, spill) |
| for key in ( |
| "stackTable", |
| "frameTable", |
| "funcTable", |
| "resourceTable", |
| "nativeSymbols", |
| ): |
| first = _write_json_member(file, key, thread_data[key], first) |
| first = _write_member_name(file, "markers", first) |
| self._stream_markers(file, spill) |
| file.write("}") |
| |
| def _stream_samples(self, file, spill): |
| _stream_column_table( |
| file, |
| ( |
| ("stack", spill.samples_stack.iter_tokens()), |
| ("time", spill.samples_time.iter_tokens()), |
| ("eventDelay", ("null" for _ in range(spill.sample_count))), |
| ), |
| spill.sample_count, |
| ( |
| ("weight", None), |
| ("weightType", "samples"), |
| ("length", spill.sample_count), |
| ), |
| ) |
| |
| def _stream_markers(self, file, spill): |
| _stream_column_table( |
| file, |
| ( |
| ("data", spill.markers_data.iter_tokens()), |
| ("name", spill.markers_name.iter_tokens()), |
| ("startTime", spill.markers_start_time.iter_tokens()), |
| ("endTime", spill.markers_end_time.iter_tokens()), |
| ("phase", spill.markers_phase.iter_tokens()), |
| ("category", spill.markers_category.iter_tokens()), |
| ), |
| spill.marker_count, |
| (("length", spill.marker_count),), |
| ) |
| |
| |
| def _write_json(file, value): |
| for chunk in _JSON_ENCODER.iterencode(value): |
| file.write(chunk) |
| |
| |
| def _write_member_name(file, name, first): |
| if not first: |
| file.write(",") |
| _write_json(file, name) |
| file.write(":") |
| return False |
| |
| |
| def _write_json_member(file, name, value, first): |
| first = _write_member_name(file, name, first) |
| _write_json(file, value) |
| return first |
| |
| |
| def _stream_column_table(file, columns, expected_count, trailing_members=()): |
| file.write("{") |
| first = True |
| for name, token_iter in columns: |
| first = _write_member_name(file, name, first) |
| _stream_array(file, token_iter, expected_count, name) |
| for name, value in trailing_members: |
| first = _write_json_member(file, name, value, first) |
| file.write("}") |
| |
| |
| def _stream_array(file, token_iter, expected_count, label="array"): |
| file.write("[") |
| count = 0 |
| for token in token_iter: |
| if count: |
| file.write(",") |
| file.write(token) |
| count += 1 |
| if count != expected_count: |
| raise RuntimeError( |
| f"streamed {count} {label} items, expected {expected_count}" |
| ) |
| file.write("]") |
