010 Editor Binary Template for ETL trace files

ETL.bt

//------------------------------------------------
//--- 010 Editor v9.0.2 Binary Template
//
//      File: ETL.bt
//   Authors: Zhentar
//   Version: 1.0
//   Purpose: Microsoft Event Tracing for Windows ETL file format
//  Category: Misc
// File Mask: *.etl
//   History: 
//   1.0    zhent: Initial Release
//------------------------------------------------
struct GUID {  DWORD Data1 <format=hex>;  ushort Data2 <format=hex>;  ushort Data3 <format=hex>;   uint64 Data4 <format=hex>; };

// ##########################################
// #         ETW Event Definitions          #
// ##########################################

// This should always be the very first event in an ETL file
struct TRACE_LOGFILE_HEADER { // https://docs.microsoft.com/en-us/windows/win32/etw/trace-logfile-header
  ulong   BufferSize;

  uchar   MajorVersion;
  uchar   MinorVersion;
  uchar   SubVersion;
  uchar   SubMinorVersion;
  
  ulong   ProviderVersion;
  ulong   NumberOfProcessors;
  int64   EndTime;
  ulong   TimerResolution;
  ulong   MaximumFileSize;
  ulong   LogFileMode;
  ulong   BuffersWritten;
  ulong   StartBuffers;
  ulong   PointerSize;
  ulong   EventsLost;
  ulong   CpuSpeedInMHz;
  int64   LoggerNamePtr;
  int64   LogFileNamePtr;
  byte    TimeZone[176]; //TIME_ZONE_INFORMATION
  int64   BootTime;
  int64   PerfFreq;
  int64   StartTime;
  ulong   ReservedFlags;
  ulong   BuffersLost;
  wstring LoggerName;
  wstring LogFileName;
};

typedef struct {
    ulong Masks[8];
} PERFINFO_GROUPMASK;

// ##########################################
// #      ETW Event Header Definitions      #
// ##########################################

enum <byte> HeaderType {
    TRACE_HEADER_TYPE_SYSTEM32       = 1,
    TRACE_HEADER_TYPE_SYSTEM64       = 2,
    TRACE_HEADER_TYPE_COMPACT32      = 3,
    TRACE_HEADER_TYPE_COMPACT64      = 4,
    TRACE_HEADER_TYPE_FULL_HEADER32  = 10,
    TRACE_HEADER_TYPE_INSTANCE32     = 11,
    TRACE_HEADER_TYPE_TIMED          = 12,  // Not used
    TRACE_HEADER_TYPE_ERROR          = 13,  // Error while logging event
    TRACE_HEADER_TYPE_WNODE_HEADER   = 14,  // Not used
    TRACE_HEADER_TYPE_MESSAGE        = 15,
    TRACE_HEADER_TYPE_PERFINFO32     = 16,
    TRACE_HEADER_TYPE_PERFINFO64     = 17,
    TRACE_HEADER_TYPE_EVENT_HEADER32 = 18,
    TRACE_HEADER_TYPE_EVENT_HEADER64 = 19,
    TRACE_HEADER_TYPE_FULL_HEADER64  = 20,
    TRACE_HEADER_TYPE_INSTANCE64     = 21
};

struct SYSTEM_TRACE_HEADER (byte isFullSize) {
    ushort Version; //Likely event version, 
    HeaderType  Type;
    uchar  MarkerFlags <format=hex>;
    ushort Size;   // total event size, including this header
    ushort HookId; // An identifier for the event type: https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/etw/callouts/hookid.htm

    ulong   ThreadId;
    ulong   ProcessId;
    int64   SystemTime;
    if(isFullSize) {
        ulong   KernelTime;
        ulong   UserTime;
    }
};

struct PERFINFO_TRACE_HEADER  {
    ushort Version; //Likely event version, 
    HeaderType  Type;
    uchar  MarkerFlags <format=hex>;
    ushort Size;   // total event size, including this header
    ushort HookId; // An identifier for the event type: https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/etw/callouts/hookid.htm

    int64   SystemTime;
};

struct EVENT_TRACE_HEADER {
    ushort  Size;                   // Size of entire record
    
    HeaderType  Type;               // Header type - internal use only
    uchar  MarkerFlags <format=hex>;// Marker - internal use only
    
    uchar   EventType;                   // event type
    uchar   Level;                  // trace instrumentation level
    ushort  Version;                // version of trace record
    ulong   ThreadId;               // Thread Id
    ulong   ProcessId;              // Process Id
    int64   TimeStamp;              // time when event happens
    GUID    Guid;                   // Guid that identifies event
    
    ulong   KernelTime;             // Kernel Mode CPU ticks
    ulong   UserTime;               // User mode CPU ticks
};

//this is just an EVENT_TRACE_HEADER with three id fields at the end. 
//Note that this header is the one used for user mode sourced events
struct EVENT_INSTANCE_GUID_HEADER {
    ushort  Size;                   // Size of entire record
    
    HeaderType  Type;               // Header type - internal use only
    uchar  MarkerFlags <format=hex>;// Marker - internal use only
    
    uchar   EventType;              // event type
    uchar   Level;                  // trace instrumentation level
    ushort  Version;                // version of trace record
    ulong   ThreadId;               // Thread Id
    ulong   ProcessId;              // Process Id
    int64   TimeStamp;              // time when event happens
    GUID    Guid;                   // Guid that identifies event
    
    ulong   KernelTime;             // Kernel Mode CPU ticks
    ulong   UserTime;               // User mode CPU ticks
    ulong   InstanceId;
    ulong   ParentInstanceId;
    GUID    ParentGuid;             // Guid that identifies event
};

// This header is the official public API header
// when consuming events with ProcessTrace, all other headers get transformed to this behind the scenes
struct EVENT_HEADER {

    USHORT      Size;                   // Event Size
    HeaderType  Type;                   // Header Type
    uchar       MarkerFlags <format=hex>;             
    USHORT      Flags;                  // Flags
    USHORT      EventProperty;          // User given event property
    ULONG       ThreadId;               // Thread Id
    ULONG       ProcessId;              // Process Id
    int64       TimeStamp;              // Event Timestamp
    GUID        ProviderId;             // Provider Id
    struct EVENT_DESCRIPTOR {
        USHORT    Id;
        UCHAR     Version;
        UCHAR     Channel;
        UCHAR     Level;
        UCHAR     Opcode;
        USHORT    Task;
        uint64 Keyword;
    } EventDescriptor;        // Event Descriptor
    ULONG       KernelTime;             // Kernel Mode CPU ticks
    ULONG       UserTime;               // User mode CPU ticks
    GUID        ActivityId;             // Activity Id
};

typedef struct {
    local byte isFullSize = FALSE;
    switch(ReadByte(FTell()+2)) 
    {
        case 1  : //TRACE_HEADER_TYPE_SYSTEM32
        case 2  : //TRACE_HEADER_TYPE_SYSTEM64
            isFullSize = TRUE;
        case 3  : //TRACE_HEADER_TYPE_COMPACT32
        case 4  : //TRACE_HEADER_TYPE_COMPACT64
            SYSTEM_TRACE_HEADER header(isFullSize) <bgcolor=cGray>;
            switch(header.HookId){
                case 0:
                    TRACE_LOGFILE_HEADER LogfileHeader;
                    break;
                default:
                    byte raw_event[header.Size-sizeof(header)];
                    break;
            }
            break;
        case 16 : //TRACE_HEADER_TYPE_PERFINFO32
        case 17 : //TRACE_HEADER_TYPE_PERFINFO64
            PERFINFO_TRACE_HEADER header <bgcolor=cGray>;
            byte raw_event[header.Size-sizeof(header)];
            break;
        case 10 : //TRACE_HEADER_TYPE_FULL_HEADER32
        case 20 : //TRACE_HEADER_TYPE_FULL_HEADER64
            EVENT_TRACE_HEADER header <bgcolor=cGray>;
            byte raw_event[header.Size-sizeof(header)];
            break;
            
        case 18 : //TRACE_HEADER_TYPE_EVENT_HEADER32
        case 19 : //TRACE_HEADER_TYPE_EVENT_HEADER64
            EVENT_HEADER header <bgcolor=cGray>;
            byte raw_event[header.Size-sizeof(header)];
            break;
        case 11 : //TRACE_HEADER_TYPE_INSTANCE32
        case 21 : //TRACE_HEADER_TYPE_INSTANCE64
            EVENT_INSTANCE_GUID_HEADER header <bgcolor=cGray>;
            byte raw_event[header.Size-sizeof(header)];
            break;
        case 13 : //TRACE_HEADER_TYPE_ERROR
            //Details unknown
        case 15 : //TRACE_HEADER_TYPE_MESSAGE
            //MESSAGE_TRACE_HEADER
        default :
            ushort Size;
            ubyte Type;
            ubyte MarkerFlags;
            byte unknown_event[Max(Size - 4, 0)]; //Max prevents error if we get out of sync
            Warning("Unrecognized header type");
            break;
    }
    //TODO: I'm guessing this padding depends upon event bitness (so for all vaguely recent versions, Kernel bitness)
    local int alignment = 8 - (FTell() % 8);
    if(alignment < 8)
    {
        byte padding[alignment];
    }

} EtwEvent <optimize=false>;


// ##########################################
// #         ETW Buffer Definitions         #
// ##########################################

typedef enum {
   EtwBufferStateFree,
   EtwBufferStateGeneralLogging,
   EtwBufferStateCSwitch,
   EtwBufferStateFlush,
   EtwBufferStateMaximum //MaxState should always be the last enum
} ETW_BUFFER_STATE;

typedef struct {
     uchar ProcessorNumber;
     uchar Alignment;
     ushort LoggerId;
} ETW_BUFFER_CONTEXT;

typedef struct {
    int64     StartTime;
    int64     StartPerfClock;
} ETW_REF_CLOCK;

enum <ushort> EtwBufferType
{
    GENERIC      = 0,
    RUNDOWN      = 1,
    CTX_SWAP     = 2,
    REFTIME      = 3,
    HEADER       = 4,
    BATCHED      = 5,
    EMPTY_MARKER = 6,
    DBG_INFO     = 7,
    MAXIMUM      = 8
};

typedef struct {
    ulong     BufferSize;         // BufferSize
    ulong     SavedOffset;        // Temp saved offset
    ulong     CurrentOffset;      // Current offset
    long      ReferenceCount;     // Reference count
    int64     TimeStamp;          // Flush time stamp
    int64     SequenceNumber;     // Buffer sequence number
    uint64    ClockType : 3;      // DBG_INFO buffers send to debugger
    uint64    Frequency : 61;

    ETW_BUFFER_CONTEXT  ClientContext;      // LoggerId/ProcessorIndex
    ETW_BUFFER_STATE    State;              // (Free/GeneralLogging/Flush)

    ulong               Offset;             // Offset when flushing (can overlap SavedOffset)
    ushort              BufferFlag;         // (flush marker, events lost)
    EtwBufferType       BufferType;         // (generic/rundown/cswitch/reftime)

    //note - not sure this isn't just padding on disk
    ETW_REF_CLOCK       ReferenceTime;      // persistent real-time
} WMI_BUFFER_HEADER;



typedef struct {
    WMI_BUFFER_HEADER header;
    if((header.BufferFlag & 0x40) == 0x40)
    {
        byte LZNT1_buffer[header.BufferSize - sizeof(header)];
    }
    else
    {
        while( FTell() < (startof(this) + header.Offset))
        {
            EtwEvent events;
        }
    }
} EtwBuffer <size=SizeEtwBuffer>;

int SizeEtwBuffer(EtwBuffer &buff)
{   //All versions Windows Vista+ have the buffer size (or relative offset to the next buffer) as the first 4 bytes.
    //No idea what Win2K and XP do.
    return ReadUInt(startof(buff));
}


while( !FEof() )
{
    EtwBuffer Buffer;
}

The GUID struct is wrong as it is only 12 bytes instead of 16.

struct GUID {  DWORD Data1;  ushort Data2;  ushort Data3;  ulong Data4; };

should be

struct GUID {  byte data[16]; };

whoops, thanks - I'm too used to C# where ulong is 8 bytes. (I intentionally went with a non-standard definition to get the default display more GUID-like... though really it should be using a read function to make a properly formatted GUID)

you are the king. thanks

struct _SYSTEMTIME {
ushort wYear;
ushort wMonth;
ushort wDayOfWeek;
ushort wDay;
ushort wHour;
ushort wMinute;
ushort wSecond;
ushort wMilliseconds;
};

//byte TimeZone[176]; //TIME_ZONE_INFORMATION
struct _TIME_ZONE_INFORMATION
{
long Bias;
wchar_t StandardName[32] <bgcolor=cGreen>;
_SYSTEMTIME StandardDate;
long StandardBias;
wchar_t DaylightName[32] <bgcolor=cGreen>;
_SYSTEMTIME DaylightDate;
long DaylightBias;
byte padding[4]; //align int64
}TIME_ZONE_INFORMATION;