Add infrastructure for fragmented blobs

public/client/TracyProfiler.cpp

@@ -2462,11 +2462,13 @@ Profiler::DequeueStatus Profiler::Dequeue( moodycamel::ConsumerToken& token )
                         break;
                     case QueueType::Blob:
                     case QueueType::BlobCallstack:
+                    {
                         ptr = MemRead<uint64_t>( &item->blobData.data );
-                        size = MemRead<uint16_t>( &item->blobData.size );
+                        auto size = MemRead<uint32_t>( &item->blobData.size );
                         SendBlob( (const char*)ptr, size );
                         tracy_free_fast( (void*)ptr );
                         break;
+                    }
                     case QueueType::ZoneBeginAllocSrcLoc:
                     case QueueType::ZoneBeginAllocSrcLocCallstack:
                     {
@@ -2998,7 +3000,7 @@ Profiler::DequeueStatus Profiler::DequeueSerial()
                 {
                     ThreadCtxCheckSerial( blobDataThread );
                     ptr = MemRead<uint64_t>( &item->blobData.data );
-                    uint16_t size = MemRead<uint16_t>( &item->blobData.size );
+                    uint32_t size = MemRead<uint32_t>( &item->blobData.size );
                     SendBlob( (const char*)ptr, size );
                     tracy_free_fast( (void*)ptr );
                     break;
@@ -3270,18 +3272,23 @@ void Profiler::SendLongString( uint64_t str, const char* ptr, size_t len, QueueT
 void Profiler::SendBlob( const char* ptr, size_t len )
 {
     QueueItem item;
+    uint32_t offset = 0;
+    uint32_t full_size = len;
     MemWrite( &item.hdr.type, QueueType::BlobFragment );
     while (len)
     {
-        const uint32_t max_fragment_size = TargetFrameSize - QueueDataSize[(int)QueueType::BlobFragment] - sizeof( uint32_t );
+        const uint32_t max_fragment_size = TargetFrameSize - QueueDataSize[(int)QueueType::BlobFragment] - 3 * sizeof ( uint32_t );
         uint32_t fragment_size = len > max_fragment_size ? max_fragment_size : len;
         len -= fragment_size;
 
-        NeedDataSize( QueueDataSize[(int)QueueType::BlobFragment] + sizeof( fragment_size ) + fragment_size );
+        NeedDataSize( QueueDataSize[(int)QueueType::BlobFragment] + 2 * sizeof ( uint32_t ) + sizeof( fragment_size ) + fragment_size );
 
         AppendDataUnsafe( &item, QueueDataSize[(int)QueueType::BlobFragment] );
+        AppendDataUnsafe( &full_size, sizeof(uint32_t) );
+        AppendDataUnsafe( &offset, sizeof(uint32_t) );
         AppendDataUnsafe( &fragment_size, sizeof( fragment_size ) );
-        AppendDataUnsafe( ptr, fragment_size );
+        AppendDataUnsafe( ptr + offset, fragment_size );
+        offset += fragment_size;
     }
 }
 

public/client/TracyProfiler.hpp

@@ -486,7 +486,6 @@ public:
 
     static tracy_force_inline void Blob( uint64_t encoding, const void* data, size_t size, int callstack )
     {
-        assert( size < (std::numeric_limits<uint16_t>::max)() );
 #ifdef TRACY_ON_DEMAND
         if( !GetProfiler().IsConnected() ) return;
 #endif

public/common/TracyQueue.hpp

@@ -405,11 +405,16 @@ struct QueueBlob
 
 struct QueueBlobData : public QueueBlob
 {
-    uint64_t encoding;
+    uint32_t encoding;
     uint64_t data;      // ptr
     uint32_t size;
 };
 
+struct QueueBlobDataThread : public QueueBlobData
+{
+    uint32_t thread;
+};
+
 // Don't change order, only add new entries at the end, this is also used on trace dumps!
 enum class GpuContextType : uint8_t
 {
@@ -757,6 +762,7 @@ struct QueueItem
         QueueMessageColorFatThread messageColorFatThread;
         QueueBlob blob;
         QueueBlobData blobData;
+        QueueBlobDataThread blobDataThread;
         QueueGpuNewContext gpuNewContext;
         QueueGpuZoneBegin gpuZoneBegin;
         QueueGpuZoneBeginLean gpuZoneBeginLean;

server/TracyWorker.cpp

@@ -275,6 +275,7 @@ Worker::Worker( const char* addr, uint16_t port, int64_t memoryLimit )
     , m_callstackFrameStaging( nullptr )
     , m_traceVersion( CurrentVersion )
     , m_loadTime( 0 )
+    , m_blob( nullptr )
 {
     m_data.sourceLocationExpand.push_back( 0 );
     m_data.localThreadCompress.InitZero();
@@ -3063,6 +3064,9 @@ void Worker::DispatchFailure( const QueueItem& ev, const char*& ptr )
                 AddFiberName( ev.stringTransfer.ptr, ptr, sz );
                 m_serverQuerySpaceLeft++;
                 break;
+            case QueueType::BlobFragment:
+                m_serverQuerySpaceLeft++;
+                break;
             case QueueType::PlotName:
             case QueueType::FrameName:
             case QueueType::ExternalName:
@@ -3093,6 +3097,33 @@ void Worker::DispatchFailure( const QueueItem& ev, const char*& ptr )
             AddSecondString( ptr, sz );
             ptr += sz;
             break;
+        case QueueType::BlobFragment:
+        {
+            ptr += sizeof( QueueHeader );
+            uint32_t full_size;
+            memcpy( &full_size, ptr, sizeof( uint32_t ) );
+            ptr += sizeof( uint32_t );
+            uint32_t offset;
+            memcpy( &offset, ptr, sizeof( uint32_t ) );
+            ptr += sizeof( uint32_t );
+            memcpy( &sz, ptr, sizeof( sz ) );
+            ptr += sizeof( sz );
+            if ( offset == 0 )
+            {
+                assert( m_blob == nullptr );
+                m_blob = (unsigned char *)malloc( full_size );
+            }
+            assert( m_blob != nullptr );
+            memcpy( m_blob, ptr, sz);
+            if (offset + sz > full_size)
+            {
+                AddSingleString( (const char*)m_blob, full_size );
+                free( m_blob );
+                m_blob = nullptr;
+            }
+            ptr += sz;
+            break;
+        }
         default:
             ptr += QueueDataSize[ev.hdr.idx];
             switch( ev.hdr.type )
@@ -3218,6 +3249,34 @@ bool Worker::DispatchProcess( const QueueItem& ev, const char*& ptr )
             }
             ptr += sz;
         }
+        else if( ev.hdr.type == QueueType::BlobFragment )
+        {
+            uint32_t full_size;
+            memcpy( &full_size, ptr, sizeof( uint32_t ) );
+            ptr += sizeof( uint32_t );
+            uint32_t offset;
+            memcpy( &offset, ptr, sizeof( uint32_t ) );
+            ptr += sizeof( uint32_t );
+            uint32_t sz;
+            memcpy( &sz, ptr, sizeof( sz ) );
+            ptr += sizeof( sz );
+            if ( offset == 0 )
+            {
+                assert( m_blob == nullptr );
+                m_blob = (unsigned char *)malloc( full_size );
+            }
+            assert( m_blob != nullptr );
+            memcpy( m_blob, ptr, sz);
+            if (offset + sz >= full_size)
+            {
+                AddSingleString((const char*)m_blob, full_size );
+                free( m_blob );
+                m_blob = nullptr;
+            }
+            m_serverQuerySpaceLeft++;
+            ptr += sz;
+        }
+
         else
         {
             uint16_t sz;
@@ -3272,6 +3331,7 @@ bool Worker::DispatchProcess( const QueueItem& ev, const char*& ptr )
     else
     {
         uint16_t sz;
+        uint32_t sz32;
         switch( ev.hdr.type )
         {
         case QueueType::SingleStringData:
@@ -3288,6 +3348,12 @@ bool Worker::DispatchProcess( const QueueItem& ev, const char*& ptr )
             AddSecondString( ptr, sz );
             ptr += sz;
             return true;
+        case QueueType::BlobFragment:
+            ptr += sizeof( QueueHeader );
+            memcpy( &sz32, ptr, sizeof( sz32 ) );
+            ptr += sizeof( sz32 );
+            ptr += sz32;
+            return true;
         default:
             ptr += QueueDataSize[ev.hdr.idx];
             return Process( ev );

server/TracyWorker.hpp

@@ -1057,6 +1057,8 @@ private:
     DataBlock m_data;
     MbpsBlock m_mbpsData;
 
+    unsigned char *m_blob;
+
     int m_traceVersion;
     std::atomic<uint8_t> m_handshake { 0 };