Runtime Initialization Tracing (#1105)

* Runtime initialization tracing

- calbacks and buffer entries notifying when a runtime has been initialized

* Minor cleanup to registration.cpp

* JSON tool implementation

* Increase perfetto_reader timeout

* Handle perfetto_reader timeout when attr doesn't exist

* clang-tidy fixes to memory_allocation.cpp

source/include/rocprofiler-sdk/buffer_tracing.h

@@ -157,6 +157,7 @@ typedef struct
     /// @brief ::ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API,
     /// ::ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API, or
     /// ::ROCPROFILER_CALLBACK_TRACING_MARKER_NAME_API
+    /// @var operation
     /// @brief Specification of the API function, e.g., ::rocprofiler_marker_core_api_id_t,
     /// ::rocprofiler_marker_control_api_id_t, or
     /// ::rocprofiler_marker_name_api_id_t
@@ -176,8 +177,9 @@ typedef struct
     rocprofiler_thread_id_t           thread_id;        ///< id for thread generating this record
 
     /// @var kind
-    /// @brief ::ROCPROFILER_CALLBACK_TRACING_RCCL_API,
-    /// @brief Specification of the API function, e.g., ::rocprofiler_rccl_api_id_t,
+    /// @brief ::ROCPROFILER_CALLBACK_TRACING_RCCL_API
+    /// @var operation
+    /// @brief Specification of the API function, e.g., ::rocprofiler_rccl_api_id_t
 } rocprofiler_buffer_tracing_rccl_api_record_t;
 
 /**
@@ -265,13 +267,13 @@ typedef struct
 {
     uint64_t                               size;  ///< size of this struct
     rocprofiler_buffer_tracing_kind_t      kind;  ///< ::ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY
-    rocprofiler_scratch_memory_operation_t operation;  ///< specification of the kind
-    rocprofiler_agent_id_t                 agent_id;   ///< agent kernel was dispatched on
-    rocprofiler_queue_id_t                 queue_id;   ///< queue kernel was dispatched on
+    rocprofiler_scratch_memory_operation_t operation;       ///< specification of the kind
+    rocprofiler_correlation_id_t           correlation_id;  ///< correlation ids for record
+    rocprofiler_agent_id_t                 agent_id;        ///< agent kernel was dispatched on
+    rocprofiler_queue_id_t                 queue_id;        ///< queue kernel was dispatched on
     rocprofiler_thread_id_t                thread_id;  ///< id for thread generating this record
     rocprofiler_timestamp_t                start_timestamp;  ///< start time in nanoseconds
     rocprofiler_timestamp_t                end_timestamp;    ///< end time in nanoseconds
-    rocprofiler_correlation_id_t           correlation_id;   ///< correlation ids for record
     rocprofiler_scratch_alloc_flag_t       flags;
 } rocprofiler_buffer_tracing_scratch_memory_record_t;
 
@@ -296,6 +298,38 @@ typedef struct
     /// @brief Only internal correlation ID is provided
 } rocprofiler_buffer_tracing_correlation_id_retirement_record_t;
 
+/**
+ * @brief ROCProfiler Buffer Runtime Initialization Tracer Record.
+ */
+typedef struct rocprofiler_buffer_tracing_runtime_initialization_record_t
+{
+    uint64_t                                       size;  ///< size of this struct
+    rocprofiler_buffer_tracing_kind_t              kind;
+    rocprofiler_runtime_initialization_operation_t operation;
+    rocprofiler_correlation_id_t                   correlation_id;
+    rocprofiler_thread_id_t                        thread_id;
+    rocprofiler_timestamp_t                        timestamp;
+    uint64_t                                       version;
+    uint64_t                                       instance;
+
+    /// @var kind
+    /// @brief ::ROCPROFILER_BUFFER_TRACING_RUNTIME_INITIALIZATION
+    /// @var operation
+    /// @brief Indicates which runtime was initialized/loaded
+    /// @var correlation_id
+    /// @brief Correlation ID for these records are always zero
+    /// @var thread_id
+    /// @brief ID for thread which loaded this runtime
+    /// @var timestamp
+    /// @brief Timestamp (in nanosec) of when runtime was initialized/loaded
+    /// @var version
+    /// @brief The version number of the runtime
+    ///
+    /// Version number is encoded as: (10000 * MAJOR) + (100 * MINOR) + PATCH
+    /// @var instance
+    /// @brief Number of times this runtime had been loaded previously
+} rocprofiler_buffer_tracing_runtime_initialization_record_t;
+
 /**
  * @brief Callback function for mapping @ref rocprofiler_buffer_tracing_kind_t ids to
  * string names. @see rocprofiler_iterate_buffer_trace_kind_names.

source/include/rocprofiler-sdk/callback_tracing.h

@@ -236,6 +236,21 @@ typedef struct rocprofiler_callback_tracing_scratch_memory_data_t
     rocprofiler_scratch_memory_args_t args;
 } rocprofiler_callback_tracing_scratch_memory_data_t;
 
+/**
+ * @brief ROCProfiler Runtime Initialization Data.
+ */
+typedef struct rocprofiler_callback_tracing_runtime_initialization_data_t
+{
+    uint64_t size;  ///< size of this struct
+    uint64_t version;
+    uint64_t instance;  ///< Number of times this runtime had been loaded previously
+
+    /// @var version
+    /// @brief The version number of the library
+    ///
+    /// Version number is encoded as: (10000 * MAJOR) + (100 * MINOR) + PATCH
+} rocprofiler_callback_tracing_runtime_initialization_data_t;
+
 /**
  * @brief API Tracing callback function. This function is invoked twice per API function: once
  * before the function is invoked and once after the function is invoked.  The external correlation

source/include/rocprofiler-sdk/cxx/serialization.hpp

@@ -871,6 +871,29 @@ save(ArchiveT& ar, rocprofiler_record_dimension_info_t data)
     ROCP_SDK_SAVE_DATA_CSTR(name);
 }
 
+template <typename ArchiveT>
+void
+save(ArchiveT& ar, rocprofiler_callback_tracing_runtime_initialization_data_t data)
+{
+    ROCP_SDK_SAVE_DATA_FIELD(size);
+    ROCP_SDK_SAVE_DATA_FIELD(version);
+    ROCP_SDK_SAVE_DATA_FIELD(instance);
+}
+
+template <typename ArchiveT>
+void
+save(ArchiveT& ar, rocprofiler_buffer_tracing_runtime_initialization_record_t data)
+{
+    ROCP_SDK_SAVE_DATA_FIELD(size);
+    ROCP_SDK_SAVE_DATA_FIELD(kind);
+    ROCP_SDK_SAVE_DATA_FIELD(operation);
+    ROCP_SDK_SAVE_DATA_FIELD(correlation_id);
+    ROCP_SDK_SAVE_DATA_FIELD(timestamp);
+    ROCP_SDK_SAVE_DATA_FIELD(thread_id);
+    ROCP_SDK_SAVE_DATA_FIELD(version);
+    ROCP_SDK_SAVE_DATA_FIELD(instance);
+}
+
 template <typename ArchiveT, typename EnumT, typename ValueT>
 void
 save(ArchiveT& ar, const rocprofiler::sdk::utility::name_info<EnumT, ValueT>& data)

source/include/rocprofiler-sdk/fwd.h

@@ -173,6 +173,8 @@ typedef enum  // NOLINT(performance-enum-size)
     ROCPROFILER_CALLBACK_TRACING_OPENMP,             ///< @see ::rocprofiler_ompt_operation_t
     ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION,  ///< @see
                                                      ///< ::rocprofiler_memory_allocation_operation_t
+    ROCPROFILER_CALLBACK_TRACING_RUNTIME_INITIALIZATION,  ///< Callback notifying that a runtime
+                                                          ///< library has been initialized
     ROCPROFILER_CALLBACK_TRACING_LAST,
 } rocprofiler_callback_tracing_kind_t;
 
@@ -201,11 +203,14 @@ typedef enum  // NOLINT(performance-enum-size)
     ROCPROFILER_BUFFER_TRACING_OPENMP,                     ///< @see ::rocprofiler_ompt_operation_t
     ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION,          ///< @see
                                                    ///< ::rocprofiler_memory_allocation_operation_t
+    ROCPROFILER_BUFFER_TRACING_RUNTIME_INITIALIZATION,  ///< Record indicating a runtime library has
+                                                        ///< been initialized. @see
+                                                        ///< ::rocprofiler_runtime_initialization_operation_t
     ROCPROFILER_BUFFER_TRACING_LAST,
 } rocprofiler_buffer_tracing_kind_t;
 
 /**
- * @brief ROCProfiler Code Object Tracer Operation.
+ * @brief ROCProfiler Code Object Tracer Operations.
  */
 typedef enum  // NOLINT(performance-enum-size)
 {
@@ -216,7 +221,7 @@ typedef enum  // NOLINT(performance-enum-size)
 } rocprofiler_code_object_operation_t;
 
 /**
- * @brief Memory Copy Operation.
+ * @brief Memory Copy Operations.
  */
 typedef enum  // NOLINT(performance-enum-size)
 {
@@ -378,6 +383,19 @@ typedef enum
     ROCPROFILER_TABLE_LAST           = ROCPROFILER_RCCL_TABLE,
 } rocprofiler_intercept_table_t;
 
+/**
+ * @brief ROCProfiler Runtime Initialization Tracer Operations.
+ */
+typedef enum  // NOLINT(performance-enum-size)
+{
+    ROCPROFILER_RUNTIME_INITIALIZATION_NONE = 0,  ///< Unknown runtime initialization
+    ROCPROFILER_RUNTIME_INITIALIZATION_HSA,       ///< Application loaded HSA runtime
+    ROCPROFILER_RUNTIME_INITIALIZATION_HIP,       ///< Application loaded HIP runtime
+    ROCPROFILER_RUNTIME_INITIALIZATION_MARKER,    ///< Application loaded Marker (ROCTx) runtime
+    ROCPROFILER_RUNTIME_INITIALIZATION_RCCL,      ///< Application loaded RCCL runtime
+    ROCPROFILER_RUNTIME_INITIALIZATION_LAST,
+} rocprofiler_runtime_initialization_operation_t;
+
 /**
  * @brief Enumeration for specifying the counter info struct version you want.
  */

source/lib/rocprofiler-sdk/CMakeLists.txt

@@ -3,8 +3,9 @@
 #
 rocprofiler_activate_clang_tidy()
 
-set(ROCPROFILER_LIB_HEADERS agent.hpp buffer.hpp external_correlation.hpp
-                            intercept_table.hpp internal_threading.hpp registration.hpp)
+set(ROCPROFILER_LIB_HEADERS
+    agent.hpp buffer.hpp external_correlation.hpp intercept_table.hpp
+    internal_threading.hpp registration.hpp runtime_initialization.hpp)
 set(ROCPROFILER_LIB_SOURCES
     agent.cpp
     buffer.cpp
@@ -19,8 +20,9 @@ set(ROCPROFILER_LIB_SOURCES
     internal_threading.cpp
     pc_sampling.cpp
     profile_config.cpp
+    registration.cpp
     rocprofiler.cpp
-    registration.cpp)
+    runtime_initialization.cpp)
 
 # ----------------------------------------------------------------------------------------#
 #

source/lib/rocprofiler-sdk/buffer_tracing.cpp

@@ -20,13 +20,6 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-#include <rocprofiler-sdk/fwd.h>
-#include <rocprofiler-sdk/hip/table_id.h>
-#include <rocprofiler-sdk/hsa/table_id.h>
-#include <rocprofiler-sdk/marker/table_id.h>
-#include <rocprofiler-sdk/rccl/table_id.h>
-#include <rocprofiler-sdk/rocprofiler.h>
-
 #include "lib/common/logging.hpp"
 #include "lib/rocprofiler-sdk/context/context.hpp"
 #include "lib/rocprofiler-sdk/context/domain.hpp"
@@ -40,6 +33,14 @@
 #include "lib/rocprofiler-sdk/page_migration/page_migration.hpp"
 #include "lib/rocprofiler-sdk/rccl/rccl.hpp"
 #include "lib/rocprofiler-sdk/registration.hpp"
+#include "lib/rocprofiler-sdk/runtime_initialization.hpp"
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/hip/table_id.h>
+#include <rocprofiler-sdk/hsa/table_id.h>
+#include <rocprofiler-sdk/marker/table_id.h>
+#include <rocprofiler-sdk/rccl/table_id.h>
+#include <rocprofiler-sdk/rocprofiler.h>
 
 #include <atomic>
 #include <limits>
@@ -88,6 +89,7 @@ ROCPROFILER_BUFFER_TRACING_KIND_STRING(SCRATCH_MEMORY)
 ROCPROFILER_BUFFER_TRACING_KIND_STRING(CORRELATION_ID_RETIREMENT)
 ROCPROFILER_BUFFER_TRACING_KIND_STRING(RCCL_API)
 ROCPROFILER_BUFFER_TRACING_KIND_STRING(OPENMP)
+ROCPROFILER_BUFFER_TRACING_KIND_STRING(RUNTIME_INITIALIZATION)
 
 template <size_t Idx, size_t... Tail>
 std::pair<const char*, size_t>
@@ -271,6 +273,11 @@ rocprofiler_query_buffer_tracing_kind_operation_name(rocprofiler_buffer_tracing_
             val = rocprofiler::page_migration::name_by_id(operation);
             break;
         }
+        case ROCPROFILER_BUFFER_TRACING_RUNTIME_INITIALIZATION:
+        {
+            val = rocprofiler::runtime_init::name_by_id(operation);
+            break;
+        }
         case ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT:
         {
             return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED;
@@ -396,6 +403,11 @@ rocprofiler_iterate_buffer_tracing_kind_operations(
             ops = rocprofiler::page_migration::get_ids();
             break;
         }
+        case ROCPROFILER_BUFFER_TRACING_RUNTIME_INITIALIZATION:
+        {
+            ops = rocprofiler::runtime_init::get_ids();
+            break;
+        }
         case ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT:
         {
             return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED;

source/lib/rocprofiler-sdk/callback_tracing.cpp

@@ -20,14 +20,6 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-#include <rocprofiler-sdk/callback_tracing.h>
-#include <rocprofiler-sdk/fwd.h>
-#include <rocprofiler-sdk/hip/table_id.h>
-#include <rocprofiler-sdk/hsa/table_id.h>
-#include <rocprofiler-sdk/marker/table_id.h>
-#include <rocprofiler-sdk/rccl/table_id.h>
-#include <rocprofiler-sdk/rocprofiler.h>
-
 #include "lib/rocprofiler-sdk/code_object/code_object.hpp"
 #include "lib/rocprofiler-sdk/context/context.hpp"
 #include "lib/rocprofiler-sdk/context/domain.hpp"
@@ -40,6 +32,15 @@
 #include "lib/rocprofiler-sdk/marker/marker.hpp"
 #include "lib/rocprofiler-sdk/rccl/rccl.hpp"
 #include "lib/rocprofiler-sdk/registration.hpp"
+#include "lib/rocprofiler-sdk/runtime_initialization.hpp"
+
+#include <rocprofiler-sdk/callback_tracing.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/hip/table_id.h>
+#include <rocprofiler-sdk/hsa/table_id.h>
+#include <rocprofiler-sdk/marker/table_id.h>
+#include <rocprofiler-sdk/rccl/table_id.h>
+#include <rocprofiler-sdk/rocprofiler.h>
 
 #include <atomic>
 #include <cstdint>
@@ -85,6 +86,7 @@ ROCPROFILER_CALLBACK_TRACING_KIND_STRING(MEMORY_COPY)
 ROCPROFILER_CALLBACK_TRACING_KIND_STRING(MEMORY_ALLOCATION)
 ROCPROFILER_CALLBACK_TRACING_KIND_STRING(RCCL_API)
 ROCPROFILER_CALLBACK_TRACING_KIND_STRING(OPENMP)
+ROCPROFILER_CALLBACK_TRACING_KIND_STRING(RUNTIME_INITIALIZATION)
 
 template <size_t Idx, size_t... Tail>
 std::pair<const char*, size_t>
@@ -263,6 +265,10 @@ rocprofiler_query_callback_tracing_kind_operation_name(rocprofiler_callback_trac
             val = rocprofiler::hsa::memory_allocation::name_by_id(operation);
             break;
         }
+        case ROCPROFILER_CALLBACK_TRACING_RUNTIME_INITIALIZATION:
+        {
+            val = rocprofiler::runtime_init::name_by_id(operation);
+        }
     };
 
     if(!val)
@@ -387,6 +393,10 @@ rocprofiler_iterate_callback_tracing_kind_operations(
             ops = rocprofiler::hsa::memory_allocation::get_ids();
             break;
         }
+        case ROCPROFILER_CALLBACK_TRACING_RUNTIME_INITIALIZATION:
+        {
+            ops = rocprofiler::runtime_init::get_ids();
+        }
     };
 
     for(const auto& itr : ops)
@@ -519,6 +529,7 @@ rocprofiler_iterate_callback_tracing_kind_operation_args(
         case ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION:
         case ROCPROFILER_CALLBACK_TRACING_RCCL_API:
         case ROCPROFILER_CALLBACK_TRACING_OPENMP:
+        case ROCPROFILER_CALLBACK_TRACING_RUNTIME_INITIALIZATION:
         {
             return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED;
         }

source/lib/rocprofiler-sdk/hsa/async_copy.cpp

@@ -105,15 +105,15 @@ id_by_name(const char* name, std::index_sequence<Idx, IdxTail...>)
     if constexpr(sizeof...(IdxTail) > 0)
         return id_by_name(name, std::index_sequence<IdxTail...>{});
     else
-        return ROCPROFILER_HSA_AMD_EXT_API_ID_NONE;
+        return ROCPROFILER_MEMORY_COPY_LAST;
 }
 
 template <size_t... Idx>
 void
 get_ids(std::vector<uint32_t>& _id_list, std::index_sequence<Idx...>)
 {
     auto _emplace = [](auto& _vec, uint32_t _v) {
-        if(_v < static_cast<uint32_t>(ROCPROFILER_HSA_AMD_EXT_API_ID_LAST)) _vec.emplace_back(_v);
+        if(_v < static_cast<uint32_t>(ROCPROFILER_MEMORY_COPY_LAST)) _vec.emplace_back(_v);
     };
 
     (_emplace(_id_list, async_copy_info<Idx>::operation_idx), ...);

source/lib/rocprofiler-sdk/hsa/memory_allocation.cpp

@@ -318,12 +318,12 @@ get_agent(T val, IterateFunc iterate_func, CallbackFunc callback)
     if(existing.count(val) == 0)
     {
         auto agents = rocprofiler::agent::get_agents();
-        for(auto itr : agents)
+        for(const auto* itr : agents)
         {
             auto hsa_agent = rocprofiler::agent::get_hsa_agent(itr);
             if(hsa_agent)
             {
-                auto rocprof_agent = rocprofiler::agent::get_rocprofiler_agent(*hsa_agent);
+                const auto* rocprof_agent = rocprofiler::agent::get_rocprofiler_agent(*hsa_agent);
                 if(rocprof_agent)
                 {
                     auto data = typename memory_allocation_info<OpIdx>::pairtype{&existing,