Installation and Usage Guide for ONNX Runtime C++ Static Library

The official ONNX Runtime has provided dynamic libraries for both GPU and CPU versions across various operating systems, but there is no static library version available. This tutorial mainly explains how to install and use the static library version, starting with the differences between them.

Difference Between Static and Dynamic Libraries

Static Library

File Extension: .lib (Windows), .a (Linux/macOS)
Linking Method: Linked at compile time, code is copied into the final executable file
Deployment: A single executable file, no additional DLLs required
Size: Executable file is larger
Compatibility: Better, does not depend on the system environment

Dynamic Library

File Extension: .dll (Windows), .so (Linux), .dylib (macOS)
Linking Method: Loaded at runtime, multiple programs can share
Deployment: Requires distribution of DLL files
Size: Executable file is smaller
Memory Usage: More efficient (shared library)

ONNX Runtime Static Library Usage Guide

1. Environment Preparation

Download ONNX Runtime

Download the precompiled package from the official GitHub or compile it yourself.

Directory Structure:

onnxruntime/
├── include/
├── lib/
│   ├── onnxruntime.lib (Static Library)
│   └── onnxruntime.dll (Dynamic Library)
└── bin/

2. CMake Configuration Example

cmake_minimum_required(VERSION 3.18)
project(ONNXRuntimeDemo)

set(CMAKE_CXX_STANDARD 14)

# Set ONNX Runtime path
set(ONNXRUNTIME_ROOT "path/to/onnxruntime")

# Include directories
include_directories(${ONNXRUNTIME_ROOT}/include)

# Static library linking
if(WIN32)
    # Windows static library configuration
    set(ONNXRUNTIME_LIB ${ONNXRUNTIME_ROOT}/lib/onnxruntime.lib)

    # Required system libraries
    set(SYSTEM_LIBS 
        ws2_32
        advapi32
        dbghelp
    )
else()
    # Linux/macOS static library configuration
    set(ONNXRUNTIME_LIB ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.a)

    # Required system libraries
    set(SYSTEM_LIBS
        pthread
        dl
    )
endif()

# Create executable file
add_executable(onnx_demo main.cpp)

# Link libraries
target_link_libraries(onnx_demo 
    ${ONNXRUNTIME_LIB}
    ${SYSTEM_LIBS}
)

3. C++ Code Example

#include <onnxruntime_cxx_api.h>
#include <iostream>
#include <vector>
#include <chrono>

class ONNXModel {
private:
    Ort::Env env;
    Ort::Session session;
    std::vector<const char*> input_names;
    std::vector<const char*> output_names;

public:
    ONNXModel(const std::string& model_path, int device_id = 0) 
        : env(ORT_LOGGING_LEVEL_WARNING, "ONNXModel") {

        // Session options
        Ort::SessionOptions session_options;

        // Set number of threads
        session_options.SetIntraOpNumThreads(1);
        session_options.SetInterOpNumThreads(1);

        // Uncomment the following lines for GPU usage
        // OrtCUDAProviderOptions cuda_options;
        // cuda_options.device_id = device_id;
        // session_options.AppendExecutionProvider_CUDA(cuda_options);

        // Create session
        session = Ort::Session(env, model_path.c_str(), session_options);

        // Get input and output information
        setup_io_names();
    }

private:
    void setup_io_names() {
        Ort::AllocatorWithDefaultOptions allocator;

        // Input names
        size_t num_input_nodes = session.GetInputCount();
        for(size_t i = 0; i < num_input_nodes; i++) {
            char* input_name = session.GetInputName(i, allocator);
            input_names.push_back(input_name);
        }

        // Output names
        size_t num_output_nodes = session.GetOutputCount();
        for(size_t i = 0; i < num_output_nodes; i++) {
            char* output_name = session.GetOutputName(i, allocator);
            output_names.push_back(output_name);
        }
    }

public:
    std::vector<float> inference(const std::vector<float>& input_data, 
                                const std::vector<int64_t>& input_shape) {

        Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(
            OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);

        // Create input tensor
        auto input_tensor = Ort::Value::CreateTensor<float>(
            memory_info, 
            const_cast<float*>(input_data.data()), 
            input_data.size(),
            input_shape.data(), 
            input_shape.size()
        );

        // Run inference
        auto output_tensors = session.Run(
            Ort::RunOptions{nullptr}, 
            input_names.data(), 
            &input_tensor, 
            input_names.size(),
            output_names.data(), 
            output_names.size()
        );

        // Get output
        float* floatarr = output_tensors[0].GetTensorMutableData<float>();
        auto tensor_shape = output_tensors[0].GetTensorTypeAndShapeInfo().GetShape();

        size_t output_size = 1;
        for(auto dim : tensor_shape) {
            output_size *= dim;
        }

        return std::vector<float>(floatarr, floatarr + output_size);
    }

    void print_model_info() {
        Ort::AllocatorWithDefaultOptions allocator;

        std::cout << "=== Model Information ===" << std::endl;

        // Input information
        std::cout << "Inputs:" << std::endl;
        size_t num_inputs = session.GetInputCount();
        for(size_t i = 0; i < num_inputs; i++) {
            auto type_info = session.GetInputTypeInfo(i);
            auto tensor_info = type_info.GetTensorTypeAndShapeInfo();
            auto shape = tensor_info.GetShape();

            char* name = session.GetInputName(i, allocator);
            std::cout << "  " << i << ": " << name << " Shape: [";
            for(size_t j = 0; j < shape.size(); j++) {
                std::cout << shape[j];
                if(j < shape.size() - 1) std::cout << ", ";
            }
            std::cout << "]" << std::endl;
        }

        // Output information
        std::cout << "Outputs:" << std::endl;
        size_t num_outputs = session.GetOutputCount();
        for(size_t i = 0; i < num_outputs; i++) {
            auto type_info = session.GetOutputTypeInfo(i);
            auto tensor_info = type_info.GetTensorTypeAndShapeInfo();
            auto shape = tensor_info.GetShape();

            char* name = session.GetOutputName(i, allocator);
            std::cout << "  " << i << ": " << name << " Shape: [";
            for(size_t j = 0; j < shape.size(); j++) {
                std::cout << shape[j];
                if(j < shape.size() - 1) std::cout << ", ";
            }
            std::cout << "]" << std::endl;
        }
    }
};

// Example usage
int main() {
    try {
        // Initialize model
        ONNXModel model("model.onnx");

        // Print model information
        model.print_model_info();

        // Prepare input data (example)
        std::vector<float> input_data(1 * 3 * 224 * 224, 0.5f); // Assume it's an image classification model
        std::vector<int64_t> input_shape = {1, 3, 224, 224};

        // Execute inference
        auto start_time = std::chrono::high_resolution_clock::now();

        auto output = model.inference(input_data, input_shape);

        auto end_time = std::chrono::high_resolution_clock::now();
        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);

        std::cout << "Inference time: " << duration.count() << " ms" << std::endl;

        // Output results
        std::cout << "Output size: " << output.size() << std::endl;
        std::cout << "First 10 values: ";
        for(int i = 0; i < 10 && i < output.size(); i++) {
            std::cout << output[i] << " ";
        }
        std::cout << std::endl;

    } catch(const std::exception& e) {
        std::cerr << "Error: " << e.what() << std::endl;
        return -1;
    }

    return 0;
}

4. Compilation and Execution

Windows (Command Line)

mkdir build
cd build
cmake -G "Visual Studio 16 2019" ..
cmake --build . --config Release

Linux/macOS

mkdir build
cd build
cmake ..
make -j4

5. Advanced Features

Using GPU

// Add GPU support in the constructor
#ifdef USE_CUDA
    OrtCUDAProviderOptions cuda_options;
    cuda_options.device_id = device_id;
    session_options.AppendExecutionProvider_CUDA(cuda_options);
#endif

Custom Memory Allocation

class CustomAllocator : public Ort::Allocator {
    // Implement custom memory allocator
};

Notes

Static Library Size: The ONNX Runtime static library is large; consider using the dynamic library if executable file size is a concern.
Dependency Management: The static library includes all dependencies; ensure there are no conflicting system library versions.
Compilation Time: Static linking may increase compilation time.
Memory Usage: Each program using the static library has its own copy of ONNX Runtime.

Common Issues

Linking Errors: Ensure all necessary system libraries are linked.
Version Compatibility: Ensure the ONNX Runtime version is compatible with the model.
Memory Leaks: Use RAII to manage Ort objects.

This tutorial provides the basic usage of the ONNX Runtime static library, and you can adjust and extend it according to your specific needs. The ONNX Runtime static library can currently be downloaded from the repository futureflsl/onnxruntime_chinese_mirror with the following versions:

Project Name

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.23.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.22.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.22.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.20.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.20.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.19.2

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.19.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.18.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.18.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.17.3

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.17.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.17.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.16.3

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.16.2

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.16.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x64-static-lib-1.16.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.23.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.22.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.22.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.20.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.19.2

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.19.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.18.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.18.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.17.3

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.17.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.17.0

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.16.3

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.16.2

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.16.1

[C++ ONNX Runtime Static Library] onnxruntime-win-x86-static-lib-1.16.0