Integrating Elasticsearch with HttpClient in ASP.NET Core for Logging and Querying

In development, centralized management of system logs and support for real-time search is an important means to improve operational efficiency. This article will introduce how to integrate <span>HttpClient</span> with Elasticsearch in an ASP.NET Core project to achieve automatic logging, indexing, and querying functionality.

Technical Architecture Overview

• • ASP.NET Core: Building Web API services
• • HttpClient: Communicating with the Elasticsearch REST interface
• • Elasticsearch: For storing and retrieving log data
• • Middleware: Intercepting requests and logging
• • Basic Auth: Ensuring secure access to Elasticsearch

1. Configuration File Settings

First, configure the address and authentication information for Elasticsearch in <span>appsettings.json</span>:

{
  "Elasticsearch":{
    "Url":"https://es-host:9200",
    "Username":"username",
    "Password":"password"
},
"Logging":{
    "LogLevel":{
      "Default":"Information"
    }
},
"AllowedHosts":"*"
}

2. Create Elasticsearch Service Class

Create the <span>ElasticsearchService.cs</span> class to encapsulate basic operations on Elasticsearch, such as writing documents and querying data:

public class ElasticsearchService
{
    private readonly HttpClient _client;
    private readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.Web);

    public ElasticsearchService(HttpClient client, IConfiguration config)
    {
        var uri = config["Elasticsearch:Url"];
        var username = config["Elasticsearch:Username"];
        var password = config["Elasticsearch:Password"];

        client.BaseAddress = new Uri(uri);
        var byteArray = Encoding.ASCII.GetBytes($"{username}:{password}");
        client.DefaultRequestHeaders.Authorization =
            new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));

        _client = client;
    }

    public async Task<bool> IndexAsync<t>(T document, string indexName, string? id = null)
    {
        var url = id != null ? $"/{indexName}/_doc/{id}" : $"/{indexName}/_doc";
        var content = new StringContent(JsonSerializer.Serialize(document, _jsonOptions), Encoding.UTF8, "application/json");

        var response = await _client.PostAsync(url, content);
        return response.IsSuccessStatusCode;
    }

    public async Task<list<t>> SearchAsync<t>(object query, string indexName)
    {
        var content = new StringContent(JsonSerializer.Serialize(query, _jsonOptions), Encoding.UTF8, "application/json");

        var response = await _client.PostAsync($"/{indexName}/_search", content);
        response.EnsureSuccessStatusCode();

        var json = await response.Content.ReadAsStringAsync();
        using var doc = JsonDocument.Parse(json);

        var hits = doc.RootElement.GetProperty("hits").GetProperty("hits");
        var result = new List<t>();

        foreach (var hit in hits.EnumerateArray())
        {
            var source = hit.GetProperty("_source").GetRawText();
            var obj = JsonSerializer.Deserialize<t>(source, _jsonOptions);
            if (obj != null) result.Add(obj);
        }

        return result;
    }
}</t></t></t></list<t></t></bool>

3. Register Services

In <span>Program.cs</span>, register <span>HttpClient</span> and <span>ElasticsearchService</span>:

builder.Services.AddHttpClient<elasticsearchservice>();</elasticsearchservice>

4. Define Log Entity Class

Create an entity class for recording request logs called <span>ApiLogEntity</span>:

public class ApiLogEntity
{
    public string Name { get; set; }
    public string Ip { get; set; }
    public string Url { get; set; }
    public string ReqMethod { get; set; }
    public DateTime OpTime { get; set; }
    public string Account { get; set; }
    public int StatusCode { get; set; }
    public string ResponseBody { get; set; }
}

5. Write Logging Middleware

Intercept requests through custom middleware and write logs to Elasticsearch:

public class RequestLoggingMiddleware
{
    private readonly RequestDelegate _next;
    private readonly ILogger<requestloggingmiddleware> _logger;
    private readonly ElasticsearchService _esService;

    public RequestLoggingMiddleware(RequestDelegate next, ILogger<requestloggingmiddleware> logger, ElasticsearchService esService)
    {
        _next = next;
        _logger = logger;
        _esService = esService;
    }

    public async Task Invoke(HttpContext context)
    {
        // Check if logging is enabled
        if (!App.Configuration["AppLogEntity"].ToBool())
        {
            await _next(context);
            return;
        }

        var request = context.Request;

        if (!request.Path.Value.Contains("api"))
        {
            await _next(context);
            return;
        }

        request.EnableBuffering();
        var originalBodyStream = context.Response.Body;
        await using var responseBody = new MemoryStream();
        context.Response.Body = responseBody;

        try
        {
            await _next(context);

            context.Response.Body.Seek(0, SeekOrigin.Begin);
            var responseText = await new StreamReader(context.Response.Body).ReadToEndAsync();
            context.Response.Body.Seek(0, SeekOrigin.Begin);

            var entity = new ApiLogEntity
            {
                Name = Environment.MachineName,
                Ip = context.Connection.RemoteIpAddress?.ToString(),
                Url = request.Path + request.QueryString,
                ReqMethod = request.Method,
                OpTime = DateTime.Now,
                Account = "wx-" + App.User.FindFirst("UserID")?.Value,
                StatusCode = context.Response.StatusCode,
                ResponseBody = responseText
            };

            await _esService.IndexAsync(entity, "jltlogs");
        }
        finally
        {
            await responseBody.CopyToAsync(originalBodyStream);
            context.Response.Body = originalBodyStream;
        }
    }
}</requestloggingmiddleware></requestloggingmiddleware>

Register the middleware:

app.UseMiddleware<requestloggingmiddleware>();</requestloggingmiddleware>

6. Test the Interface

1. Write Log Document

• • Request Path: <span>POST /search/index</span>
• • Body Example:

{
  "title": "Test Document",
  "description": "Elasticsearch integration with HttpClient"
}

If you find this article helpful, feel free to like, bookmark, and share it with more developers! Let’s learn and progress together!