xwl2023/Yi.Framework
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增向量嵌入服务支持

Browse Source 
新增SiliconFlow向量嵌入服务实现，支持文本向量化功能：
- 新增ITextEmbeddingService接口和SiliconFlowTextEmbeddingService实现
- 新增EmbeddingCreateRequest/Response等向量相关DTO
- 在AiGateWayManager中新增EmbeddingForStatisticsAsync方法
- 在OpenApiService中新增向量生成API接口
- 扩展ModelTypeEnum枚举支持Embedding类型
- 优化ThorChatMessage的Content属性处理逻辑
This commit is contained in:
chenchun
2025-08-11 15:29:24 +08:00
parent bbe5b01872
commit 25eebec8f7
11 changed files with 405 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application.Contracts/Dtos/OpenAi/Embeddings/EmbeddingCreateRequest.cs Normal file
View File

@@ -0,0 +1,79 @@
using System.ComponentModel.DataAnnotations;
using System.Text.Json.Serialization;
namespace Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Embeddings;
//TODO add model validation
//TODO check what is string or array for prompt,..
public record EmbeddingCreateRequest
{
/// <summary>
/// Input text to get embeddings for, encoded as a string or array of tokens. To get embeddings for multiple inputs
/// in a single request, pass an array of strings or array of token arrays. Each input must not exceed 2048 tokens in
/// length.
/// Unless your are embedding code, we suggest replacing newlines (`\n`) in your input with a single space, as we have
/// observed inferior results when newlines are present.
/// </summary>
/// <see href="https://platform.openai.com/docs/api-reference/embeddings/create#embeddings/create-input" />
[JsonIgnore]
public List<string>? InputAsList { get; set; }
/// <summary>
/// Input text to get embeddings for, encoded as a string or array of tokens. To get embeddings for multiple inputs
/// in a single request, pass an array of strings or array of token arrays. Each input must not exceed 2048 tokens in
/// length.
/// Unless your are embedding code, we suggest replacing newlines (`\n`) in your input with a single space, as we have
/// observed inferior results when newlines are present.
/// </summary>
/// <see href="https://platform.openai.com/docs/api-reference/embeddings/create#embeddings/create-input" />
[JsonIgnore]
public string? Input { get; set; }
[JsonPropertyName("input")]
public IList<string>? InputCalculated
{
get
{
if (Input != null && InputAsList != null)
{
throw new ValidationException(
"Input and InputAsList can not be assigned at the same time. One of them is should be null.");
}
if (Input != null)
{
return new List<string> { Input };
}
return InputAsList;
}
}
/// <summary>
/// ID of the model to use. You can use the [List models](/docs/api-reference/models/list) API to see all of your
/// available models, or see our [Model overview](/docs/models/overview) for descriptions of them.
/// </summary>
/// <see href="https://platform.openai.com/docs/api-reference/embeddings/create#embeddings/create-model" />
[JsonPropertyName("model")]
public string? Model { get; set; }
/// <summary>
/// The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.
/// </summary>
/// <see href="https://platform.openai.com/docs/api-reference/embeddings/create#embeddings-create-dimensions" />
[JsonPropertyName("dimensions")]
public int? Dimensions { get; set; }
/// <summary>
/// The format to return the embeddings in. Can be either float or base64.
/// </summary>
/// <returns></returns>
[JsonPropertyName("encoding_format")]
public string? EncodingFormat { get; set; }
public IEnumerable<ValidationResult> Validate()
{
throw new NotImplementedException();
}
}

111
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application.Contracts/Dtos/OpenAi/Embeddings/EmbeddingCreateResponse.cs Normal file
View File

@@ -0,0 +1,111 @@
using System.Buffers;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Embeddings;
public record EmbeddingCreateResponse : ThorBaseResponse
{
[JsonPropertyName("model")] public string Model { get; set; }
[JsonPropertyName("data")] public List<EmbeddingResponse> Data { get; set; } = [];
/// <summary>
/// 类型转换，如果类型是base64,则将float[]转换为base64,如果是空或是float和原始类型一样，则不转换
/// </summary>
public void ConvertEmbeddingData(string? encodingFormat)
{
if (Data.Count == 0)
{
return;
}
switch (encodingFormat)
{
// 判断第一个是否是float[]，如果是则不转换
case null or "float" when Data[0].Embedding is float[]:
return;
// 否则转换成float[]
case null or "float":
{
foreach (var embeddingResponse in Data)
{
if (embeddingResponse.Embedding is string base64)
{
embeddingResponse.Embedding = Convert.FromBase64String(base64);
}
}
return;
}
// 判断第一个是否是string，如果是则不转换
case "base64" when Data[0].Embedding is string:
return;
// 否则转换成base64
case "base64":
{
foreach (var embeddingResponse in Data)
{
if (embeddingResponse.Embedding is JsonElement str)
{
if (str.ValueKind == JsonValueKind.Array)
{
var floats = str.EnumerateArray().Select(element => element.GetSingle()).ToArray();
embeddingResponse.Embedding = ConvertFloatArrayToBase64(floats);
}
}
else if (embeddingResponse.Embedding is IList<double> doubles)
{
embeddingResponse.Embedding = ConvertFloatArrayToBase64(doubles.ToArray());
}
}
break;
}
}
}
public static string ConvertFloatArrayToBase64(double[] floatArray)
{
// 将 float[] 转换成 byte[]
byte[] byteArray = ArrayPool<byte>.Shared.Rent(floatArray.Length * sizeof(float));
try
{
Buffer.BlockCopy(floatArray, 0, byteArray, 0, byteArray.Length);
// 将 byte[] 转换成 base64 字符串
return Convert.ToBase64String(byteArray);
}
finally
{
ArrayPool<byte>.Shared.Return(byteArray);
}
}
public static string ConvertFloatArrayToBase64(float[] floatArray)
{
// 将 float[] 转换成 byte[]
byte[] byteArray = ArrayPool<byte>.Shared.Rent(floatArray.Length * sizeof(float));
try
{
Buffer.BlockCopy(floatArray, 0, byteArray, 0, floatArray.Length);
// 将 byte[] 转换成 base64 字符串
return Convert.ToBase64String(byteArray);
}
finally
{
ArrayPool<byte>.Shared.Return(byteArray);
}
}
[JsonPropertyName("usage")] public ThorUsageResponse? Usage { get; set; }
}
public record EmbeddingResponse
{
[JsonPropertyName("index")] public int? Index { get; set; }
[JsonPropertyName("embedding")] public object Embedding { get; set; }
}

22
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application.Contracts/Dtos/OpenAi/Embeddings/ThorEmbeddingInput.cs Normal file
View File

@@ -0,0 +1,22 @@
using System.Text.Json.Serialization;
namespace Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Embeddings;
public sealed class ThorEmbeddingInput
{
[JsonPropertyName("model")]
public string Model { get; set; }
[JsonPropertyName("input")]
public object Input { get; set; }
[JsonPropertyName("encoding_format")]
public string EncodingFormat { get; set; }
[JsonPropertyName("dimensions")]
public int? Dimensions { get; set; }
[JsonPropertyName("user")]
public string? User { get; set; }
}

26
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application.Contracts/Dtos/OpenAi/ThorChatMessage.cs
View File

@@ -14,7 +14,6 @@ public class ThorChatMessage
/// </summary>
public ThorChatMessage()
{
}
/// <summary>
@@ -74,20 +73,19 @@ public class ThorChatMessage
{
if (value is JsonElement str)
{
if (str.ValueKind == JsonValueKind.String)
{
Content = value?.ToString();
}
else if (str.ValueKind == JsonValueKind.Array)
if (str.ValueKind == JsonValueKind.Array)
{
Contents = JsonSerializer.Deserialize<IList<ThorChatMessageContent>>(value?.ToString());
}
}
else if (value is string strInput)
{
Content = strInput;
}
else
{
Content = value?.ToString();
}
}
}
@@ -108,15 +106,14 @@ public class ThorChatMessage
/// </summary>
[JsonPropertyName("function_call")]
public ThorChatMessageFunction? FunctionCall { get; set; }
/// <summary>
/// 【可选】推理内容
/// </summary>
[JsonPropertyName("reasoning_content")]
public string? ReasoningContent { get; set; }
[JsonPropertyName("id")]
public string? Id { get; set; }
[JsonPropertyName("id")] public string? Id { get; set; }
/// <summary>
/// 工具调用列表，模型生成的工具调用，例如函数调用。<br/>
@@ -164,14 +161,15 @@ public class ThorChatMessage
/// <param name="name">参与者的可选名称。提供模型信息以区分同一角色的参与者。</param>
/// <param name="toolCalls">工具调用参数列表</param>
/// <returns></returns>
public static ThorChatMessage CreateAssistantMessage(string content, string? name = null, List<ThorToolCall> toolCalls = null)
public static ThorChatMessage CreateAssistantMessage(string content, string? name = null,
List<ThorToolCall> toolCalls = null)
{
return new()
{
Role = ThorChatMessageRoleConst.Assistant,
Content = content,
Name = name,
ToolCalls=toolCalls,
ToolCalls = toolCalls,
};
}
@@ -187,7 +185,7 @@ public class ThorChatMessage
{
Role = ThorChatMessageRoleConst.Tool,
Content = content,
ToolCallId= toolCallId
ToolCallId = toolCallId
};
}
}

15
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application/Services/OpenApiService.cs
View File

@@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Volo.Abp.Application.Services;
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi;
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Embeddings;
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Images;
using Yi.Framework.AiHub.Domain.Entities.Model;
using Yi.Framework.AiHub.Domain.Extensions;
@@ -72,6 +73,20 @@ public class OpenApiService : ApplicationService
await _aiBlacklistManager.VerifiyAiBlacklist(userId);
await _aiGateWayManager.CreateImageForStatisticsAsync(httpContext, userId, null, input);
}
/// <summary>
/// 向量生成
/// </summary>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
[HttpPost("openApi/v1/embeddings")]
public async Task EmbeddingAsync([FromBody] ThorEmbeddingInput input, CancellationToken cancellationToken)
{
var httpContext = this._httpContextAccessor.HttpContext;
var userId = await _tokenManager.GetUserIdAsync(GetTokenByHttpContext(httpContext));
await _aiBlacklistManager.VerifiyAiBlacklist(userId);
await _aiGateWayManager.EmbeddingForStatisticsAsync(httpContext, userId, null, input);
}
/// <summary>

4
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Enums/ModelTypeEnum.cs
View File

@@ -1,8 +1,8 @@
namespace Yi.Framework.AiHub.Domain.Shared.Enums;
public enum ModelTypeEnum
{
Chat = 0,
Image = 1
Image = 1,
Embedding = 2
}

19
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/AiGateWay/ITextEmbeddingService.cs Normal file
View File

@@ -0,0 +1,19 @@
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Embeddings;
using Yi.Framework.AiHub.Domain.Shared.Dtos;
namespace Yi.Framework.AiHub.Domain.AiGateWay;
public interface ITextEmbeddingService
{
/// <summary>
///
/// </summary>
/// <param name="createEmbeddingModel"></param>
/// <param name="aiModelDescribe"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
Task<EmbeddingCreateResponse> EmbeddingAsync(
EmbeddingCreateRequest createEmbeddingModel,
AiModelDescribe? aiModelDescribe = null,
CancellationToken cancellationToken = default);
}

24
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/AiGateWay/Impl/ThorSiliconFlow/Embeddings/SiliconFlowTextEmbeddingService.cs Normal file
View File

@@ -0,0 +1,24 @@
using System.Net.Http.Json;
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Embeddings;
using Yi.Framework.AiHub.Domain.Shared.Dtos;
namespace Yi.Framework.AiHub.Domain.AiGateWay.Impl.ThorSiliconFlow.Embeddings;
public sealed class SiliconFlowTextEmbeddingService
: ITextEmbeddingService
{
public async Task<EmbeddingCreateResponse> EmbeddingAsync(
EmbeddingCreateRequest createEmbeddingModel,
AiModelDescribe? options = null,
CancellationToken cancellationToken = default)
{
var response = await HttpClientFactory.GetHttpClient(options.Endpoint).PostJsonAsync(
options?.Endpoint.TrimEnd('/') + "/v1/embeddings",
createEmbeddingModel, options!.ApiKey);
var result =
await response.Content.ReadFromJsonAsync<EmbeddingCreateResponse>(cancellationToken: cancellationToken);
return result;
}
}

119
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs
View File

@@ -2,6 +2,7 @@ using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@@ -10,6 +11,7 @@ using Newtonsoft.Json.Serialization;
using Volo.Abp.Domain.Services;
using Yi.Framework.AiHub.Application.Contracts.Dtos;
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi;
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Embeddings;
using Yi.Framework.AiHub.Application.Contracts.Dtos.OpenAi.Images;
using Yi.Framework.AiHub.Domain.AiGateWay;
using Yi.Framework.AiHub.Domain.AiGateWay.Exceptions;
@@ -276,19 +278,20 @@ public class AiGateWayManager : DomainService
/// <param name="request"></param>
/// <exception cref="BusinessException"></exception>
/// <exception cref="Exception"></exception>
public async Task CreateImageForStatisticsAsync(HttpContext context,Guid? userId,Guid? sessionId, ImageCreateRequest request)
public async Task CreateImageForStatisticsAsync(HttpContext context, Guid? userId, Guid? sessionId,
ImageCreateRequest request)
{
try
{
var model = request.Model;
if (string.IsNullOrEmpty(model)) model = "dall-e-2";
var modelDescribe = await GetModelAsync(model);
// 获取渠道指定的实现类型的服务
var imageService =
LazyServiceProvider.GetRequiredKeyedService<IImageService>(modelDescribe.HandlerName);
var response = await imageService.CreateImage(request, modelDescribe);
if (response.Error != null || response.Results.Count == 0)
@@ -297,7 +300,7 @@ public class AiGateWayManager : DomainService
}
await context.Response.WriteAsJsonAsync(response);
await _aiMessageManager.CreateUserMessageAsync(userId, sessionId,
new MessageInputDto
{
@@ -322,4 +325,110 @@ public class AiGateWayManager : DomainService
throw new UserFriendlyException(errorContent);
}
}
/// <summary>
/// 向量生成
/// </summary>
/// <param name="context"></param>
/// <param name="sessionId"></param>
/// <param name="input"></param>
/// <param name="userId"></param>
/// <exception cref="Exception"></exception>
/// <exception cref="BusinessException"></exception>
public async Task EmbeddingForStatisticsAsync(HttpContext context, Guid? userId, Guid? sessionId,
ThorEmbeddingInput input)
{
try
{
if (input == null) throw new Exception("模型校验异常");
using var embedding =
Activity.Current?.Source.StartActivity("向量模型调用");
var modelDescribe = await GetModelAsync(input.Model);
// 获取渠道指定的实现类型的服务
var embeddingService =
LazyServiceProvider.GetRequiredKeyedService<ITextEmbeddingService>(modelDescribe.HandlerName);
var embeddingCreateRequest = new EmbeddingCreateRequest
{
Model = input.Model,
EncodingFormat = input.EncodingFormat
};
//dto进行转换，支持多种格式
if (input.Input is JsonElement str)
{
if (str.ValueKind == JsonValueKind.Array)
{
var inputString = str.EnumerateArray().Select(x => x.ToString()).ToArray();
embeddingCreateRequest.InputAsList = inputString.ToList();
}
else
{
throw new Exception("Input，输入格式错误，非string或Array类型");
}
}
else if (input.Input is string strInput)
{
embeddingCreateRequest.Input = strInput;
}
else
{
throw new Exception("Input，输入格式错误，未找到类型");
}
var stream =
await embeddingService.EmbeddingAsync(embeddingCreateRequest, modelDescribe, context.RequestAborted);
var usage = new ThorUsageResponse()
{
InputTokens = stream.Usage?.InputTokens ?? 0,
CompletionTokens = 0,
TotalTokens = stream.Usage?.InputTokens ?? 0
};
await context.Response.WriteAsJsonAsync(new
{
input.Model,
stream.Data,
stream.Error,
stream.ObjectTypeName,
Usage = usage
});
await _aiMessageManager.CreateUserMessageAsync(userId, sessionId,
new MessageInputDto
{
Content = string.Empty,
ModelId = input.Model,
TokenUsage = usage,
});
await _aiMessageManager.CreateSystemMessageAsync(userId, sessionId,
new MessageInputDto
{
Content = string.Empty,
ModelId = input.Model,
TokenUsage = usage
});
await _usageStatisticsManager.SetUsageAsync(userId, input.Model, usage);
}
catch (ThorRateLimitException)
{
context.Response.StatusCode = 429;
}
catch (UnauthorizedAccessException e)
{
context.Response.StatusCode = 401;
}
catch (Exception e)
{
var errorContent = $"嵌入Ai异常，异常信息：\n当前Ai模型：{input.Model}\n异常信息：{e.Message}\n异常堆栈:{e}";
throw new UserFriendlyException(errorContent);
}
}
}

4
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Yi.Framework.AiHub.Domain.csproj
View File

@@ -13,4 +13,8 @@
<ProjectReference Include="..\Yi.Framework.AiHub.Domain.Shared\Yi.Framework.AiHub.Domain.Shared.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="AiGateWay\Impl\ThorSiliconFlow\" />
</ItemGroup>
</Project>

3
Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/YiFrameworkAiHubDomainModule.cs
View File

@@ -7,6 +7,7 @@ using Yi.Framework.AiHub.Domain.AiGateWay.Impl.ThorAzureDatabricks.Chats;
using Yi.Framework.AiHub.Domain.AiGateWay.Impl.ThorAzureOpenAI.Chats;
using Yi.Framework.AiHub.Domain.AiGateWay.Impl.ThorAzureOpenAI.Images;
using Yi.Framework.AiHub.Domain.AiGateWay.Impl.ThorDeepSeek.Chats;
using Yi.Framework.AiHub.Domain.AiGateWay.Impl.ThorSiliconFlow.Embeddings;
using Yi.Framework.AiHub.Domain.Shared;
using Yi.Framework.Mapster;
@@ -35,6 +36,8 @@ namespace Yi.Framework.AiHub.Domain
services.AddKeyedTransient<IImageService, AzureOpenAIServiceImageService>(
nameof(AzureOpenAIServiceImageService));
services.AddKeyedTransient<ITextEmbeddingService, SiliconFlowTextEmbeddingService>(nameof(SiliconFlowTextEmbeddingService));
//ai模型特殊性兼容处理
Configure<SpecialCompatibleOptions>(options =>