diff --git a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application/Services/OpenApiService.cs b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application/Services/OpenApiService.cs index 26dcda77..5b150cb1 100644 --- a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application/Services/OpenApiService.cs +++ b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Application/Services/OpenApiService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Volo.Abp.Application.Services; using Volo.Abp.Users; using Yi.Framework.AiHub.Domain.Entities; +using Yi.Framework.AiHub.Domain.Entities.Chat; using Yi.Framework.AiHub.Domain.Entities.Model; using Yi.Framework.AiHub.Domain.Extensions; using Yi.Framework.AiHub.Domain.Managers; @@ -30,11 +31,11 @@ public class OpenApiService : ApplicationService private readonly AiBlacklistManager _aiBlacklistManager; private readonly IAccountService _accountService; private readonly PremiumPackageManager _premiumPackageManager; - + private readonly ISqlSugarRepository _imageStoreRepository; public OpenApiService(IHttpContextAccessor httpContextAccessor, ILogger logger, TokenManager tokenManager, AiGateWayManager aiGateWayManager, ISqlSugarRepository aiModelRepository, AiBlacklistManager aiBlacklistManager, - IAccountService accountService, PremiumPackageManager premiumPackageManager) + IAccountService accountService, PremiumPackageManager premiumPackageManager, ISqlSugarRepository imageStoreRepository) { _httpContextAccessor = httpContextAccessor; _logger = logger; @@ -44,6 +45,7 @@ public class OpenApiService : ApplicationService _aiBlacklistManager = aiBlacklistManager; _accountService = accountService; _premiumPackageManager = premiumPackageManager; + _imageStoreRepository = imageStoreRepository; } /// @@ -259,11 +261,13 @@ public class OpenApiService : ApplicationService /// 生成-Gemini (尊享服务专用) /// /// + /// /// /// /// [HttpPost("openApi/v1beta/models/{modelId}:{action:regex(^(generateContent|streamGenerateContent)$)}")] public async Task GenerateContentAsync([FromBody] JsonElement input, + [FromQuery] bool isAsync, [FromRoute] string modelId, [FromQuery] string? alt, CancellationToken cancellationToken) { @@ -294,6 +298,18 @@ public class OpenApiService : ApplicationService throw new UserFriendlyException("尊享token包用量不足，请先购买尊享token包"); } + //如果异步，直接走job处理进行存储 + if (isAsync) + { + var task = new ImageStoreTaskAggregateRoot(); + await _imageStoreRepository.InsertAsync(task); + await _httpContextAccessor.HttpContext.Response.WriteAsJsonAsync(new + { + Id = task.Id + }, cancellationToken); + //todo 发送job,参数怎么办？需要先全存下来吗？全存下来，就要解析全部提示词 和 附件内容了 + + } //ai网关代理httpcontext if (alt == "sse") { diff --git a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs index 642fd850..849d9c15 100644 --- a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs +++ b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs @@ -30,4 +30,16 @@ public static class GeminiGenerateContentAcquirer TotalTokens = inputTokens + outputTokens, }; } + + /// + /// 获取图片url，不包含前缀 + /// + /// + /// + public static string GetImageBase64(JsonElement response) + { + //todo + //获取他的base64字符串 + return string.Empty; + } } \ No newline at end of file diff --git a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Enums/TaskStatusEnum.cs b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Enums/TaskStatusEnum.cs new file mode 100644 index 00000000..85f45f51 --- /dev/null +++ b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Enums/TaskStatusEnum.cs @@ -0,0 +1,8 @@ +namespace Yi.Framework.AiHub.Domain.Shared.Enums; + +public enum TaskStatusEnum +{ + Processing, + Success, + Fail +} \ No newline at end of file diff --git a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Entities/Chat/ImageStoreTaskAggregateRoot.cs b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Entities/Chat/ImageStoreTaskAggregateRoot.cs new file mode 100644 index 00000000..cd26ab8e --- /dev/null +++ b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Entities/Chat/ImageStoreTaskAggregateRoot.cs @@ -0,0 +1,34 @@ +using SqlSugar; +using Volo.Abp.Domain.Entities.Auditing; +using Yi.Framework.AiHub.Domain.Shared.Enums; + +namespace Yi.Framework.AiHub.Domain.Entities.Chat; + +[SugarTable("Ai_ImageStoreTask")] +public class ImageStoreTaskAggregateRoot : FullAuditedAggregateRoot +{ + /// + /// 图片绝对路径 + /// + public string? StoreUrl { get; set; } + + /// + /// 任务状态 + /// + public TaskStatusEnum TaskStatus { get; set; } = TaskStatusEnum.Processing; + + /// + /// 用户id + /// + public Guid UserId { get; set; } + + /// + /// 设置成功 + /// + /// + public void SetSuccess(string storeUrl) + { + TaskStatus = TaskStatusEnum.Success; + StoreUrl = storeUrl; + } +} \ No newline at end of file diff --git a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs index 17325f5c..1c30bb17 100644 --- a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs +++ b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs @@ -12,6 +12,7 @@ using Newtonsoft.Json.Serialization; using Volo.Abp.Domain.Services; using Yi.Framework.AiHub.Domain.AiGateWay; using Yi.Framework.AiHub.Domain.AiGateWay.Exceptions; +using Yi.Framework.AiHub.Domain.Entities.Chat; using Yi.Framework.AiHub.Domain.Entities.Model; using Yi.Framework.AiHub.Domain.Shared.Consts; using Yi.Framework.AiHub.Domain.Shared.Dtos; @@ -39,11 +40,12 @@ public class AiGateWayManager : DomainService private readonly UsageStatisticsManager _usageStatisticsManager; private readonly ISpecialCompatible _specialCompatible; private PremiumPackageManager? _premiumPackageManager; - + private readonly ISqlSugarRepository _imageStoreTaskRepository; public AiGateWayManager(ISqlSugarRepository aiAppRepository, ILogger logger, AiMessageManager aiMessageManager, UsageStatisticsManager usageStatisticsManager, - ISpecialCompatible specialCompatible, ISqlSugarRepository aiModelRepository) + ISpecialCompatible specialCompatible, ISqlSugarRepository aiModelRepository, + ISqlSugarRepository imageStoreTaskRepository) { _aiAppRepository = aiAppRepository; _logger = logger; @@ -51,6 +53,7 @@ public class AiGateWayManager : DomainService _usageStatisticsManager = usageStatisticsManager; _specialCompatible = specialCompatible; _aiModelRepository = aiModelRepository; + _imageStoreTaskRepository = imageStoreTaskRepository; } private PremiumPackageManager PremiumPackageManager => @@ -93,7 +96,7 @@ public class AiGateWayManager : DomainService return aiModelDescribe; } - + /// /// 聊天完成-非流式 /// @@ -155,7 +158,7 @@ public class AiGateWayManager : DomainService await response.WriteAsJsonAsync(data, cancellationToken); } - + /// /// 聊天完成-缓存处理 /// @@ -185,8 +188,8 @@ public class AiGateWayManager : DomainService var modelDescribe = await GetModelAsync(ModelApiTypeEnum.OpenAi, request.Model); var chatService = LazyServiceProvider.GetRequiredKeyedService(modelDescribe.HandlerName); - - var completeChatResponse = chatService.CompleteChatStreamAsync(modelDescribe,request, cancellationToken); + + var completeChatResponse = chatService.CompleteChatStreamAsync(modelDescribe, request, cancellationToken); var tokenUsage = new ThorUsageResponse(); //缓存队列算法 @@ -301,8 +304,8 @@ public class AiGateWayManager : DomainService } } } - - + + /// /// 图片生成 /// @@ -370,8 +373,8 @@ public class AiGateWayManager : DomainService throw new UserFriendlyException(errorContent); } } - - + + /// /// 向量生成 /// @@ -483,7 +486,7 @@ public class AiGateWayManager : DomainService throw new UserFriendlyException(errorContent); } } - + /// /// Anthropic聊天完成-非流式 @@ -544,7 +547,7 @@ public class AiGateWayManager : DomainService await response.WriteAsJsonAsync(data, cancellationToken); } - + /// /// Anthropic聊天完成-缓存处理 /// @@ -568,13 +571,13 @@ public class AiGateWayManager : DomainService response.ContentType = "text/event-stream;charset=utf-8;"; response.Headers.TryAdd("Cache-Control", "no-cache"); response.Headers.TryAdd("Connection", "keep-alive"); - + _specialCompatible.AnthropicCompatible(request); var modelDescribe = await GetModelAsync(ModelApiTypeEnum.Claude, request.Model); var chatService = LazyServiceProvider.GetRequiredKeyedService(modelDescribe.HandlerName); - - var completeChatResponse = chatService.StreamChatCompletionsAsync(modelDescribe,request, cancellationToken); + + var completeChatResponse = chatService.StreamChatCompletionsAsync(modelDescribe, request, cancellationToken); ThorUsageResponse? tokenUsage = null; StringBuilder backupSystemContent = new StringBuilder(); try @@ -655,10 +658,10 @@ public class AiGateWayManager : DomainService var chatService = LazyServiceProvider.GetRequiredKeyedService(modelDescribe.HandlerName); var data = await chatService.ResponsesAsync(modelDescribe, request, cancellationToken); - + data.SupplementalMultiplier(modelDescribe.Multiplier); - - var tokenUsage= new ThorUsageResponse + + var tokenUsage = new ThorUsageResponse { InputTokens = data.Usage.InputTokens, OutputTokens = data.Usage.OutputTokens, @@ -673,7 +676,7 @@ public class AiGateWayManager : DomainService ModelId = request.Model, TokenUsage = tokenUsage, }, tokenId); - + await _aiMessageManager.CreateSystemMessageAsync(userId.Value, sessionId, new MessageInputDto { @@ -681,9 +684,9 @@ public class AiGateWayManager : DomainService ModelId = request.Model, TokenUsage = tokenUsage }, tokenId); - + await _usageStatisticsManager.SetUsageAsync(userId.Value, request.Model, tokenUsage, tokenId); - + // 扣减尊享token包用量 var totalTokens = tokenUsage.TotalTokens ?? 0; if (totalTokens > 0) @@ -694,8 +697,8 @@ public class AiGateWayManager : DomainService await response.WriteAsJsonAsync(data, cancellationToken); } - - + + /// /// OpenAi响应-流式-缓存处理 /// @@ -719,12 +722,12 @@ public class AiGateWayManager : DomainService response.ContentType = "text/event-stream;charset=utf-8;"; response.Headers.TryAdd("Cache-Control", "no-cache"); response.Headers.TryAdd("Connection", "keep-alive"); - + var modelDescribe = await GetModelAsync(ModelApiTypeEnum.Response, request.Model); var chatService = LazyServiceProvider.GetRequiredKeyedService(modelDescribe.HandlerName); - - var completeChatResponse = chatService.ResponsesStreamAsync(modelDescribe,request, cancellationToken); + + var completeChatResponse = chatService.ResponsesStreamAsync(modelDescribe, request, cancellationToken); ThorUsageResponse? tokenUsage = null; try { @@ -734,20 +737,20 @@ public class AiGateWayManager : DomainService if (responseResult.Item1.Contains("response.completed")) { var obj = responseResult.Item2!.Value; - int inputTokens = obj.GetPath("response","usage","input_tokens").GetInt(); - int outputTokens = obj.GetPath("response","usage","output_tokens").GetInt(); - inputTokens=Convert.ToInt32(inputTokens * modelDescribe.Multiplier); - outputTokens=Convert.ToInt32(outputTokens * modelDescribe.Multiplier); + int inputTokens = obj.GetPath("response", "usage", "input_tokens").GetInt(); + int outputTokens = obj.GetPath("response", "usage", "output_tokens").GetInt(); + inputTokens = Convert.ToInt32(inputTokens * modelDescribe.Multiplier); + outputTokens = Convert.ToInt32(outputTokens * modelDescribe.Multiplier); tokenUsage = new ThorUsageResponse { - PromptTokens =inputTokens, + PromptTokens = inputTokens, InputTokens = inputTokens, OutputTokens = outputTokens, CompletionTokens = outputTokens, - TotalTokens = inputTokens+outputTokens, + TotalTokens = inputTokens + outputTokens, }; } - + await WriteAsEventStreamDataAsync(httpContext, responseResult.Item1, responseResult.Item2, cancellationToken); } @@ -762,7 +765,7 @@ public class AiGateWayManager : DomainService await _aiMessageManager.CreateUserMessageAsync(userId, sessionId, new MessageInputDto { - Content = "不予存储" , + Content = "不予存储", ModelId = request.Model, TokenUsage = tokenUsage, }, tokenId); @@ -770,7 +773,7 @@ public class AiGateWayManager : DomainService await _aiMessageManager.CreateSystemMessageAsync(userId, sessionId, new MessageInputDto { - Content = "不予存储" , + Content = "不予存储", ModelId = request.Model, TokenUsage = tokenUsage }, tokenId); @@ -814,9 +817,9 @@ public class AiGateWayManager : DomainService LazyServiceProvider.GetRequiredKeyedService(modelDescribe.HandlerName); var data = await chatService.GenerateContentAsync(modelDescribe, request, cancellationToken); - var tokenUsage = GeminiGenerateContentAcquirer.GetUsage(data); + var tokenUsage = GeminiGenerateContentAcquirer.GetUsage(data); tokenUsage.SetSupplementalMultiplier(modelDescribe.Multiplier); - + if (userId is not null) { await _aiMessageManager.CreateUserMessageAsync(userId.Value, sessionId, @@ -826,7 +829,7 @@ public class AiGateWayManager : DomainService ModelId = modelId, TokenUsage = tokenUsage, }, tokenId); - + await _aiMessageManager.CreateSystemMessageAsync(userId.Value, sessionId, new MessageInputDto { @@ -834,9 +837,9 @@ public class AiGateWayManager : DomainService ModelId = modelId, TokenUsage = tokenUsage }, tokenId); - + await _usageStatisticsManager.SetUsageAsync(userId.Value, modelId, tokenUsage, tokenId); - + // 扣减尊享token包用量 var totalTokens = tokenUsage.TotalTokens ?? 0; if (totalTokens > 0) @@ -874,23 +877,25 @@ public class AiGateWayManager : DomainService response.ContentType = "text/event-stream;charset=utf-8;"; response.Headers.TryAdd("Cache-Control", "no-cache"); response.Headers.TryAdd("Connection", "keep-alive"); - + var modelDescribe = await GetModelAsync(ModelApiTypeEnum.GenerateContent, modelId); var chatService = LazyServiceProvider.GetRequiredKeyedService(modelDescribe.HandlerName); - - var completeChatResponse = chatService.GenerateContentStreamAsync(modelDescribe,request, cancellationToken); + + var completeChatResponse = chatService.GenerateContentStreamAsync(modelDescribe, request, cancellationToken); ThorUsageResponse? tokenUsage = null; try { await foreach (var responseResult in completeChatResponse) { - if ( responseResult!.Value.GetPath("candidates", 0, "finishReason").GetString() == "STOP") + if (responseResult!.Value.GetPath("candidates", 0, "finishReason").GetString() == "STOP") { tokenUsage = GeminiGenerateContentAcquirer.GetUsage(responseResult!.Value); tokenUsage.SetSupplementalMultiplier(modelDescribe.Multiplier); } - await response.WriteAsync($"data: {JsonSerializer.Serialize(responseResult)}\n\n", Encoding.UTF8, cancellationToken).ConfigureAwait(false); + + await response.WriteAsync($"data: {JsonSerializer.Serialize(responseResult)}\n\n", Encoding.UTF8, + cancellationToken).ConfigureAwait(false); await response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); } } @@ -904,7 +909,7 @@ public class AiGateWayManager : DomainService await _aiMessageManager.CreateUserMessageAsync(userId, sessionId, new MessageInputDto { - Content = "不予存储" , + Content = "不予存储", ModelId = modelId, TokenUsage = tokenUsage, }, tokenId); @@ -912,7 +917,7 @@ public class AiGateWayManager : DomainService await _aiMessageManager.CreateSystemMessageAsync(userId, sessionId, new MessageInputDto { - Content = "不予存储" , + Content = "不予存储", ModelId = modelId, TokenUsage = tokenUsage }, tokenId); @@ -929,9 +934,102 @@ public class AiGateWayManager : DomainService } } } - - - + + + /// + /// Gemini 生成(Image)-非流式-缓存处理 + /// 返回图片绝对路径 + /// + /// + /// + /// + /// + /// + /// + /// + public async Task GeminiGenerateContentImageForStatisticsAsync( + Guid taskId, + string modelId, + JsonElement request, + Guid userId, + Guid? sessionId = null, + Guid? tokenId = null, + CancellationToken cancellationToken = default) + { + var imageStoreTask = await _imageStoreTaskRepository.GetFirstAsync(x => x.Id == taskId); + var modelDescribe = await GetModelAsync(ModelApiTypeEnum.GenerateContent, modelId); + + var chatService = + LazyServiceProvider.GetRequiredKeyedService(modelDescribe.HandlerName); + var data = await chatService.GenerateContentAsync(modelDescribe, request, cancellationToken); + + //解析json，获取base64字符串 + var imageBase64 = GeminiGenerateContentAcquirer.GetImageBase64(data); + + //base64字符串存储，返回绝对路径，用于最后存储 + var storeUrl = Base64ToPng(imageBase64, "存储的路径？这个放什么"); + + + var tokenUsage = new ThorUsageResponse + { + InputTokens = (int)modelDescribe.Multiplier, + OutputTokens = (int)modelDescribe.Multiplier, + TotalTokens = (int)modelDescribe.Multiplier, + }; + + await _aiMessageManager.CreateSystemMessageAsync(userId, sessionId, + new MessageInputDto + { + Content = "不予存储", + ModelId = modelId, + TokenUsage = tokenUsage + }, tokenId); + + await _usageStatisticsManager.SetUsageAsync(userId, modelId, tokenUsage, tokenId); + + // 扣减尊享token包用量 + var totalTokens = tokenUsage.TotalTokens ?? 0; + if (totalTokens > 0) + { + await PremiumPackageManager.TryConsumeTokensAsync(userId, totalTokens); + } + + //设置存储url + imageStoreTask.SetSuccess(storeUrl); + await _imageStoreTaskRepository.UpdateAsync(imageStoreTask); + } + + /// + /// 将 Base64 字符串转换为 PNG 图片并保存 + /// + /// Base64 字符串（可以包含或不包含 data:image/png;base64, 前缀） + /// 输出文件路径 + private string Base64ToPng(string base64String, string outputPath) + { + try + { + // 移除可能存在的 data URI scheme 前缀 + if (base64String.Contains(",")) + { + base64String = base64String.Substring(base64String.IndexOf(",") + 1); + } + + // 将 base64 字符串转换为字节数组 + byte[] imageBytes = Convert.FromBase64String(base64String); + + // 将字节数组写入文件 + File.WriteAllBytes(outputPath, imageBytes); + } + catch (Exception ex) + { + throw new UserFriendlyException("gemini Base64转图像失败", innerException: ex); + } + + //todo + //路径拼接一下？ + return outputPath; + } + #region 流式传输格式Http响应 private static readonly byte[] EventPrefix = "event: "u8.ToArray();