From 09ecddb55226dbbd9c31508fcac8122fe699bd91 Mon Sep 17 00:00:00 2001 From: ccnetcore Date: Sun, 18 Jan 2026 17:21:07 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E3=80=81=E8=A7=92=E8=89=B2Claim=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E5=8F=8A=E9=94=99=E8=AF=AF=E6=97=A5=E5=BF=97=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 优化 Gemini 图片解析逻辑，递归遍历 JSON 并支持从 markdown 中提取图片 - 修复管理员角色 Claim 使用错误类型的问题，统一为 ClaimTypes.Role - 修正图片生成失败时日志内容，输出完整响应数据以便排查 --- .../Gemini/GeminiGenerateContentAcquirer.cs | 148 +++++++++++++++--- .../Managers/AiGateWayManager.cs | 2 +- .../Managers/AccountManager.cs | 2 +- 3 files changed, 126 insertions(+), 26 deletions(-) diff --git a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs index ca70532a..bcd00370 100644 --- a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs +++ b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain.Shared/Dtos/Gemini/GeminiGenerateContentAcquirer.cs @@ -109,41 +109,107 @@ public static class GeminiGenerateContentAcquirer /// /// 获取图片 base64（包含 data:image 前缀） - /// 从最后一个 part 开始查找 inlineData，找不到再从最后一个 part 开始查找 text + /// Step 1: 递归遍历整个 JSON 查找最后一个 base64 + /// Step 2: 从 text 中查找 markdown 图片 /// public static string GetImagePrefixBase64(JsonElement response) { - var parts = response.GetPath("candidates", 0, "content", "parts"); - if (!parts.HasValue || parts.Value.ValueKind != JsonValueKind.Array) + // Step 1: 递归遍历整个 JSON 查找最后一个 base64 + string? lastBase64 = null; + string? lastMimeType = null; + CollectLastBase64(response, ref lastBase64, ref lastMimeType); + + if (!string.IsNullOrEmpty(lastBase64)) { - return string.Empty; + var mimeType = lastMimeType ?? "image/png"; + return $"data:{mimeType};base64,{lastBase64}"; } - var partsArray = parts.Value.EnumerateArray().ToList(); - if (partsArray.Count == 0) + // Step 2: 从 text 中查找 markdown 图片 + return FindMarkdownImageInResponse(response); + } + + /// + /// 递归遍历 JSON 查找最后一个 base64 + /// + private static void CollectLastBase64(JsonElement element, ref string? lastBase64, ref string? lastMimeType, int minLength = 100) + { + switch (element.ValueKind) { - return string.Empty; + case JsonValueKind.Object: + string? currentMimeType = null; + string? currentData = null; + + foreach (var prop in element.EnumerateObject()) + { + var name = prop.Name.ToLowerInvariant(); + + // 记录 mimeType / mime_type + if (name is "mimetype" or "mime_type" && prop.Value.ValueKind == JsonValueKind.String) + { + currentMimeType = prop.Value.GetString(); + } + // 记录 data 字段（检查是否像 base64） + else if (name == "data" && prop.Value.ValueKind == JsonValueKind.String) + { + var val = prop.Value.GetString(); + if (!string.IsNullOrEmpty(val) && val.Length >= minLength && LooksLikeBase64(val)) + { + currentData = val; + } + } + else + { + // 递归处理其他属性 + CollectLastBase64(prop.Value, ref lastBase64, ref lastMimeType, minLength); + } + } + + // 如果当前对象有 data，更新为"最后一个" + if (currentData != null) + { + lastBase64 = currentData; + lastMimeType = currentMimeType; + } + break; + + case JsonValueKind.Array: + foreach (var item in element.EnumerateArray()) + { + CollectLastBase64(item, ref lastBase64, ref lastMimeType, minLength); + } + break; + } + } + + /// + /// 检查字符串是否像 base64 + /// + private static bool LooksLikeBase64(string str) + { + // 常见图片 base64 开头: JPEG(/9j/), PNG(iVBOR), GIF(R0lGO), WebP(UklGR) + if (str.StartsWith("/9j/") || str.StartsWith("iVBOR") || + str.StartsWith("R0lGO") || str.StartsWith("UklGR")) + { + return true; } - // Step 1: 从最后一个 part 开始查找 inlineData - for (int i = partsArray.Count - 1; i >= 0; i--) - { - var inlineBase64 = partsArray[i].GetPath("inlineData", "data").GetString(); - if (!string.IsNullOrEmpty(inlineBase64)) - { - var mimeType = partsArray[i].GetPath("inlineData", "mimeType").GetString() ?? "image/png"; - return $"data:{mimeType};base64,{inlineBase64}"; - } - } + // 检查前100个字符是否都是 base64 合法字符 + return str.Take(100).All(c => char.IsLetterOrDigit(c) || c == '+' || c == '/' || c == '='); + } - // Step 2: 从最后一个 part 开始查找 text 中的 markdown 图片 - for (int i = partsArray.Count - 1; i >= 0; i--) + /// + /// 递归查找 text 字段中的 markdown 图片 + /// + private static string FindMarkdownImageInResponse(JsonElement element) + { + var allTexts = new List(); + CollectTextFields(element, allTexts); + + // 从最后一个 text 开始查找 + for (int i = allTexts.Count - 1; i >= 0; i--) { - var text = partsArray[i].GetPath("text").GetString(); - if (string.IsNullOrEmpty(text)) - { - continue; - } + var text = allTexts[i]; // markdown 图片格式: ![image](data:image/png;base64,xxx) var startMarker = "(data:image/"; @@ -163,4 +229,38 @@ public static class GeminiGenerateContentAcquirer return string.Empty; } + + /// + /// 递归收集所有 text 字段 + /// + private static void CollectTextFields(JsonElement element, List texts) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + foreach (var prop in element.EnumerateObject()) + { + if (prop.Name == "text" && prop.Value.ValueKind == JsonValueKind.String) + { + var val = prop.Value.GetString(); + if (!string.IsNullOrEmpty(val)) + { + texts.Add(val); + } + } + else + { + CollectTextFields(prop.Value, texts); + } + } + break; + + case JsonValueKind.Array: + foreach (var item in element.EnumerateArray()) + { + CollectTextFields(item, texts); + } + break; + } + } } \ No newline at end of file diff --git a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs index a0d4f355..2a08157e 100644 --- a/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs +++ b/Yi.Abp.Net8/module/ai-hub/Yi.Framework.AiHub.Domain/Managers/AiGateWayManager.cs @@ -1080,7 +1080,7 @@ public class AiGateWayManager : DomainService var imagePrefixBase64 = GeminiGenerateContentAcquirer.GetImagePrefixBase64(data); if (string.IsNullOrWhiteSpace(imagePrefixBase64)) { - _logger.LogError($"图片生成解析失败，模型id:，请求信息：【{request}】，请求响应信息：{imagePrefixBase64}"); + _logger.LogError($"图片生成解析失败，模型id:，请求信息：【{request}】，请求响应信息：【{data}】"); throw new UserFriendlyException("大模型没有返回图片，请调整提示词或稍后再试"); } diff --git a/Yi.Abp.Net8/module/rbac/Yi.Framework.Rbac.Domain/Managers/AccountManager.cs b/Yi.Abp.Net8/module/rbac/Yi.Framework.Rbac.Domain/Managers/AccountManager.cs index a4f226e4..b346e760 100644 --- a/Yi.Abp.Net8/module/rbac/Yi.Framework.Rbac.Domain/Managers/AccountManager.cs +++ b/Yi.Abp.Net8/module/rbac/Yi.Framework.Rbac.Domain/Managers/AccountManager.cs @@ -235,7 +235,7 @@ namespace Yi.Framework.Rbac.Domain.Managers if (UserConst.Admin.Equals(dto.User.UserName)) { AddToClaim(claims, TokenTypeConst.Permission, UserConst.AdminPermissionCode); - AddToClaim(claims, TokenTypeConst.Roles, UserConst.AdminRolesCode); + AddToClaim(claims, ClaimTypes.Role, UserConst.AdminRolesCode); } else {