fix: 修复图片解析、角色Claim类型及错误日志问题
- 优化 Gemini 图片解析逻辑,递归遍历 JSON 并支持从 markdown 中提取图片 - 修复管理员角色 Claim 使用错误类型的问题,统一为 ClaimTypes.Role - 修正图片生成失败时日志内容,输出完整响应数据以便排查
This commit is contained in:
@@ -109,41 +109,107 @@ public static class GeminiGenerateContentAcquirer
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 获取图片 base64(包含 data:image 前缀)
|
/// 获取图片 base64(包含 data:image 前缀)
|
||||||
/// 从最后一个 part 开始查找 inlineData,找不到再从最后一个 part 开始查找 text
|
/// Step 1: 递归遍历整个 JSON 查找最后一个 base64
|
||||||
|
/// Step 2: 从 text 中查找 markdown 图片
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static string GetImagePrefixBase64(JsonElement response)
|
public static string GetImagePrefixBase64(JsonElement response)
|
||||||
{
|
{
|
||||||
var parts = response.GetPath("candidates", 0, "content", "parts");
|
// Step 1: 递归遍历整个 JSON 查找最后一个 base64
|
||||||
if (!parts.HasValue || parts.Value.ValueKind != JsonValueKind.Array)
|
string? lastBase64 = null;
|
||||||
|
string? lastMimeType = null;
|
||||||
|
CollectLastBase64(response, ref lastBase64, ref lastMimeType);
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(lastBase64))
|
||||||
{
|
{
|
||||||
return string.Empty;
|
var mimeType = lastMimeType ?? "image/png";
|
||||||
|
return $"data:{mimeType};base64,{lastBase64}";
|
||||||
}
|
}
|
||||||
|
|
||||||
var partsArray = parts.Value.EnumerateArray().ToList();
|
// Step 2: 从 text 中查找 markdown 图片
|
||||||
if (partsArray.Count == 0)
|
return FindMarkdownImageInResponse(response);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 递归遍历 JSON 查找最后一个 base64
|
||||||
|
/// </summary>
|
||||||
|
private static void CollectLastBase64(JsonElement element, ref string? lastBase64, ref string? lastMimeType, int minLength = 100)
|
||||||
|
{
|
||||||
|
switch (element.ValueKind)
|
||||||
{
|
{
|
||||||
return string.Empty;
|
case JsonValueKind.Object:
|
||||||
|
string? currentMimeType = null;
|
||||||
|
string? currentData = null;
|
||||||
|
|
||||||
|
foreach (var prop in element.EnumerateObject())
|
||||||
|
{
|
||||||
|
var name = prop.Name.ToLowerInvariant();
|
||||||
|
|
||||||
|
// 记录 mimeType / mime_type
|
||||||
|
if (name is "mimetype" or "mime_type" && prop.Value.ValueKind == JsonValueKind.String)
|
||||||
|
{
|
||||||
|
currentMimeType = prop.Value.GetString();
|
||||||
|
}
|
||||||
|
// 记录 data 字段(检查是否像 base64)
|
||||||
|
else if (name == "data" && prop.Value.ValueKind == JsonValueKind.String)
|
||||||
|
{
|
||||||
|
var val = prop.Value.GetString();
|
||||||
|
if (!string.IsNullOrEmpty(val) && val.Length >= minLength && LooksLikeBase64(val))
|
||||||
|
{
|
||||||
|
currentData = val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// 递归处理其他属性
|
||||||
|
CollectLastBase64(prop.Value, ref lastBase64, ref lastMimeType, minLength);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果当前对象有 data,更新为"最后一个"
|
||||||
|
if (currentData != null)
|
||||||
|
{
|
||||||
|
lastBase64 = currentData;
|
||||||
|
lastMimeType = currentMimeType;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case JsonValueKind.Array:
|
||||||
|
foreach (var item in element.EnumerateArray())
|
||||||
|
{
|
||||||
|
CollectLastBase64(item, ref lastBase64, ref lastMimeType, minLength);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 检查字符串是否像 base64
|
||||||
|
/// </summary>
|
||||||
|
private static bool LooksLikeBase64(string str)
|
||||||
|
{
|
||||||
|
// 常见图片 base64 开头: JPEG(/9j/), PNG(iVBOR), GIF(R0lGO), WebP(UklGR)
|
||||||
|
if (str.StartsWith("/9j/") || str.StartsWith("iVBOR") ||
|
||||||
|
str.StartsWith("R0lGO") || str.StartsWith("UklGR"))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 1: 从最后一个 part 开始查找 inlineData
|
// 检查前100个字符是否都是 base64 合法字符
|
||||||
for (int i = partsArray.Count - 1; i >= 0; i--)
|
return str.Take(100).All(c => char.IsLetterOrDigit(c) || c == '+' || c == '/' || c == '=');
|
||||||
{
|
}
|
||||||
var inlineBase64 = partsArray[i].GetPath("inlineData", "data").GetString();
|
|
||||||
if (!string.IsNullOrEmpty(inlineBase64))
|
|
||||||
{
|
|
||||||
var mimeType = partsArray[i].GetPath("inlineData", "mimeType").GetString() ?? "image/png";
|
|
||||||
return $"data:{mimeType};base64,{inlineBase64}";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Step 2: 从最后一个 part 开始查找 text 中的 markdown 图片
|
/// <summary>
|
||||||
for (int i = partsArray.Count - 1; i >= 0; i--)
|
/// 递归查找 text 字段中的 markdown 图片
|
||||||
|
/// </summary>
|
||||||
|
private static string FindMarkdownImageInResponse(JsonElement element)
|
||||||
|
{
|
||||||
|
var allTexts = new List<string>();
|
||||||
|
CollectTextFields(element, allTexts);
|
||||||
|
|
||||||
|
// 从最后一个 text 开始查找
|
||||||
|
for (int i = allTexts.Count - 1; i >= 0; i--)
|
||||||
{
|
{
|
||||||
var text = partsArray[i].GetPath("text").GetString();
|
var text = allTexts[i];
|
||||||
if (string.IsNullOrEmpty(text))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// markdown 图片格式: 
|
// markdown 图片格式: 
|
||||||
var startMarker = "(data:image/";
|
var startMarker = "(data:image/";
|
||||||
@@ -163,4 +229,38 @@ public static class GeminiGenerateContentAcquirer
|
|||||||
|
|
||||||
return string.Empty;
|
return string.Empty;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// 递归收集所有 text 字段
|
||||||
|
/// </summary>
|
||||||
|
private static void CollectTextFields(JsonElement element, List<string> texts)
|
||||||
|
{
|
||||||
|
switch (element.ValueKind)
|
||||||
|
{
|
||||||
|
case JsonValueKind.Object:
|
||||||
|
foreach (var prop in element.EnumerateObject())
|
||||||
|
{
|
||||||
|
if (prop.Name == "text" && prop.Value.ValueKind == JsonValueKind.String)
|
||||||
|
{
|
||||||
|
var val = prop.Value.GetString();
|
||||||
|
if (!string.IsNullOrEmpty(val))
|
||||||
|
{
|
||||||
|
texts.Add(val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
CollectTextFields(prop.Value, texts);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case JsonValueKind.Array:
|
||||||
|
foreach (var item in element.EnumerateArray())
|
||||||
|
{
|
||||||
|
CollectTextFields(item, texts);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1080,7 +1080,7 @@ public class AiGateWayManager : DomainService
|
|||||||
var imagePrefixBase64 = GeminiGenerateContentAcquirer.GetImagePrefixBase64(data);
|
var imagePrefixBase64 = GeminiGenerateContentAcquirer.GetImagePrefixBase64(data);
|
||||||
if (string.IsNullOrWhiteSpace(imagePrefixBase64))
|
if (string.IsNullOrWhiteSpace(imagePrefixBase64))
|
||||||
{
|
{
|
||||||
_logger.LogError($"图片生成解析失败,模型id:,请求信息:【{request}】,请求响应信息:{imagePrefixBase64}");
|
_logger.LogError($"图片生成解析失败,模型id:,请求信息:【{request}】,请求响应信息:【{data}】");
|
||||||
throw new UserFriendlyException("大模型没有返回图片,请调整提示词或稍后再试");
|
throw new UserFriendlyException("大模型没有返回图片,请调整提示词或稍后再试");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -235,7 +235,7 @@ namespace Yi.Framework.Rbac.Domain.Managers
|
|||||||
if (UserConst.Admin.Equals(dto.User.UserName))
|
if (UserConst.Admin.Equals(dto.User.UserName))
|
||||||
{
|
{
|
||||||
AddToClaim(claims, TokenTypeConst.Permission, UserConst.AdminPermissionCode);
|
AddToClaim(claims, TokenTypeConst.Permission, UserConst.AdminPermissionCode);
|
||||||
AddToClaim(claims, TokenTypeConst.Roles, UserConst.AdminRolesCode);
|
AddToClaim(claims, ClaimTypes.Role, UserConst.AdminRolesCode);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user