feat: 完成yi.tool.web爬虫

This commit is contained in:
橙子
2024-06-09 00:29:53 +08:00
parent f44a099469
commit f267d820bf
11 changed files with 144 additions and 10 deletions

View File

@@ -0,0 +1,25 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Volo.Abp.Application.Services;
using Yi.Abp.Tool.Domain;
namespace Yi.Abp.Tool.Application
{
public class NueGetInfoService : ApplicationService
{
private NugetCrawlerManager _nugetCrawlerManager;
public NueGetInfoService(NugetCrawlerManager nugetCrawlerManager) { _nugetCrawlerManager = nugetCrawlerManager; }
/// <summary>
/// 获取爬虫结果
/// </summary>
/// <returns></returns>
public NugetResult GetInfo()
{
return _nugetCrawlerManager.GetNugetResult();
}
}
}

View File

@@ -0,0 +1,85 @@
using HtmlAgilityPack;
using Microsoft.Extensions.Caching.Distributed;
using Volo.Abp.Caching;
using Volo.Abp.DependencyInjection;
namespace Yi.Abp.Tool.Domain
{
public class NugetCrawlerManager : ITransientDependency
{
private const string NugetVersionUrl = "https://www.nuget.org/packages/Yi.Abp.Tool#versions-body-tab";
public NugetCrawlerManager(IDistributedCache<NugetResult> cache)
{
//缓存设置1分钟获取一次结果
this.NugetResult = cache.GetOrAdd("NugetResult", () =>
{
return InitData();
}, () =>
{
var options = new DistributedCacheEntryOptions();
options.AbsoluteExpiration = DateTimeOffset.Now.AddMinutes(1);
return options;
})!;
}
private HtmlDocument HtmlDoc { get; set; }
private NugetResult NugetResult { get; set; } = new NugetResult();
private NugetResult InitData()
{
NugetResult nugetResult = new NugetResult();
HtmlWeb web = new HtmlWeb();
this.HtmlDoc = web.Load(NugetVersionUrl);
nugetResult.Versions = GetVersionList();
nugetResult.DownloadNumberNumber = GetDownloadNumber();
return nugetResult;
}
public NugetResult GetNugetResult()
{
return this.NugetResult;
}
/// <summary>
/// 获取版本号列表
/// </summary>
/// <returns></returns>
private List<string> GetVersionList()
{
List<string> versions = new List<string>();
var versionDoc = HtmlDoc.DocumentNode.SelectNodes("//*[@id=\"version-history\"]/table/tbody");
var trDoc = versionDoc.First().ChildNodes.Where(x => x.Name == "tr").ToList();
foreach (var tr in trDoc)
{
var version = tr.ChildNodes.Where(x => x.Name == "td").First().ChildNodes.Where(x => x.Name == "a").First().GetAttributes("title").First().Value;
versions.Add(version);
}
return versions;
}
/// <summary>
/// 获取下载总数
/// </summary>
/// <returns></returns>
private long GetDownloadNumber()
{
var spanDoc = HtmlDoc.DocumentNode.SelectNodes("//*[@id=\"skippedToContent\"]/section/div/aside/div[1]/div[2]/div[1]/span[2]");
var downLoadNumber = spanDoc.First().InnerText;
return long.Parse(downLoadNumber);
}
}
public class NugetResult
{
public long DownloadNumberNumber { get; set; }
public List<string> Versions { get; set; }
}
}

View File

@@ -5,8 +5,13 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.61" />
<PackageReference Include="Volo.Abp.Caching" Version="$(AbpVersion)" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Yi.Abp.Tool.Domain.Shared\Yi.Abp.Tool.Domain.Shared.csproj" />
</ItemGroup>
</Project>

View File

@@ -1,9 +1,11 @@
using Yi.Abp.Tool.Web;
var builder = WebApplication.CreateBuilder(args);
builder.WebHost.UseUrls(builder.Configuration["App:SelfUrl"]);
builder.Host.UseAutofac();
await builder.Services.AddApplicationAsync<YiAbpToolWebModule>();
var app = builder.Build();
await app.InitializeApplicationAsync();
await app.RunAsync();

View File

@@ -13,7 +13,13 @@ namespace Yi.Abp.Tool.Commands
public Task InvokerAsync(Dictionary<string, string> options, string[] args)
{
List<string> delDirBlacklist = ["obj", "bin"];
DeleteObjBinFolders("./", delDirBlacklist);
options.TryGetValue("path", out var path);
if (string.IsNullOrEmpty(path))
{
path = "./";
}
DeleteObjBinFolders(path, delDirBlacklist);
return Task.CompletedTask;
}
@@ -24,7 +30,7 @@ namespace Yi.Abp.Tool.Commands
{
foreach (string subDir in Directory.GetDirectories(directory))
{
if (delDirBlacklist.Contains(subDir))
if (delDirBlacklist.Contains(Path.GetFileName( subDir)))
{
Directory.Delete(subDir, true);
Console.WriteLine($"已删除文件夹:{subDir}");

View File

@@ -31,6 +31,7 @@ namespace Yi.Abp.Tool.Commands
> new: 创建模块模板` yi-abp new <name> -t module -csf `
> new: 创建项目模板` yi-abp new <name> -csf `
> add-module: 将内容添加到当前解决方案` yi-abp add-module <moduleName> [-modulePath <path>] [-s <slnPath>] `
> clear: 清除当前目录及子目录下的obj、bin文件夹` yi-abp clear `
""");
return Task.CompletedTask;

View File

@@ -16,7 +16,8 @@ class Program
//args = ["12312"];
//args = ["new", "Acme.Book", "-t", "module", "-csf"];
//args = ["new", "Acme.Book", "-t", "module"];
args = ["add-module", "Acme.Demo", "-s", "D:\\code\\csharp\\source\\Yi\\Yi.Abp.Net8", "-modulePath", "D:\\code\\csharp\\source\\Yi\\Yi.Abp.Net8\\module\\acme-demo"];
//args = ["add-module", "Acme.Demo", "-s", "D:\\code\\csharp\\source\\Yi\\Yi.Abp.Net8", "-modulePath", "D:\\code\\csharp\\source\\Yi\\Yi.Abp.Net8\\module\\acme-demo"];
args = ["clear", "-path", "D:\\code\\csharp\\source\\Yi\\Yi.Abp.Net8\\src"];
#endif
try
{

View File

@@ -5,7 +5,7 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Version>0.8.0</Version>
<Version>1.0.0</Version>
<Authors>橙子老哥</Authors>
<Description>yi-framework框架配套工具</Description>
<PackageProjectUrl>https://ccnetcore.com</PackageProjectUrl>