feat(TwitcastingService, TwitchService): Implement UpdateChannelData

- Added a new method to the `IPlatformService` interface called `GetAllChannels`
- Updated the `HtmlAgilityPack` package version from `1.11.60` to `1.11.61` in `LivestreamRecorderService.csproj`
- Defined the new `GetAllChannels` method in the abstract `PlatformService` class
- Added logic to prepend `https` to a URL if it doesn't start with `http` in the `PlatformService` class
- Removed an unused `OperationInvalidException` and implemented the `UpdateChannelDataAsync` methods in `TwitcastingService` and `TwitchService` classes
- Modified `UpdateChannelInfoWorker` to use the new `GetAllChannels` method and Act on channels with `AutoUpdateInfo == true`. Additionally, it checks whether the Twitch platform is enabled before updating it.

Signed-off-by: 陳鈞 <[email protected]>

Interfaces/IPlatformService.cs

@@ -18,6 +18,7 @@ public interface IPlatformService
 
     Task<YtdlpVideoData?> GetVideoInfoByYtdlpAsync(string url, CancellationToken cancellation = default);
     Task UpdateChannelDataAsync(Channel channel, CancellationToken stoppingToken);
+    Task<List<Channel>> GetAllChannels();
     Task<List<Channel>> GetMonitoringChannels();
 
     /// <summary>

LivestreamRecorderService.csproj

@@ -38,7 +38,7 @@
     <PackageReference Include="CodeHollow.FeedReader" Version="1.2.6" />
     <PackageReference Include="CouchDB.NET.DependencyInjection" Version="3.6.0" />
     <PackageReference Include="Discord.Net.Webhook" Version="3.14.1" />
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.60" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.61" />
     <PackageReference Include="KubernetesClient" Version="13.0.26" />
     <PackageReference Include="Microsoft.Extensions.Azure" Version="1.7.3" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.20.1" />

ScopedServices/PlatformService/PlatformService.cs

@@ -48,8 +48,11 @@ protected PlatformService(
             DiscordService = serviceProvider.GetRequiredService<DiscordService>();
     }
 
+    public Task<List<Channel>> GetAllChannels()
+        => ChannelRepository.GetChannelsBySourceAsync(PlatformName);
+
     public async Task<List<Channel>> GetMonitoringChannels()
-        => (await ChannelRepository.GetChannelsBySourceAsync(PlatformName))
+        => (await GetAllChannels())
            .Where(p => p.Monitoring)
            .ToList();
 
@@ -177,6 +180,11 @@ public bool StepInterval(int elapsedTime)
             throw new ArgumentNullException(nameof(url));
         }
 
+        if (!url.StartsWith("http"))
+        {
+            url = "https:" + url;
+        }
+
         if (string.IsNullOrEmpty(path))
         {
             throw new ArgumentNullException(nameof(path));

ScopedServices/PlatformService/TwitcastingService.cs

@@ -400,6 +400,73 @@ await DownloadThumbnailAsync(
         _unitOfWorkPublic.Commit();
     }
 
-    public override Task UpdateChannelDataAsync(Channel channel, CancellationToken stoppingToken)
-        => throw new InvalidOperationException();
+    public override async Task UpdateChannelDataAsync(Channel channel, CancellationToken stoppingToken)
+    {
+        var htmlDoc = new HtmlWeb().Load($"https://twitcasting.tv/{NameHelper.ChangeId.ChannelId.PlatformType(channel.id, PlatformName)}");
+        if (null == htmlDoc)
+        {
+            logger.LogWarning("Failed to get channel page for {channelId}", channel.id);
+            return;
+        }
+
+        var avatarBlobUrl = await getAvatarBlobUrl() ?? channel.Avatar;
+        var bannerBlobUrl = await getBannerBlobUrl() ?? channel.Banner;
+        var channelName = getChannelName() ?? channel.ChannelName;
+
+        channel = await ChannelRepository.ReloadEntityFromDBAsync(channel) ?? channel;
+        channel.ChannelName = channelName;
+        channel.Avatar = avatarBlobUrl?.Replace("avatar/", "");
+        channel.Banner = bannerBlobUrl?.Replace("banner/", "");
+        await ChannelRepository.AddOrUpdateAsync(channel);
+        _unitOfWorkPublic.Commit();
+        return;
+
+        async Task<string?> getAvatarBlobUrl()
+        {
+            var avatarImgNode = htmlDoc.DocumentNode.SelectSingleNode("//a[@class='tw-user-nav-icon']/img");
+            var avatarUrl = avatarImgNode?.Attributes["src"]?.Value
+                                         .Replace("_bigger", "");
+
+            if (string.IsNullOrEmpty(avatarUrl)) return null;
+
+            avatarBlobUrl = await DownloadImageAndUploadToBlobStorageAsync(avatarUrl, $"avatar/{channel.id}", stoppingToken);
+
+            return avatarBlobUrl;
+        }
+
+        async Task<string?> getBannerBlobUrl()
+        {
+            var bannerNode = htmlDoc.DocumentNode.SelectSingleNode("//div[@class='tw-user-banner-image']");
+            var bannerUrl = extractBackgroundImageUrl(bannerNode?.GetAttributeValue("style", "") ?? "");
+            if (string.IsNullOrEmpty(bannerUrl)) return null;
+
+            bannerBlobUrl = await DownloadImageAndUploadToBlobStorageAsync(bannerUrl, $"banner/{channel.id}", stoppingToken);
+
+            return bannerBlobUrl;
+        }
+
+        static string? extractBackgroundImageUrl(string style)
+        {
+            if (string.IsNullOrEmpty(style)) return null;
+
+            const string searchString = "background-image: url(";
+            var startIndex = style.IndexOf(searchString, StringComparison.Ordinal);
+            if (startIndex == -1) return null;
+
+            startIndex += searchString.Length;
+            var endIndex = style.IndexOf(')', startIndex);
+            if (endIndex == -1) return null;
+
+            var url = style[startIndex..endIndex].Trim('\'', '\"');
+            if (url.StartsWith("//"))
+            {
+                url = "https:" + url;
+            }
+
+            return url;
+        }
+
+        string? getChannelName()
+            => htmlDoc.DocumentNode.SelectSingleNode("//span[@class='tw-user-nav-name']").InnerText;
+    }
 }

ScopedServices/PlatformService/TwitchService.cs

@@ -175,6 +175,51 @@ public override async Task UpdateVideoDataAsync(Video video, CancellationToken c
         _unitOfWorkPublic.Commit();
     }
 
-    public override Task UpdateChannelDataAsync(Channel channel, CancellationToken stoppingToken)
-        => throw new InvalidOperationException();
+    public override async Task UpdateChannelDataAsync(Channel channel, CancellationToken stoppingToken)
+    {
+        var channelId = channel.id;
+        var usersResponse = await twitchApi.Helix.Users.GetUsersAsync(logins: [NameHelper.ChangeId.ChannelId.PlatformType(channelId, PlatformName)]);
+        if (null == usersResponse || usersResponse.Users.Length == 0)
+        {
+            logger.LogWarning("Failed to get channel info for {channelId}", channelId);
+            return;
+        }
+
+        var user = usersResponse.Users.First();
+
+        var avatarBlobUrl = await getAvatarBlobUrl() ?? channel.Avatar;
+        var bannerBlobUrl = await getBannerBlobUrl() ?? channel.Banner;
+        var channelName = getChannelName() ?? channel.ChannelName;
+
+        channel = await ChannelRepository.ReloadEntityFromDBAsync(channel) ?? channel;
+        channel.ChannelName = channelName;
+        channel.Avatar = avatarBlobUrl?.Replace("avatar/", "");
+        channel.Banner = bannerBlobUrl?.Replace("banner/", "");
+        await ChannelRepository.AddOrUpdateAsync(channel);
+        _unitOfWorkPublic.Commit();
+        return;
+
+        async Task<string?> getAvatarBlobUrl()
+        {
+            var avatarUrl = user.ProfileImageUrl.Replace("70x70", "300x300");
+            if (string.IsNullOrEmpty(avatarUrl)) return null;
+
+            avatarBlobUrl = await DownloadImageAndUploadToBlobStorageAsync(avatarUrl, $"avatar/{channelId}", stoppingToken);
+
+            return avatarBlobUrl;
+        }
+
+        async Task<string?> getBannerBlobUrl()
+        {
+            var bannerUrl = user.OfflineImageUrl;
+            if (string.IsNullOrEmpty(bannerUrl)) return null;
+
+            bannerBlobUrl = await DownloadImageAndUploadToBlobStorageAsync(bannerUrl, $"banner/{channelId}", stoppingToken);
+
+            return bannerBlobUrl;
+        }
+
+        string? getChannelName()
+            => user.DisplayName;
+    }
 }

Workers/UpdateChannelInfoWorker.cs

@@ -1,17 +1,20 @@
 using LivestreamRecorder.DB.Models;
 using LivestreamRecorderService.Interfaces;
+using LivestreamRecorderService.Models.Options;
 using LivestreamRecorderService.ScopedServices.PlatformService;
+using Microsoft.Extensions.Options;
 using Serilog.Context;
 
 namespace LivestreamRecorderService.Workers;
 
 public class UpdateChannelInfoWorker(
     ILogger<UpdateChannelInfoWorker> logger,
-    IServiceProvider serviceProvider) : BackgroundService
+    IServiceProvider serviceProvider,
+    IOptions<TwitchOption> twitchOptions) : BackgroundService
 {
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        using IDisposable _ = LogContext.PushProperty("Worker", nameof(UpdateChannelInfoWorker));
+        using var _ = LogContext.PushProperty("Worker", nameof(UpdateChannelInfoWorker));
 
 #if RELEASE
         logger.LogInformation("{Worker} will sleep 60 seconds avoid being overloaded with {WorkerToWait}.", nameof(RecordWorker), nameof(MonitorWorker));
@@ -22,32 +25,41 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
         while (!stoppingToken.IsCancellationRequested)
         {
-            using IDisposable __ = LogContext.PushProperty("WorkerRunId", $"{nameof(UpdateChannelInfoWorker)}_{DateTime.UtcNow:yyyyMMddHHmmssfff}");
+            using var __ = LogContext.PushProperty("WorkerRunId", $"{nameof(UpdateChannelInfoWorker)}_{DateTime.UtcNow:yyyyMMddHHmmssfff}");
 
             #region DI
 
-            using IServiceScope scope = serviceProvider.CreateScope();
-            YoutubeService youtubeSerivce = scope.ServiceProvider.GetRequiredService<YoutubeService>();
-            Fc2Service fC2Service = scope.ServiceProvider.GetRequiredService<Fc2Service>();
+            using var scope = serviceProvider.CreateScope();
+            var youtubeService = scope.ServiceProvider.GetRequiredService<YoutubeService>();
+            var twitcastingService = scope.ServiceProvider.GetRequiredService<TwitcastingService>();
+            var fC2Service = scope.ServiceProvider.GetRequiredService<Fc2Service>();
 
             #endregion
 
-            await UpdatePlatformAsync(youtubeSerivce, stoppingToken);
+            await UpdatePlatformAsync(youtubeService, stoppingToken);
+            await UpdatePlatformAsync(twitcastingService, stoppingToken);
             await UpdatePlatformAsync(fC2Service, stoppingToken);
 
+            if (twitchOptions.Value.Enabled)
+            {
+                var twitchService = scope.ServiceProvider.GetRequiredService<TwitchService>();
+                await UpdatePlatformAsync(twitchService, stoppingToken);
+            }
+
             logger.LogTrace("{Worker} ends. Sleep 1 day.", nameof(UpdateChannelInfoWorker));
             await Task.Delay(TimeSpan.FromDays(1), stoppingToken);
         }
     }
 
     private async Task UpdatePlatformAsync(IPlatformService platformService, CancellationToken stoppingToken = default)
     {
-        List<Channel> channels = await platformService.GetMonitoringChannels();
+        var channels = (await platformService.GetAllChannels())
+                       .Where(p => p.AutoUpdateInfo == true)
+                       .ToList();
+
         logger.LogDebug("Get {channelCount} channels for {platform}", channels.Count, platformService.PlatformName);
-        foreach (Channel? channel in channels)
+        foreach (var channel in channels)
         {
-            if (channel.AutoUpdateInfo != true) continue;
-
             await platformService.UpdateChannelDataAsync(channel, stoppingToken);
         }
     }