From dd9f877d6700a7c4479edaa93ba48f26f7e53d37 Mon Sep 17 00:00:00 2001 From: James Wilkinson Date: Sat, 11 Apr 2020 21:35:11 +0100 Subject: [PATCH 1/5] #532 Change to return all meetups in the next 30 days Also added try-catch to handle Invalid responses --- OurUmbraco/Community/Meetup/MeetupService.cs | 115 ++++++++++--------- 1 file changed, 62 insertions(+), 53 deletions(-) diff --git a/OurUmbraco/Community/Meetup/MeetupService.cs b/OurUmbraco/Community/Meetup/MeetupService.cs index 48286c6f9a..212a1533e5 100644 --- a/OurUmbraco/Community/Meetup/MeetupService.cs +++ b/OurUmbraco/Community/Meetup/MeetupService.cs @@ -25,60 +25,67 @@ public class MeetupService { public void UpdateMeetupStats() { - var configPath = HostingEnvironment.MapPath("~/config/MeetupUmbracoGroups.txt"); - // Get the alias (urlname) of each group from the config file - var aliases = File.ReadAllLines(configPath).Where(x => x.Trim() != "").Distinct().ToArray(); - - var counterPath = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCounter.txt"); - var counter = 0; - if (File.Exists(counterPath)) + try { - var savedCounter = File.ReadAllLines(counterPath).First(); - int.TryParse(savedCounter, out counter); - } + var configPath = HostingEnvironment.MapPath("~/config/MeetupUmbracoGroups.txt"); + // Get the alias (urlname) of each group from the config file + var aliases = File.ReadAllLines(configPath).Where(x => x.Trim() != "").Distinct().ToArray(); - var newCounter = aliases.Length <= counter ? 0 : counter + 1; - File.WriteAllText(counterPath, newCounter.ToString(), Encoding.UTF8); + var counterPath = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCounter.txt"); + var counter = 0; + if (File.Exists(counterPath)) + { + var savedCounter = File.ReadAllLines(counterPath).First(); + int.TryParse(savedCounter, out counter); + } - var client = new MeetupOAuth2Client(); - var response = client.DoHttpGetRequest(string.Format("https://api.meetup.com/{0}/events?page=1000&status=past", aliases[counter])); - var events = MeetupGetEventsResponse.ParseResponse(response).Body; + var newCounter = aliases.Length <= counter ? 0 : counter + 1; + File.WriteAllText(counterPath, newCounter.ToString(), Encoding.UTF8); - var meetupCache = new List(); - var meetupCacheFile = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCache.json"); - if (File.Exists(meetupCacheFile)) - { - var json = File.ReadAllText(meetupCacheFile); - using (var stringReader = new StringReader(json)) - using (var jsonTextReader = new JsonTextReader(stringReader)) + var client = new MeetupOAuth2Client(); + var response = client.DoHttpGetRequest(string.Format("https://api.meetup.com/{0}/events?page=1000&status=past", aliases[counter])); + var events = MeetupGetEventsResponse.ParseResponse(response).Body; + + var meetupCache = new List(); + var meetupCacheFile = HostingEnvironment.MapPath("~/App_Data/TEMP/MeetupStatisticsCache.json"); + if (File.Exists(meetupCacheFile)) { - var jsonSerializer = new JsonSerializer(); - meetupCache = jsonSerializer.Deserialize>(jsonTextReader); + var json = File.ReadAllText(meetupCacheFile); + using (var stringReader = new StringReader(json)) + using (var jsonTextReader = new JsonTextReader(stringReader)) + { + var jsonSerializer = new JsonSerializer(); + meetupCache = jsonSerializer.Deserialize>(jsonTextReader); + } } - } - - foreach (var meetupEvent in events) - { - if (meetupCache.Any(x => x.Id == meetupEvent.Id)) - continue; - var meetupCacheItem = new MeetupCacheItem + foreach (var meetupEvent in events) { - Time = meetupEvent.Time, - Created = meetupEvent.Created, - Description = meetupEvent.Description, - HasVenue = meetupEvent.HasVenue, - Id = meetupEvent.Id, - Link = meetupEvent.Link, - Name = meetupEvent.Name, - Updated = meetupEvent.Updated, - Visibility = meetupEvent.Visibility - }; - meetupCache.Add(meetupCacheItem); - } + if (meetupCache.Any(x => x.Id == meetupEvent.Id)) + continue; - var rawJson = JsonConvert.SerializeObject(meetupCache, Formatting.Indented); - File.WriteAllText(meetupCacheFile, rawJson, Encoding.UTF8); + var meetupCacheItem = new MeetupCacheItem + { + Time = meetupEvent.Time, + Created = meetupEvent.Created, + Description = meetupEvent.Description, + HasVenue = meetupEvent.HasVenue, + Id = meetupEvent.Id, + Link = meetupEvent.Link, + Name = meetupEvent.Name, + Updated = meetupEvent.Updated, + Visibility = meetupEvent.Visibility + }; + meetupCache.Add(meetupCacheItem); + } + + var rawJson = JsonConvert.SerializeObject(meetupCache, Formatting.Indented); + File.WriteAllText(meetupCacheFile, rawJson, Encoding.UTF8); + } + catch (Exception ex) + { + LogHelper.Error("Could not get events from meetup.com", ex); + } } @@ -95,6 +102,7 @@ public MeetupEventsModel GetUpcomingMeetups() if (File.Exists(configPath) == false) { LogHelper.Debug("Config file was not found: " + configPath); + return meetups; } @@ -106,7 +114,6 @@ public MeetupEventsModel GetUpcomingMeetups() { // Initialize a new service instance (we don't specify an API key since we're accessing public data) var service = new Skybrud.Social.Meetup.MeetupService(); - var items = new List(); foreach (var alias in aliases) @@ -119,17 +126,19 @@ public MeetupEventsModel GetUpcomingMeetups() if (meetupGroup.JObject.HasValue("next_event") == false) continue; - var nextEventId = meetupGroup.JObject.GetString("next_event.id"); - // Make the call to the Meetup.com API to get upcoming events var events = service.Events.GetEvents(alias); - // Get the next event(s) - var nextEvent = events.Body.FirstOrDefault(x => x.Id == nextEventId); + // Get the events in the next 30 days + var nextEvents = events.Body.Where(x => x.Time < DateTime.Now.AddDays(30) ); - // Append the first event of the group - if (nextEvent != null) - items.Add(new MeetupItem(meetupGroup, nextEvent)); + if (nextEvents.Any()) + { + foreach (var item in nextEvents) + { + items.Add(new MeetupItem(meetupGroup, item)); + } + } } catch (Exception ex) { From 06232b93c50ff39065af0733d080c521f4c3c367 Mon Sep 17 00:00:00 2001 From: James Wilkinson Date: Sat, 11 Apr 2020 21:36:33 +0100 Subject: [PATCH 2/5] Added try-catch to handle Invalid responses to aid debugging --- OurUmbraco/Videos/VideosService.cs | 98 ++++++++++++++++-------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/OurUmbraco/Videos/VideosService.cs b/OurUmbraco/Videos/VideosService.cs index cf289f05fd..60ad3ffdbc 100644 --- a/OurUmbraco/Videos/VideosService.cs +++ b/OurUmbraco/Videos/VideosService.cs @@ -9,6 +9,7 @@ using Skybrud.Social.Vimeo; using Skybrud.Social.Vimeo.Models.Videos; using Umbraco.Core.IO; +using Umbraco.Core.Logging; namespace OurUmbraco.Videos { @@ -18,68 +19,75 @@ public class VideosService public void UpdateVimeoVideos(string username) { - if (ConfigurationManager.AppSettings["VimeoAccessToken"] == "it's a secret.. and no, this is not the secret, this key will transformed on the build server") + try { - return; - } + if (ConfigurationManager.AppSettings["VimeoAccessToken"] == "it's a secret.. and no, this is not the secret, this key will transformed on the build server") + { + return; + } - if (string.IsNullOrWhiteSpace(username)) throw new ArgumentNullException("username"); + if (string.IsNullOrWhiteSpace(username)) throw new ArgumentNullException("username"); - // Map the path to the directory - var savePath = IOHelper.MapPath(SaveDirectory); - if (Directory.Exists(savePath) == false) - Directory.CreateDirectory(savePath); + // Map the path to the directory + var savePath = IOHelper.MapPath(SaveDirectory); + if (Directory.Exists(savePath) == false) + Directory.CreateDirectory(savePath); - // Map the path to the JSON file - var path = savePath + "VimeoVideos_" + username + ".json"; + // Map the path to the JSON file + var path = savePath + "VimeoVideos_" + username + ".json"; - // Initialize a new service from an OAuth 2.0 access token - var vimeo = VimeoService.CreateFromAccessToken(ConfigurationManager.AppSettings["VimeoAccessToken"]); + // Initialize a new service from an OAuth 2.0 access token + var vimeo = VimeoService.CreateFromAccessToken(ConfigurationManager.AppSettings["VimeoAccessToken"]); - var page = 1; - const int maxPages = 10; + var page = 1; + const int maxPages = 10; - // Initialize a list for all the videos - var videos = new List(); + // Initialize a list for all the videos + var videos = new List(); - // Create a loop for the pages - while (page <= maxPages) - { - // Make the request to the Vimeo API - var response = vimeo.Videos.GetVideos(username, page, 100); + // Create a loop for the pages + while (page <= maxPages) + { + // Make the request to the Vimeo API + var response = vimeo.Videos.GetVideos(username, page, 100); - // Append the videos of the response to the list - videos.AddRange(response.Body.Data); + // Append the videos of the response to the list + videos.AddRange(response.Body.Data); - // Download the thumnails for each video - foreach (var video in videos) - { - var thumbnail = video.Pictures.Sizes.FirstOrDefault(x => x.Width >= 350); + // Download the thumnails for each video + foreach (var video in videos) + { + var thumbnail = video.Pictures.Sizes.FirstOrDefault(x => x.Width >= 350); - const string mediaRoot = "~/media/Vimeo"; - var thumbnailFile = IOHelper.MapPath($"{mediaRoot}/{video.Id}.jpg"); + const string mediaRoot = "~/media/Vimeo"; + var thumbnailFile = IOHelper.MapPath($"{mediaRoot}/{video.Id}.jpg"); - var mediaPath = IOHelper.MapPath(mediaRoot); - if (Directory.Exists(mediaPath) == false) - Directory.CreateDirectory(mediaPath); + var mediaPath = IOHelper.MapPath(mediaRoot); + if (Directory.Exists(mediaPath) == false) + Directory.CreateDirectory(mediaPath); - if (File.Exists(thumbnailFile)) - continue; + if (File.Exists(thumbnailFile)) + continue; - using (var client = new WebClient()) - client.DownloadFile(thumbnail.Link, thumbnailFile); - } + using (var client = new WebClient()) + client.DownloadFile(thumbnail.Link, thumbnailFile); + } - // Break the loop if there are no further pages - if (response.Body.Paging.Next == null) - break; + // Break the loop if there are no further pages + if (response.Body.Paging.Next == null) + break; - // Increment the page count - page++; - } + // Increment the page count + page++; + } - // Save the videos as a JSON file - JsonUtils.SaveJsonArray(path, videos); + // Save the videos as a JSON file + JsonUtils.SaveJsonArray(path, videos); + } + catch (Exception ex) + { + LogHelper.Error("Could not get data from Vimeo", ex); + } } public VimeoVideo[] GetVimeoVideosFromDisk(string username) From 0ea9cdfe6f6ce298fd76995e3fc49ee7af9e9ee7 Mon Sep 17 00:00:00 2001 From: James Wilkinson Date: Sat, 11 Apr 2020 21:37:04 +0100 Subject: [PATCH 3/5] Added try-catch to handle Invalid responses to aid debugging --- .../Videos/CommunityVideosService.cs | 148 +++++++++--------- 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/OurUmbraco/Community/Videos/CommunityVideosService.cs b/OurUmbraco/Community/Videos/CommunityVideosService.cs index 33ecdf639e..615e003d9d 100644 --- a/OurUmbraco/Community/Videos/CommunityVideosService.cs +++ b/OurUmbraco/Community/Videos/CommunityVideosService.cs @@ -96,103 +96,111 @@ public void UpdateYouTubePlaylistVideos() public void UpdateYouTubePlaylistVideo(string playlistId) { - - // Make sure we have a TEMP directory - string dir = IOHelper.MapPath("~/App_Data/TEMP/YouTube/"); - Directory.CreateDirectory(dir); - - // Make an initial request to get information about the playlist - var response1 = Api.YouTube.Playlists.GetPlaylists(new YouTubeGetPlaylistListOptions + try { - Ids = new[] { playlistId } - }); + // Make sure we have a TEMP directory + string dir = IOHelper.MapPath("~/App_Data/TEMP/YouTube/"); + Directory.CreateDirectory(dir); - // Get a reference to the playlist (using "Single" as there should be exactly one playlist) - var playlist = response1.Body.Items.Single(); - - // Save the playlist to the disk - JsonUtils.SaveJsonObject(dir + "Playlist_" + playlist.Id + ".json", playlist); + // Make an initial request to get information about the playlist + var response1 = Api.YouTube.Playlists.GetPlaylists(new YouTubeGetPlaylistListOptions + { + Ids = new[] {playlistId} + }); - // List of all video IDs - List ids = new List(); + // Get a reference to the playlist (using "Single" as there should be exactly one playlist) + var playlist = response1.Body.Items.Single(); - // Initialize the options for getting the playlist items - var playlistItemsOptions = new YouTubeGetPlaylistItemListOptions(playlistId) - { + // Save the playlist to the disk + JsonUtils.SaveJsonObject(dir + "Playlist_" + playlist.Id + ".json", playlist); - // Maximum allowed value is 50 - MaxResults = 50 + // List of all video IDs + List ids = new List(); - }; + // Initialize the options for getting the playlist items + var playlistItemsOptions = new YouTubeGetPlaylistItemListOptions(playlistId) + { - int page = 0; - while (page < 10) - { + // Maximum allowed value is 50 + MaxResults = 50 - // Get the playlist items - var response2 = Api.YouTube.PlaylistItems.GetPlaylistItems(playlistItemsOptions); + }; - // Append each video ID to the list - foreach (var item in response2.Body.Items) + int page = 0; + while (page < 10) { - ids.Add(item.VideoId); - } - // Break the loop if there are no additional pages - if (String.IsNullOrWhiteSpace(response2.Body.NextPageToken)) - { - break; - } + // Get the playlist items + var response2 = Api.YouTube.PlaylistItems.GetPlaylistItems(playlistItemsOptions); - // Update the options with the page token - playlistItemsOptions.PageToken = response2.Body.NextPageToken; + // Append each video ID to the list + foreach (var item in response2.Body.Items) + { + ids.Add(item.VideoId); + } - page++; + // Break the loop if there are no additional pages + if (String.IsNullOrWhiteSpace(response2.Body.NextPageToken)) + { + break; + } - } + // Update the options with the page token + playlistItemsOptions.PageToken = response2.Body.NextPageToken; - // Iterate through groups of IDs (maximum 50 items per group) - foreach (var group in ids.Where(x => !ExistsOnDiskAndIsUpToDate(x)).InGroupsOf(50)) - { + page++; + + } - // Initialize the video options - var videosOptions = new YouTubeGetVideoListOptions + // Iterate through groups of IDs (maximum 50 items per group) + foreach (var group in ids.Where(x => !ExistsOnDiskAndIsUpToDate(x)).InGroupsOf(50)) { - Ids = group.ToArray(), - Part = YouTubeVideoParts.Snippet + YouTubeVideoParts.ContentDetails + YouTubeVideoParts.Statistics - }; - // Make a request to the APi to get video information - var res3 = Api.YouTube.Videos.GetVideos(videosOptions); + // Initialize the video options + var videosOptions = new YouTubeGetVideoListOptions + { + Ids = group.ToArray(), + Part = YouTubeVideoParts.Snippet + YouTubeVideoParts.ContentDetails + + YouTubeVideoParts.Statistics + }; - // Iterate through the videos - foreach (var video in res3.Body.Items) - { + // Make a request to the APi to get video information + var res3 = Api.YouTube.Videos.GetVideos(videosOptions); - // Save the video to the disk - string path = dir + "Video_" + video.Id + ".json"; - JsonUtils.SaveJsonObject(path, video); + // Iterate through the videos + foreach (var video in res3.Body.Items) + { - // Download the thumnails for each video - var thumbnailUrl = GetThumbnail(video).Url; + // Save the video to the disk + string path = dir + "Video_" + video.Id + ".json"; + JsonUtils.SaveJsonObject(path, video); - const string mediaRoot = "~/media/YouTube"; - var thumbnailFile = IOHelper.MapPath($"{mediaRoot}/{video.Id}.jpg"); + // Download the thumnails for each video + var thumbnailUrl = GetThumbnail(video).Url; - var mediaPath = IOHelper.MapPath(mediaRoot); - if (Directory.Exists(mediaPath) == false) - Directory.CreateDirectory(mediaPath); + const string mediaRoot = "~/media/YouTube"; + var thumbnailFile = IOHelper.MapPath($"{mediaRoot}/{video.Id}.jpg"); - if (File.Exists(thumbnailFile)) - continue; + var mediaPath = IOHelper.MapPath(mediaRoot); + if (Directory.Exists(mediaPath) == false) + Directory.CreateDirectory(mediaPath); - using (var client = new WebClient()) - client.DownloadFile(thumbnailUrl, thumbnailFile); + if (File.Exists(thumbnailFile)) + continue; + + using (var client = new WebClient()) + client.DownloadFile(thumbnailUrl, thumbnailFile); + } } - } - // Load the videos from the individual files, and save them to a common file - JsonUtils.SaveJsonArray(dir + "Playlist_" + playlistId + "_Videos.json", ids.Select(LoadYouTubeVideo).WhereNotNull()); + // Load the videos from the individual files, and save them to a common file + JsonUtils.SaveJsonArray(dir + "Playlist_" + playlistId + "_Videos.json", + ids.Select(LoadYouTubeVideo).WhereNotNull()); + } + catch (Exception ex) + { + LogHelper.Error("Could not get data from YouTube", ex); + } } private static YouTubeVideoThumbnail GetThumbnail(YouTubeVideo video) From 26a60fb72696349b574eca9c7fc9ac1530f94a8a Mon Sep 17 00:00:00 2001 From: James Wilkinson Date: Sat, 11 Apr 2020 21:37:43 +0100 Subject: [PATCH 4/5] Added null check on results before returning tweets to aid debugging --- OurUmbraco/Community/Twitter/TwitterService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OurUmbraco/Community/Twitter/TwitterService.cs b/OurUmbraco/Community/Twitter/TwitterService.cs index 7e7403f094..24973acbaf 100644 --- a/OurUmbraco/Community/Twitter/TwitterService.cs +++ b/OurUmbraco/Community/Twitter/TwitterService.cs @@ -45,7 +45,7 @@ public TweetsModel GetTweets(int numberOfResults, bool adminOverview) }; var results = service.Search(options); - return results.Statuses; + return results?.Statuses; }, TimeSpan.FromMinutes(2)); From 7da0f3d974200473f7baa9026b00eaa90964e35f Mon Sep 17 00:00:00 2001 From: James Wilkinson Date: Sat, 11 Apr 2020 22:00:03 +0100 Subject: [PATCH 5/5] #532 Increased the time-span to 60 minutes to reduce request count --- OurUmbraco/Community/Meetup/MeetupService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OurUmbraco/Community/Meetup/MeetupService.cs b/OurUmbraco/Community/Meetup/MeetupService.cs index 212a1533e5..da37bc0207 100644 --- a/OurUmbraco/Community/Meetup/MeetupService.cs +++ b/OurUmbraco/Community/Meetup/MeetupService.cs @@ -148,7 +148,7 @@ public MeetupEventsModel GetUpcomingMeetups() return items.OrderBy(x => x.Event.Time).ToArray(); - }, TimeSpan.FromMinutes(30)); + }, TimeSpan.FromMinutes(60)); } catch (Exception ex) {