Merge pull request hmol#3 from ed-graham/feature/follow-redirects

Follow specified redirects when flag set
robsiera · Sep 16, 2018 · 6248312 · 6248312
2 parents 6882365 + 0e99fe7
commit 6248312
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 9 deletions.
diff --git a/LinkCrawler/LinkCrawler/App.config b/LinkCrawler/LinkCrawler/App.config
@@ -9,9 +9,11 @@
   <appSettings>
     <add key="OnlyReportBrokenLinksToOutput" value="false"/>
     <add key="CheckImages" value="true"/>
+    <add key="FollowRedirects" value="false"/>
     <add key="BaseUrl" value="https://github.com"/>
     <add key="SuccessHttpStatusCodes" value="1xx,2xx,3xx"/>
     <add key="InterestingHttpStatusCodes" value="*"/>
+    <add key="RedirectHttpStatusCodes" value="3xx"/>
     <!--explanation of regex below: http://regexr.com/3cqt9 -->
     <add key="ValidUrlRegex" value="(^http[s]?:\/{2})|(^www)|(^\/{1,2})"/>
     <add key="PrintSummary" value="true"/>

diff --git a/LinkCrawler/LinkCrawler/LinkCrawler.cs b/LinkCrawler/LinkCrawler/LinkCrawler.cs
@@ -16,6 +16,7 @@ public class LinkCrawler
     {
         public string BaseUrl { get; set; }
         public bool CheckImages { get; set; }
+        public bool FollowRedirects { get; set; }
         public RestRequest RestRequest { get; set; }
         public IEnumerable<IOutput> Outputs { get; set; }
         public IValidUrlParser ValidUrlParser { get; set; }
@@ -30,6 +31,7 @@ public LinkCrawler(IEnumerable<IOutput> outputs, IValidUrlParser validUrlParser,
             Outputs = outputs;
             ValidUrlParser = validUrlParser;
             CheckImages = settings.CheckImages;
+            FollowRedirects = settings.FollowRedirects;
             UrlList = new List<LinkModel>();
             RestRequest = new RestRequest(Method.GET).SetHeader("Accept", "*/*");
             OnlyReportBrokenLinksToOutput = settings.OnlyReportBrokenLinksToOutput;
@@ -47,7 +49,7 @@ public void Start()
         public void SendRequest(string crawlUrl, string referrerUrl = "")
         {
             var requestModel = new RequestModel(crawlUrl, referrerUrl, BaseUrl);
-            var restClient = new RestClient(new Uri(crawlUrl)) { FollowRedirects = false };
+            var restClient = new RestClient(new Uri(crawlUrl)) { FollowRedirects = false }; // we don't want RestSharp following the redirects, otherwise we won't see them
 
             restClient.ExecuteAsync(RestRequest, response =>
             {
@@ -63,6 +65,18 @@ public void ProcessResponse(IResponseModel responseModel)
         {
             WriteOutput(responseModel);
 
+            // follow 3xx redirects
+            if (FollowRedirects && responseModel.IsRedirect)
+            {
+                string redirectUrl;
+                if (responseModel.Location.StartsWith("/"))
+                    redirectUrl = responseModel.RequestedUrl.GetUrlBase() + responseModel.Location;
+                else
+                    redirectUrl = responseModel.Location;
+                SendRequest(redirectUrl, responseModel.RequestedUrl);
+            }
+
+            // follow internal links in response
             if (responseModel.ShouldCrawl)
                 CrawlForLinksInResponse(responseModel);
         }

diff --git a/LinkCrawler/LinkCrawler/Models/IResponseModel.cs b/LinkCrawler/LinkCrawler/Models/IResponseModel.cs
@@ -7,10 +7,12 @@ public interface IResponseModel
         string Markup { get; }
         string RequestedUrl { get; }
         string ReferrerUrl { get; }
+        string Location { get; }
         HttpStatusCode StatusCode { get; }
         int StatusCodeNumber { get; }
         bool IsSuccess { get; }
         bool IsInteresting { get; }
+        bool IsRedirect{ get; }
         bool ShouldCrawl { get; }
         string ToString();
     }

diff --git a/LinkCrawler/LinkCrawler/Models/ResponseModel.cs b/LinkCrawler/LinkCrawler/Models/ResponseModel.cs
@@ -3,6 +3,7 @@
 using System;
 using System.Net;
 using LinkCrawler.Utils.Settings;
+using System.Collections.Generic;
 
 namespace LinkCrawler.Models
 {
@@ -17,6 +18,7 @@ public class ResponseModel : IResponseModel
         public int StatusCodeNumber { get { return (int)StatusCode; } }
         public bool IsSuccess { get; }
         public bool IsInteresting { get; }
+        public bool IsRedirect { get; }
         public bool ShouldCrawl { get; }
         public string ErrorMessage { get; }
 
@@ -27,11 +29,13 @@ public ResponseModel(IRestResponse restResponse, RequestModel requestModel, ISet
             RequestedUrl = requestModel.Url;
             Location = restResponse.GetHeaderByName("Location"); // returns null if no Location header present in the response
             ErrorMessage = restResponse.ErrorMessage;
+            IsSuccess = settings.IsSuccess(StatusCode);
             IsInteresting = settings.IsInteresting(StatusCode);
+            IsRedirect = settings.IsRedirect(StatusCode);
 
-            IsSuccess = settings.IsSuccess(StatusCode);
             if (!IsSuccess)
                 return;
+
             Markup = restResponse.Content;
             ShouldCrawl = IsSuccess && requestModel.IsInternalUrl && restResponse.IsHtmlDocument();
         }

diff --git a/LinkCrawler/LinkCrawler/Program.cs b/LinkCrawler/LinkCrawler/Program.cs
@@ -3,6 +3,7 @@
 using System;
 using LinkCrawler.Utils.Parsers;
 using LinkCrawler.Utils.Settings;
+using LinkCrawler.Utils.Extensions;
 
 namespace LinkCrawler
 {
@@ -21,13 +22,7 @@ static void Main(string[] args)
                     if (result)
                     {
                         // make sure the base URL is just a domain
-                        int prefixLength = parsed.IndexOf("//") + 2;
-                        if (parsed.Substring(prefixLength).IndexOf("/") > 0)
-                        {
-                            parsed = parsed.Substring(0, parsed.Substring(prefixLength).IndexOf("/") + prefixLength);
-                        }
-                        linkCrawler.BaseUrl = parsed;
-                        validUrlParser.BaseUrl = parsed;
+                        linkCrawler.BaseUrl = validUrlParser.BaseUrl = parsed.GetUrlBase();
                         linkCrawler.ValidUrlParser = validUrlParser;
                     }
                 }

diff --git a/LinkCrawler/LinkCrawler/Utils/Extensions/StringExtensions.cs b/LinkCrawler/LinkCrawler/Utils/Extensions/StringExtensions.cs
@@ -30,5 +30,15 @@ public static string TrimEnd(this string input, string suffixToRemove)
             }
             return input;
         }
+
+        public static string GetUrlBase(this string url)
+        {
+            int prefixLength = url.IndexOf("//") + 2;
+            if (url.Substring(prefixLength).IndexOf("/") > 0)
+            {
+                return url.Substring(0, url.Substring(prefixLength).IndexOf("/") + prefixLength);
+            }
+            else return url;
+        }
     }
 }
diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs b/LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs
@@ -8,6 +8,7 @@ public static class AppSettings
             public const string ValidUrlRegex = "ValidUrlRegex";
             public const string OnlyReportBrokenLinksToOutput = "OnlyReportBrokenLinksToOutput";
             public const string CheckImages = "CheckImages";
+            public const string FollowRedirects = "FollowRedirects";
             public const string SlackWebHookUrl = "Slack.WebHook.Url";
             public const string SlackWebHookBotName = "Slack.WebHook.Bot.Name";
             public const string SlackWebHookBotIconEmoji = "Slack.WebHook.Bot.IconEmoji";
@@ -17,6 +18,7 @@ public static class AppSettings
             public const string CsvDelimiter = "Csv.Delimiter";
             public const string SuccessHttpStatusCodes = "SuccessHttpStatusCodes";
             public const string InterestingHttpStatusCodes = "InterestingHttpStatusCodes";
+            public const string RedirectHttpStatusCodes = "RedirectHttpStatusCodes";
             public const string OutputProviders = "outputProviders";
             public const string PrintSummary = "PrintSummary";
         }

diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs b/LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs
@@ -10,6 +10,8 @@ public interface ISettings
 
         bool CheckImages { get; }
 
+        bool FollowRedirects { get; }
+
         bool OnlyReportBrokenLinksToOutput { get; }
 
         string SlackWebHookUrl { get; }
@@ -30,6 +32,8 @@ public interface ISettings
 
         bool IsInteresting(HttpStatusCode statusCode);
 
+        bool IsRedirect(HttpStatusCode statusCode);
+
         bool PrintSummary { get; }
     }
 }
diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs b/LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs
@@ -8,6 +8,8 @@ public class MockSettings : ISettings {
 
         public bool CheckImages => true;
 
+        public bool FollowRedirects => true;
+
         public string CsvDelimiter => ";";
 
         public string CsvFilePath => @"C:\tmp\output.csv";
@@ -38,6 +40,11 @@ public bool IsSuccess(HttpStatusCode statusCode) {
         }
 
         public bool IsInteresting(HttpStatusCode statusCode)
+        {
+            return statusCode.IsMatch("*");
+        }
+
+        public bool IsRedirect(HttpStatusCode statusCode)
         {
             return statusCode.IsMatch("3xx");
         }

diff --git a/LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs b/LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs
@@ -15,6 +15,9 @@ public class Settings : ISettings
         public bool CheckImages =>
             ConfigurationManager.AppSettings[Constants.AppSettings.CheckImages].ToBool();
 
+        public bool FollowRedirects =>
+            ConfigurationManager.AppSettings[Constants.AppSettings.FollowRedirects].ToBool();
+
         public bool OnlyReportBrokenLinksToOutput =>
             ConfigurationManager.AppSettings[Constants.AppSettings.OnlyReportBrokenLinksToOutput].ToBool();
 
@@ -53,5 +56,11 @@ public bool IsInteresting(HttpStatusCode statusCode)
             var configuredCodes = ConfigurationManager.AppSettings[Constants.AppSettings.InterestingHttpStatusCodes] ?? "*";
             return statusCode.IsMatch(configuredCodes);
         }
+
+        public bool IsRedirect(HttpStatusCode statusCode)
+        {
+            var configuredCodes = ConfigurationManager.AppSettings[Constants.AppSettings.RedirectHttpStatusCodes] ?? "3xx";
+            return statusCode.IsMatch(configuredCodes);
+        }
     }
 }