Merge pull request hmol#2 from ed-graham/feature/specify-HTTP-codes-t…

…o-log

Added InterestingHttpStatusCodes key to app.config

LinkCrawler/LinkCrawler.Tests/UtilsTests/ExtensionsTests/HttpStatusCodeExtensionsTests.cs

@@ -17,7 +17,7 @@ public class HttpStatusCodeExtensionsTests
         [TestCase(HttpStatusCode.OK ,"2xx,xX0", true, Description = "Multiple wildcard codes allowed")]
         public void Will_match_a_single_code_exactly(HttpStatusCode givenCode, string givenConfig, bool expectedOutcome)
         {
-            var result = givenCode.IsSuccess(givenConfig);
+            var result = givenCode.IsMatch(givenConfig);
 
             Assert.That(result, Is.EqualTo(expectedOutcome));
         }

LinkCrawler/LinkCrawler/App.config

@@ -11,9 +11,10 @@
     <add key="CheckImages" value="true"/>
     <add key="BaseUrl" value="https://github.com"/>
     <add key="SuccessHttpStatusCodes" value="1xx,2xx,3xx"/>
+    <add key="InterestingHttpStatusCodes" value="4xx,9xx"/>
     <!--explanation of regex below: http://regexr.com/3cqt9 -->
     <add key="ValidUrlRegex" value="(^http[s]?:\/{2})|(^www)|(^\/{1,2})"/>
-    <add key="PrintSummary" value="false"/>
+    <add key="PrintSummary" value="true"/>
     <add key="Csv.FilePath" value="C:\tmp\output.csv"/>
     <add key="Csv.Overwrite" value="true"/>
     <add key="Csv.Delimiter" value=";"/>

LinkCrawler/LinkCrawler/LinkCrawler.cs

@@ -86,18 +86,21 @@ public void CrawlForLinksInResponse(IResponseModel responseModel)
 
         public void WriteOutput(IResponseModel responseModel)
         {
-            if (!responseModel.IsSuccess)
+            if (responseModel.IsInteresting)
             {
-                foreach (var output in Outputs)
+                if (!responseModel.IsSuccess)
                 {
-                    output.WriteError(responseModel);
+                    foreach (var output in Outputs)
+                    {
+                        output.WriteError(responseModel);
+                    }
                 }
-            }
-            else if (!OnlyReportBrokenLinksToOutput)
-            {
-                foreach (var output in Outputs)
+                else if (!OnlyReportBrokenLinksToOutput)
                 {
-                    output.WriteInfo(responseModel);
+                    foreach (var output in Outputs)
+                    {
+                        output.WriteInfo(responseModel);
+                    }
                 }
             }
 

LinkCrawler/LinkCrawler/Models/IResponseModel.cs

@@ -10,6 +10,7 @@ public interface IResponseModel
         HttpStatusCode StatusCode { get; }
         int StatusCodeNumber { get; }
         bool IsSuccess { get; }
+        bool IsInteresting { get; }
         bool ShouldCrawl { get; }
         string ToString();
     }

LinkCrawler/LinkCrawler/Models/ResponseModel.cs

@@ -16,6 +16,7 @@ public class ResponseModel : IResponseModel
         public HttpStatusCode StatusCode { get; }
         public int StatusCodeNumber { get { return (int)StatusCode; } }
         public bool IsSuccess { get; }
+        public bool IsInteresting { get; }
         public bool ShouldCrawl { get; }
         public string ErrorMessage { get; }
 
@@ -26,6 +27,7 @@ public ResponseModel(IRestResponse restResponse, RequestModel requestModel, ISet
             RequestedUrl = requestModel.Url;
             Location = restResponse.GetHeaderByName("Location"); // returns null if no Location header present in the response
             ErrorMessage = restResponse.ErrorMessage;
+            IsInteresting = settings.IsInteresting(StatusCode);
 
             IsSuccess = settings.IsSuccess(StatusCode);
             if (!IsSuccess)
@@ -49,7 +51,7 @@ public override string ToString()
             }
             else
             {
-                if (StatusCodeNumber == 301 || StatusCodeNumber == 302)
+                if (!String.IsNullOrEmpty(Location))
                 {
                     return $"{StatusCodeNumber}\t{StatusCode}\t{RequestedUrl}{Environment.NewLine}\t->\t{Location}";
                 }

LinkCrawler/LinkCrawler/Utils/Extensions/HttpStatusCodeExtensions.cs

@@ -8,16 +8,19 @@ namespace LinkCrawler.Utils.Extensions
     public static class HttpStatusCodeExtensions
     {
         /// <summary>
-        /// This method will determine if an HttpStatusCode represents a "success" or not
-        /// based on the configuration string you pass in.
+        /// This method will determine if an HttpStatusCode matches the expression in
+        /// the configuration string you pass in.
         /// You can pass literal codes like 100, 200, 404
         /// Or you can pass in simple patterns using "x"s as wildcards like: 1xx, xx4
         /// </summary>
         /// <param name="this">The HttpStatus code to use</param>
         /// <param name="configuredCodes">CSV of HttpStatus codes</param>
         /// <returns></returns>
-        public static bool IsSuccess(this HttpStatusCode @this, string configuredCodes)
+        public static bool IsMatch(this HttpStatusCode @this, string configuredCodes)
         {
+            // if * is passed in, return true (helpful if app.config setting is missing)
+            if (configuredCodes.Equals("*")) return true;
+
             var codeCollection = configuredCodes
                 .Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries)   // split into array
                 .Select(c => c.Trim().ToLower())          // trim off any spaces, and make lowercase (this allows for "100,20X" and "100, 20x)"

LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs

@@ -16,6 +16,7 @@ public static class AppSettings
             public const string CsvOverwrite = "Csv.Overwrite";
             public const string CsvDelimiter = "Csv.Delimiter";
             public const string SuccessHttpStatusCodes = "SuccessHttpStatusCodes";
+            public const string InterestingHttpStatusCodes = "InterestingHttpStatusCodes";
             public const string OutputProviders = "outputProviders";
             public const string PrintSummary = "PrintSummary";
         }

LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs

@@ -28,6 +28,8 @@ public interface ISettings
 
         bool IsSuccess(HttpStatusCode statusCode);
 
+        bool IsInteresting(HttpStatusCode statusCode);
+
         bool PrintSummary { get; }
     }
 }

LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs

@@ -34,7 +34,12 @@ public string SlackWebHookUrl
         public string ValidUrlRegex => @"(^http[s]?:\/{2})|(^www)|(^\/{1,2})";
 
         public bool IsSuccess(HttpStatusCode statusCode) {
-            return statusCode.IsSuccess("1xx,2xx,3xx");
+            return statusCode.IsMatch("1xx,2xx,3xx");
+        }
+
+        public bool IsInteresting(HttpStatusCode statusCode)
+        {
+            return statusCode.IsMatch("3xx");
         }
 
         public MockSettings(bool includeWebHookUrl) {

LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs

@@ -45,7 +45,13 @@ public class Settings : ISettings
         public bool IsSuccess(HttpStatusCode statusCode)
         {
             var configuredCodes = ConfigurationManager.AppSettings[Constants.AppSettings.SuccessHttpStatusCodes] ?? "";
-            return statusCode.IsSuccess(configuredCodes);
+            return statusCode.IsMatch(configuredCodes);
+        }
+
+        public bool IsInteresting(HttpStatusCode statusCode)
+        {
+            var configuredCodes = ConfigurationManager.AppSettings[Constants.AppSettings.InterestingHttpStatusCodes] ?? "*";
+            return statusCode.IsMatch(configuredCodes);
         }
     }
 }