Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added SqlLite output #34

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions LinkCrawler/LinkCrawler/App.config
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
<?xml version="1.0" encoding="utf-8" ?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<section name="outputProviders" type="System.Configuration.DictionarySectionHandler" />
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
</configSections>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
Expand All @@ -20,14 +22,24 @@
<add key="Slack.WebHook.Url" value="https://hooks.slack.com/services/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" />
<add key="Slack.WebHook.Bot.Name" value="Homer Bot" />
<add key="Slack.WebHook.Bot.IconEmoji" value=":homer:" />
<add key="Slack.WebHook.Bot.MessageFormat" value="*Doh! There is a link not working*
Url: {0}
Statuscode: {1}
The link is placed on this page: {2}" />
<add key="Slack.WebHook.Bot.MessageFormat" value="*Doh! There is a link not working* Url: {0} Statuscode: {1} The link is placed on this page: {2}" />
</appSettings>
<outputProviders>
<add key="Console" value="LinkCrawler.Utils.Outputs.ConsoleOutput" />
<!--<add key="Csv" value="LinkCrawler.Utils.Outputs.CsvOutput" />-->
<!--<add key="Slack" value="LinkCrawler.Utils.Outputs.SlackOutput" />-->
</outputProviders>
<entityFramework>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.SqlConnectionFactory, EntityFramework" />
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
<provider invariantName="System.Data.SQLite.EF6" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6" />
</providers>
</entityFramework>
<system.data>
<DbProviderFactories>
<remove invariant="System.Data.SQLite.EF6" />
<add name="SQLite Data Provider (Entity Framework 6)" invariant="System.Data.SQLite.EF6" description=".NET Framework Data Provider for SQLite (Entity Framework 6)" type="System.Data.SQLite.EF6.SQLiteProviderFactory, System.Data.SQLite.EF6" />
<remove invariant="System.Data.SQLite" /><add name="SQLite Data Provider" invariant="System.Data.SQLite" description=".NET Framework Data Provider for SQLite" type="System.Data.SQLite.SQLiteFactory, System.Data.SQLite" /></DbProviderFactories>
</system.data>
</configuration>
15 changes: 15 additions & 0 deletions LinkCrawler/LinkCrawler/LinkCrawler.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
<TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
Expand Down Expand Up @@ -66,8 +68,13 @@
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.ComponentModel.DataAnnotations" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Data.SQLite, Version=1.0.106.0, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.Core.1.0.106.0\lib\net451\System.Data.SQLite.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
Expand All @@ -85,6 +92,7 @@
<Compile Include="Utils\Outputs\CsvOutput.cs" />
<Compile Include="Utils\Outputs\IOutput.cs" />
<Compile Include="Utils\Outputs\SlackOutput.cs" />
<Compile Include="Utils\Outputs\SqlLiteOutput.cs" />
<Compile Include="Utils\Settings\MockSettings.cs" />
<Compile Include="Utils\StructureMapRegistry.cs" />
<Compile Include="Utils\Extensions\RegexExtensions.cs" />
Expand Down Expand Up @@ -113,6 +121,13 @@
</ItemGroup>
<ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets" Condition="Exists('..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\System.Data.SQLite.Core.1.0.106.0\build\net451\System.Data.SQLite.Core.targets'))" />
</Target>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
Expand Down
83 changes: 83 additions & 0 deletions LinkCrawler/LinkCrawler/Utils/Outputs/SqlLiteOutput.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
using System.Data.SQLite;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using LinkCrawler.Models;
using LinkCrawler.Utils.Settings;

namespace LinkCrawler.Utils.Outputs
{
class SqlLiteOutput : IOutput, IDisposable
{
private readonly ISettings _settings;
private SQLiteConnection _sqLiteConnection;

public SqlLiteOutput(ISettings settings)
{
_settings = settings;
SetUp();
}

private void SetUp()
{
SQLiteConnection.CreateFile(_settings.SqlLiteFilePath);

_sqLiteConnection = new SQLiteConnection(String.Format("Data Source={0}Version=3;", _settings.SqlLiteFilePath));

string createStatusCodesTable = String.Format("CREATE TABLE status_codes (code SMALLINT NOT NULL PRIMARY KEY, status VARCHAR({0}))", GetMaxStatusCodeWordLength());
new SQLiteCommand(createStatusCodesTable, _sqLiteConnection).ExecuteNonQuery();

foreach (var code in Enum.GetValues(typeof(HttpStatusCode)))
{
string insertStatusCode = String.Format("INSERT OR REPLACE INTO status_codes (code, status) VALUES({0}, '{1}')", (int)code, code);
new SQLiteCommand(insertStatusCode, _sqLiteConnection).ExecuteNonQuery();
}

string createLinksTable = "CREATE TABLE links (code SMALLINT, url TEXT NOT NULL PRIMARY KEY, referrer TEXT)";
new SQLiteCommand(createLinksTable, _sqLiteConnection).ExecuteNonQuery();
}

public void WriteError(IResponseModel responseModel)
{
Write(responseModel);
}

public void WriteInfo(IResponseModel responseModel)
{
Write(responseModel);
}

public void WriteInfo(string[] Info)
{

}

public void Write(IResponseModel responseModel)
{
string insertLink = String.Format("INSERT OR UPDATE INTO links (code, url, referrer) values ({0}, '{1}', '{2}')", responseModel.StatusCodeNumber, responseModel.RequestedUrl, responseModel.ReferrerUrl);
new SQLiteCommand(insertLink, _sqLiteConnection).ExecuteNonQuery();
}

public void Dispose()
{
_sqLiteConnection.Close();
}

private static int GetMaxStatusCodeWordLength()
{
int result = 0;
foreach (var code in Enum.GetValues(typeof(HttpStatusCode)))
{
if (result < code.ToString().Length)
{
result = code.ToString().Length;
}

}
return result;
}
}
}
1 change: 1 addition & 0 deletions LinkCrawler/LinkCrawler/Utils/Settings/Constants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public static class AppSettings
public const string CsvFilePath = "Csv.FilePath";
public const string CsvOverwrite = "Csv.Overwrite";
public const string CsvDelimiter = "Csv.Delimiter";
public const string SqlLiteFilePath = "SqlLite.Filepath";
public const string SuccessHttpStatusCodes = "SuccessHttpStatusCodes";
public const string OutputProviders = "outputProviders";
public const string PrintSummary = "PrintSummary";
Expand Down
2 changes: 2 additions & 0 deletions LinkCrawler/LinkCrawler/Utils/Settings/ISettings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ public interface ISettings

string CsvDelimiter { get; }

string SqlLiteFilePath { get; }

bool IsSuccess(HttpStatusCode statusCode);

bool PrintSummary { get; }
Expand Down
2 changes: 2 additions & 0 deletions LinkCrawler/LinkCrawler/Utils/Settings/MockSettings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ public class MockSettings : ISettings {

public bool CsvOverwrite => true;

public string SqlLiteFilePath => @"C:\tmp\output.sqlite";

public bool OnlyReportBrokenLinksToOutput => false;

public string SlackWebHookBotIconEmoji => ":homer:";
Expand Down
3 changes: 3 additions & 0 deletions LinkCrawler/LinkCrawler/Utils/Settings/Settings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ public class Settings : ISettings
public string CsvDelimiter =>
ConfigurationManager.AppSettings[Constants.AppSettings.CsvDelimiter];

public string SqlLiteFilePath =>
ConfigurationManager.AppSettings[Constants.AppSettings.SqlLiteFilePath];

public bool PrintSummary =>
ConfigurationManager.AppSettings[Constants.AppSettings.PrintSummary].ToBool();

Expand Down
3 changes: 3 additions & 0 deletions LinkCrawler/LinkCrawler/packages.config
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
<package id="Bootstrapper.StructureMap" version="2.0.3.13" targetFramework="net452" />
<package id="CommonServiceLocator" version="1.3" targetFramework="net452" />
<package id="CommonServiceLocator.StructureMapAdapter.Unofficial" version="3.0.4.125" targetFramework="net452" />
<package id="EntityFramework" version="6.0.0" targetFramework="net452" />
<package id="HtmlAgilityPack" version="1.4.9" targetFramework="net452" />
<package id="RestSharp" version="105.2.3" targetFramework="net452" />
<package id="structuremap" version="3.0.4.125" targetFramework="net452" />
<package id="System.Data.SQLite" version="1.0.106.0" targetFramework="net452" />
<package id="System.Data.SQLite.Core" version="1.0.106.0" targetFramework="net452" />
</packages>