Skip to content

Commit

Permalink
Merge pull request #33 from Kretchen001/28-multipackage-analysis
Browse files Browse the repository at this point in the history 
28 multipackage analysis
  • Loading branch information
@KnYL3R
KnYL3R authored Jan 31, 2024
2 parents ac21183 + 7f5baec commit 9d78d72
Show file tree
Hide file tree
Showing 6 changed files with 346 additions and 45 deletions.
109 changes: 71 additions & 38 deletions code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using LiteDbLib.Controller;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Routing;
using Modells;
using Newtonsoft.Json;
using SerilogTimings;
Expand Down Expand Up @@ -62,15 +63,16 @@ private static void ExploreFolder(string folderPath, List<string> fileList) {
}
}

[HttpGet, Route("checkSinglePackage")]
[HttpPost]
[Route("checkSinglePackage")]
public IActionResult CheckSinglePackage([FromHeader] string packageName,
[FromHeader] bool isDbSearch = true,
[FromHeader] string? packageVersion = "") {
if (packageVersion!.Equals("")) { // search all versions
if (isDbSearch) {
SearchDbController searchDbController = new SearchDbController();
List<CveResult> res = [];
using(Operation.Time($"{packageName}")) {
using(Operation.Time($"Package \"{packageName}\"")) {
res = searchDbController.SearchSinglePackage(packageName);
}
if (res.Count > 0) {
Expand All @@ -81,51 +83,82 @@ public IActionResult CheckSinglePackage([FromHeader] string packageName,
}
}
else {
// find all json files of cve
List<string> fileList = new List<string>();
List<int> indexToDelete = new List<int>();
string path = $"{AppDomain.CurrentDomain.BaseDirectory}raw";
ExploreFolder(path, fileList);
// find all json files of cve
return Ok(JsonConvert.SerializeObject(SearchInJson(packageName)));
}
}
else {
// TODO: search after a specific version
}
return Ok();
}

foreach (int i in Enumerable.Range(0, fileList.Count)) {
if (!Regex.IsMatch(fileList[i], @"CVE-[-\S]+.json")) {
indexToDelete.Add(i);
}
private List<CveResult> SearchInJson(string packageName) {
List<string> fileList = new List<string>();
List<int> indexToDelete = new List<int>();
string path = $"{AppDomain.CurrentDomain.BaseDirectory}raw";
ExploreFolder(path, fileList);

foreach (int i in Enumerable.Range(0, fileList.Count)) {
if (!Regex.IsMatch(fileList[i], @"CVE-[-\S]+.json")) {
indexToDelete.Add(i);
}
}
foreach (int i in Enumerable.Range(0, indexToDelete.Count)) {
fileList.RemoveAt(indexToDelete[i] - i);
}
// search in the files
List<CveResult> results = [];
using (Operation.Time($"Package \"{packageName}\"")) {
int start = 0;
foreach (int i in Enumerable.Range(start, fileList.Count - start)) {
CVEcomp item = JsonConvert.DeserializeObject<CVEcomp>(System.IO.File.ReadAllText(fileList[i]))!;
if (i % 100 == 0) {
Console.WriteLine(fileList[i] + " - " + i);
}
foreach (int i in Enumerable.Range(0, indexToDelete.Count)) {
fileList.RemoveAt(indexToDelete[i] - i);
if (item.containers.cna.affected is null || item.containers.cna.affected.Any(x => x.product is null)) {
continue;
}
// search in the files
List<CveResult> results = [];
using (Operation.Time($"Packge \"{packageName}\"")) {
int start = 0;
foreach (int i in Enumerable.Range(start, fileList.Count - start)) {
CVEcomp item = JsonConvert.DeserializeObject<CVEcomp>(System.IO.File.ReadAllText(fileList[i]))!;
if (i % 100 == 0) {
Console.WriteLine(fileList[i] + " - " + i);
}
if (item.containers.cna.affected is null || item.containers.cna.affected.Any(x => x.product is null)) {
continue;
}
if (item.containers.cna.affected.Any(y => y.product.Equals(packageName))) {
foreach (int j in Enumerable.Range(0, item.containers.cna.affected.Count)) {
foreach (Modells.Version version in item.containers.cna.affected[j].versions) {
results.Add(new CveResult() {
CveNumber = item.cveMetadata.cveId,
Version = version.version,
});
}
}
if (item.containers.cna.affected.Any(y => y.product.Equals(packageName))) {
foreach (int j in Enumerable.Range(0, item.containers.cna.affected.Count)) {
foreach (Modells.Version version in item.containers.cna.affected[j].versions) {
results.Add(new CveResult() {
CveNumber = item.cveMetadata.cveId,
Version = version.version,
});
}
}
}
return Ok(JsonConvert.SerializeObject(results));
}
}
else {
// TODO: search after a specific version
return results;
}

[HttpPost]
[Route("checkPackageList")]
public async Task<IActionResult> CheckPackageListAsync([FromBody] List<Tuple<string, string>> packages) {
if (packages.Count > 0) {
SearchDbController searchDbController = new SearchDbController();
List<CveResult> results = [];
List<string> strings = [];
foreach (Tuple<string, string> item in packages) {
strings.Add(item.Item1);
if (item.Item1.Equals("")) {
continue;
}
using (Operation.Time($"")) {
//results.AddRange(searchDbController.SearchSinglePackage(item.Item1));
}
}
results = await searchDbController.SearchPackagesAsList(strings);
if (results.Count > 0) {
return Ok(JsonConvert.SerializeObject(results));
}
else {
return NoContent();
}
}
return Ok();
return Ok("No package List delivered.");
}
}
}
141 changes: 135 additions & 6 deletions code/AmIVulnerable/LiteDbLib/Controller/SearchDbController.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
using LiteDB;
using Modells;
using System.Collections;
using System.Collections.Generic;
using System.Text.RegularExpressions;

using CM = System.Configuration.ConfigurationManager;

namespace LiteDbLib.Controller {
Expand All @@ -12,7 +9,7 @@ public class SearchDbController {

#region Config
/// <summary>Return the location, where the database is located.</summary>
private string saveDir {
private static string SaveDir {
get {
if (!Directory.Exists(AppDomain.CurrentDomain.BaseDirectory + "Data")) {
Directory.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory + "Data");
Expand All @@ -32,7 +29,8 @@ private string saveDir {
/// <returns></returns>
public SearchDbController() {
Regex yearDb = new Regex(@"(\d{4}).litedb");
DirectoryInfo currentDirectory = new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory + CM.AppSettings["PathToDbs"]);
DirectoryInfo currentDirectory = new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory
+ CM.AppSettings["PathToDbs"]);
FileInfo[] files = currentDirectory.GetFiles();

foreach (FileInfo file in files) {
Expand All @@ -46,7 +44,7 @@ public List<CveResult> SearchSinglePackage(string designation) {
List<CveResult> results = [];
foreach (string dbFile in dbFiles) {
Console.WriteLine($"Akutell - {dbFile}");
using (LiteDatabase db = new LiteDatabase($"{saveDir}\\{dbFile}")) {
using (LiteDatabase db = new LiteDatabase($"{SaveDir}\\{dbFile}")) {
ILiteCollection<CVEcomp> col = db.GetCollection<CVEcomp>(tableName);

IEnumerator<CVEcomp> pointer = col.Query().ToEnumerable().GetEnumerator();
Expand All @@ -62,6 +60,7 @@ public List<CveResult> SearchSinglePackage(string designation) {
results.Add(new CveResult() {
CveNumber = item.cveMetadata.cveId,
Version = version.version,
Designation = designation,
});
}
}
Expand All @@ -71,5 +70,135 @@ public List<CveResult> SearchSinglePackage(string designation) {
}
return results;
}

public async Task<List<CveResult>> SearchPackagesAsList(List<string> designations) {
List<CveResult> results = [];
int pipeCount = 0;
int designtaionSeriesSum = Enumerable.Range(1, designations.Count).Sum();
for (int i = 0; i < designations.Count; i += 1) {
if (pipeCount < designtaionSeriesSum) { // fill pipe
int j = i;
int dbFilePosition = dbFiles.Count - 1;
while (j >= 0 && dbFilePosition >= 0) {
Task<List<CveResult>>[] tasks = new Task<List<CveResult>>[j + 1];
foreach (int taskDbIndex in Enumerable.Range(0, dbFiles.Count - 1)) {
string db = dbFiles[dbFilePosition];
string des = designations[j];
tasks[taskDbIndex] = Task.Run(() => SearchInDb(db, des));
pipeCount += 1;
dbFilePosition -= 1; j -= 1;
if (j < 0) {
break;
}
}
List<CveResult>[] res = await Task.WhenAll(tasks);
//await Console.Out.WriteLineAsync(); // only for debug check
foreach (List<CveResult> x in res) {
results.AddRange(x);
}
}
if (i == (designations.Count - 1)) {
i -= 1; // if pipe filled let check the pipeCount again and reset so the highest element
}
}
else if ((pipeCount != (designtaionSeriesSum - designations.Count))
&& (
(
(designations.Count > dbFiles.Count) && ((pipeCount - dbFiles.Count) < designtaionSeriesSum)
) // more designations, so check the dbCounter
||
(
(designations.Count < dbFiles.Count) && (pipeCount - designations.Count) < designtaionSeriesSum)
) // more dbFiles, so check the designationCounter
)
{ // fill the pipe with new items and remove old
int j = i;
int dbFilePosition = dbFiles.Count - 2;
while (j >= 0 && dbFilePosition >= 0) {
Task<List<CveResult>>[] tasks = new Task<List<CveResult>>[j + 1];
foreach (int taskDbIndex in Enumerable.Range(0, dbFiles.Count - 1)) {
string db = dbFiles[dbFilePosition];
string des = designations[j];
tasks[taskDbIndex] = Task.Run(() => SearchInDb(db, des));
pipeCount += 1;
dbFilePosition -= 1; j -= 1;
if (j < 0) {
break;
}
}
List<CveResult>[] res = await Task.WhenAll(tasks);
//await Console.Out.WriteLineAsync(); // only for debug check
foreach (List<CveResult> x in res) {
results.AddRange(x);
}
if (dbFilePosition >= 0) {
j = i;
dbFilePosition += (designations.Count - 1);
}
}
if (i == (designations.Count - 1)) {
i -= 1; // if pipe filled let check the pipeCount again and reset so the highest element
}
}
else { // drain the pipe
int j = i;
int drainCount = 1;
int dbFilePosition = designations.Count - 2;
while (j >= drainCount && dbFilePosition >= 0) {
Task<List<CveResult>>[] tasks = new Task<List<CveResult>>[dbFilePosition + 1];
foreach (int taskDbIndex in Enumerable.Range(0, dbFiles.Count - 1)) {
string db = dbFiles[dbFilePosition];
string des = designations[j];
tasks[taskDbIndex] = Task.Run(() => SearchInDb(db, des));
pipeCount += 1;
dbFilePosition -= 1; j -= 1;
if (j < 0 || dbFilePosition == -1) {
break;
}
}
List<CveResult>[] res = await Task.WhenAll(tasks);
//await Console.Out.WriteLineAsync(); // only for debug check
foreach (List<CveResult> x in res) {
results.AddRange(x);
}
drainCount += 1;
if (dbFilePosition <= 0) {
j = i;
dbFilePosition += (designations.Count - drainCount);
}
}
}
}
return results;
}

private List<CveResult> SearchInDb(string dbFile, string designation) {
//await Console.Out.WriteLineAsync($"{dbFile} search for {designation}"); // only for debug check
List<CveResult> results = [];
using (LiteDatabase db = new LiteDatabase($"{SaveDir}\\{dbFile}")) {
ILiteCollection<CVEcomp> col = db.GetCollection<CVEcomp>(tableName);

IEnumerator<CVEcomp> pointer = col.Query().ToEnumerable().GetEnumerator();

while (pointer.MoveNext()) {
CVEcomp item = pointer.Current;
if (item.containers.cna.affected is null || item.containers.cna.affected.Any(x => x.product is null)) {
continue;
}
if (item.containers.cna.affected.Any(y => y.product.Equals(designation))) {
foreach (int i in Enumerable.Range(0, item.containers.cna.affected.Count)) {
foreach (Modells.Version version in item.containers.cna.affected[i].versions) {
results.Add(new CveResult() {
CveNumber = item.cveMetadata.cveId,
Version = version.version,
Designation = designation,
});
}
}
}
}
}
return results;
}
}
}
1 change: 1 addition & 0 deletions code/AmIVulnerable/Modells/CveResult.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ public class CveResult {

public string CveNumber { get; set; } = "";
public string Version { get; set; } = "";
public string Designation { get; set; } = "";

/// <summary>Empty ctor</summary>
public CveResult() {
Expand Down
2 changes: 1 addition & 1 deletion documentation/fragments/analysis.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,5 @@ Zur Leistung ergaben sich dabei folgende Werte:
## JSON-Dateien

```log
2024-01-24 19:08:18.676 +01:00 [INF] Packge "LiteDB" completed in 853896.3 ms
2024-01-24 19:08:18.676 +01:00 [INF] Package "LiteDB" completed in 853896.3 ms
```
Loading

0 comments on commit 9d78d72

Please sign in to comment.