From f4da8ea961f7adfae2d814a3bdc396ba0369a331 Mon Sep 17 00:00:00 2001 From: Kretchen001 <83697846+Kretchen001@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:09:13 +0200 Subject: [PATCH 1/7] add Update-Endpoint --- .../AmIVulnerable/Controllers/DbController.cs | 122 +++++++++++++++++- .../Controllers/GitController.cs | 4 +- 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs index 6714d6a..ecb1241 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs @@ -5,7 +5,9 @@ using Newtonsoft.Json; using SerilogTimings; using System.Data; +using System.Diagnostics; using System.Text.RegularExpressions; +using CM = System.Configuration.ConfigurationManager; namespace AmIVulnerable.Controllers { @@ -43,7 +45,7 @@ public IActionResult IsRawDataThere() { [HttpGet] [Route("ConvertRawCveToDb")] public IActionResult ConvertRawFilesToMySql() { - using (Operation.Time("TaskDuration")) { + using (Operation.Time("ConvertRawCveToDb")) { List fileList = new List(); List indexToDelete = new List(); string path = "raw"; @@ -125,6 +127,124 @@ public IActionResult ConvertRawFilesToMySql() { } } + [HttpGet] + [Route("")] + public IActionResult UpdateCveDatabase() { + using (Operation.Time("UpdateCveDatabase")) { + try { + // MySql Connection + MySqlConnection connection = new MySqlConnection(Configuration["ConnectionStrings:cvedb"]); + + MySqlCommand cmdTestIfTableExist = new MySqlCommand($"" + + $"SELECT COUNT(*) " + + $"FROM information_schema.TABLES" + + $"WHERE (TABLE_SCHEMA = 'cve') AND (TABLE_NAME = 'cve')", connection); + + connection.Open(); + int count = cmdTestIfTableExist.ExecuteNonQuery(); + connection.Close(); + + if (count == 0) { + return BadRequest("Table not exist!\nPlease download the cve and create a database before try to update it."); + } + + //start update process + try { + ProcessStartInfo process = new ProcessStartInfo { + FileName = "cmd", + RedirectStandardInput = true, + WorkingDirectory = $"", + }; + + Process runProcess = Process.Start(process)!; + runProcess.StandardInput.WriteLine($"git " + + $"clone {CM.AppSettings["StandardCveUrlPlusTag"]!} " + // git url + $"raw"); // target dir + runProcess.StandardInput.WriteLine($"exit"); + runProcess.WaitForExit(); + } + catch (Exception ex) { + return BadRequest(ex.StackTrace); + } + + //read the file List + List fileList = new List(); + List indexToDelete = new List(); + string path = "raw"; + ExploreFolder(path, fileList); + + //filter for json files + foreach (int i in Enumerable.Range(0, fileList.Count)) { + if (!Regex.IsMatch(fileList[i], @"CVE-[-\S]+.json")) { + indexToDelete.Add(i); + } + } + foreach (int i in Enumerable.Range(0, indexToDelete.Count)) { + fileList.RemoveAt(indexToDelete[i] - i); + } + + // Drop Index for faster insert + MySqlCommand cmdIndexDrop = new MySqlCommand("DROP INDEX idx_designation ON cve;", connection); + + connection.Open(); + cmdIndexDrop.ExecuteNonQuery(); + connection.Close(); + + //start insert/update in MySQL + int insertAndUpdateIndex = 0; + foreach (string x in fileList) { + string insertIntoString = "INSERT INTO cve(cve_number, designation, version_affected, full_text) " + + "VALUES(@cve, @des, @ver, @ful) " + + "ON DUPLICATE KEY UPDATE " + + "version_affected = @ver" + + "full_text = @ful"; + MySqlCommand cmdInsert = new MySqlCommand(insertIntoString, connection); + + string json = System.IO.File.ReadAllText(x); + CVEcomp cve = JsonConvert.DeserializeObject(json)!; + + string affected = ""; + foreach (Affected y in cve.containers.cna.affected) { + foreach (Modells.Version z in y.versions) { + affected += z.version + $"({z.status}) |"; + } + } + if (affected.Length > 25_000) { + affected = "to long -> view full_text"; + } + string product = "n/a"; + try { + product = cve.containers.cna.affected[0].product; + if (product.Length > 500) { + product = product[0..500]; + } + } + catch { + product = "n/a"; + } + cmdInsert.Parameters.AddWithValue("@cve", cve.cveMetadata.cveId); + cmdInsert.Parameters.AddWithValue("@des", product); + cmdInsert.Parameters.AddWithValue("@ver", affected); + cmdInsert.Parameters.AddWithValue("@ful", JsonConvert.SerializeObject(cve, Formatting.None)); + + connection.Open(); + insertAndUpdateIndex += cmdInsert.ExecuteNonQuery(); + connection.Close(); + } + + connection.Open(); + MySqlCommand cmdIndexCreated = new MySqlCommand("CREATE INDEX idx_designation ON cve (designation);", connection); + cmdIndexCreated.ExecuteNonQuery(); + connection.Close(); + + return Ok(insertAndUpdateIndex); + } + catch (Exception ex) { + return BadRequest(ex.StackTrace + "\n\n" + ex.Message); + } + } + } + /// Check for an cve entry of a package with all its versions /// Name of package to search /// true: search db, false: search raw-json diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs index 94106ed..a3dccf7 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs @@ -1,7 +1,5 @@ -using LibGit2Sharp; -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc; using System.Diagnostics; -using System.Security.Policy; using CM = System.Configuration.ConfigurationManager; namespace AmIVulnerable.Controllers { From 28d6375169a9d7107b99c867160969462b6e61f2 Mon Sep 17 00:00:00 2001 From: Kretchen001 <83697846+Kretchen001@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:20:19 +0200 Subject: [PATCH 2/7] add Endpoint that request the fulltext CVE from the database --- .../AmIVulnerable/Controllers/DbController.cs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs index ecb1241..9363f5c 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs @@ -1,8 +1,10 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Elfie.Diagnostics; using Modells; using MySql.Data.MySqlClient; using Newtonsoft.Json; +using NuGet.Protocol.Plugins; using SerilogTimings; using System.Data; using System.Diagnostics; @@ -127,6 +129,8 @@ public IActionResult ConvertRawFilesToMySql() { } } + /// + /// [HttpGet] [Route("")] public IActionResult UpdateCveDatabase() { @@ -245,6 +249,43 @@ public IActionResult UpdateCveDatabase() { } } + /// + /// + /// + [HttpGet] + [Route("getFullTextFromCveNumber")] + public IActionResult GetFullTextCve([FromHeader] string? cve_number) { + using (Operation.Time("GetFullTextCve")) { + if (cve_number is null) { + return BadRequest("Empty Header"); + } + try { + // MySql Connection + MySqlConnection connection = new MySqlConnection(Configuration["ConnectionStrings:cvedb"]); + + connection.Open(); + MySqlCommand cmdIndexCreated = new MySqlCommand($"" + + $"SELECT full_text " + + $"FROM cve.cve " + + $"WHERE cve_number = '{cve_number}';", connection); + MySqlDataReader reader = cmdIndexCreated.ExecuteReader(); + DataTable resDataTable = new DataTable(); + resDataTable.Load(reader); + connection.Close(); + + if (resDataTable.Rows.Count == 0) { + return NoContent(); + } + + return Ok(JsonConvert.SerializeObject(resDataTable.Rows[0]["full_text"].ToString())); + } + catch (Exception ex) { + return BadRequest(ex.StackTrace + "\n\n" + ex.Message); + + } + } + } + /// Check for an cve entry of a package with all its versions /// Name of package to search /// true: search db, false: search raw-json From 307b394e73796654b98133b24cda6d4a3bdde7ba Mon Sep 17 00:00:00 2001 From: Kretchen001 <83697846+Kretchen001@users.noreply.github.com> Date: Tue, 2 Apr 2024 22:28:07 +0200 Subject: [PATCH 3/7] Drop Index error fixed --- .../AmIVulnerable/Controllers/DbController.cs | 12 +++---- .../Controllers/GitController.cs | 2 +- code/AmIVulnerable/sql/init.sql | 35 ++++++++++++++++++- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs index 9363f5c..2c7b826 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs @@ -132,7 +132,7 @@ public IActionResult ConvertRawFilesToMySql() { /// /// [HttpGet] - [Route("")] + [Route("Update")] public IActionResult UpdateCveDatabase() { using (Operation.Time("UpdateCveDatabase")) { try { @@ -141,8 +141,8 @@ public IActionResult UpdateCveDatabase() { MySqlCommand cmdTestIfTableExist = new MySqlCommand($"" + $"SELECT COUNT(*) " + - $"FROM information_schema.TABLES" + - $"WHERE (TABLE_SCHEMA = 'cve') AND (TABLE_NAME = 'cve')", connection); + $"FROM information_schema.TABLES " + + $"WHERE (TABLE_SCHEMA = 'cve') AND (TABLE_NAME = 'cve');", connection); connection.Open(); int count = cmdTestIfTableExist.ExecuteNonQuery(); @@ -155,7 +155,7 @@ public IActionResult UpdateCveDatabase() { //start update process try { ProcessStartInfo process = new ProcessStartInfo { - FileName = "cmd", + FileName = "bash", RedirectStandardInput = true, WorkingDirectory = $"", }; @@ -188,7 +188,7 @@ public IActionResult UpdateCveDatabase() { } // Drop Index for faster insert - MySqlCommand cmdIndexDrop = new MySqlCommand("DROP INDEX idx_designation ON cve;", connection); + MySqlCommand cmdIndexDrop = new MySqlCommand("CALL drop_index_on_designation_if_exists();", connection); connection.Open(); cmdIndexDrop.ExecuteNonQuery(); @@ -201,7 +201,7 @@ public IActionResult UpdateCveDatabase() { "VALUES(@cve, @des, @ver, @ful) " + "ON DUPLICATE KEY UPDATE " + "version_affected = @ver" + - "full_text = @ful"; + "full_text = @ful;" ; MySqlCommand cmdInsert = new MySqlCommand(insertIntoString, connection); string json = System.IO.File.ReadAllText(x); diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs index a3dccf7..ca5df6d 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/GitController.cs @@ -59,7 +59,7 @@ public IActionResult CloneRepo([FromHeader] bool cveRaw, [FromBody] Tuple PullAndConvertCveFiles() { try { ProcessStartInfo process = new ProcessStartInfo { - FileName = "cmd", + FileName = "bash", RedirectStandardInput = true, WorkingDirectory = $"", }; diff --git a/code/AmIVulnerable/sql/init.sql b/code/AmIVulnerable/sql/init.sql index ea77ec0..9dcac70 100644 --- a/code/AmIVulnerable/sql/init.sql +++ b/code/AmIVulnerable/sql/init.sql @@ -3,4 +3,37 @@ CREATE TABLE IF NOT EXISTS cve.cve( designation VARCHAR(500) NOT NULL, version_affected TEXT NOT NULL, full_text MEDIUMTEXT NOT NULL -); \ No newline at end of file +); + +/* PROCEDURE for secure index-drop */ +DELIMITER // + +CREATE PROCEDURE drop_index_on_designation_if_exists() +BEGIN + DECLARE index_name VARCHAR(100); + DECLARE table_name VARCHAR(100); + DECLARE CONTINUE HANDLER FOR SQLSTATE '42000' SET @error = 1; + + SET index_name := 'idx_designation'; + SET table_name := 'cve'; + + SET @error = 0; + + SELECT COUNT(*) + INTO @index_exists + FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = table_name AND index_name = index_name; + + IF @index_exists THEN + SET @sql = CONCAT('ALTER TABLE ', table_name, ' DROP INDEX ', index_name, ';'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END IF; + + IF @error = 1 THEN + SELECT 'Index not found, no action taken'; + END IF; +END // + +DELIMITER ; From d944ce32a729f3bc54dddeb9db598799869874ea Mon Sep 17 00:00:00 2001 From: Kretchen001 <83697846+Kretchen001@users.noreply.github.com> Date: Wed, 3 Apr 2024 12:53:27 +0200 Subject: [PATCH 4/7] fix update string syntax --- code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs index 2c7b826..cd3116f 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs @@ -200,7 +200,7 @@ public IActionResult UpdateCveDatabase() { string insertIntoString = "INSERT INTO cve(cve_number, designation, version_affected, full_text) " + "VALUES(@cve, @des, @ver, @ful) " + "ON DUPLICATE KEY UPDATE " + - "version_affected = @ver" + + "version_affected = @ver, " + "full_text = @ful;" ; MySqlCommand cmdInsert = new MySqlCommand(insertIntoString, connection); From 7b021bedf95d534b2f040fc2c59184f557bcd054 Mon Sep 17 00:00:00 2001 From: Kretchen001 <83697846+Kretchen001@users.noreply.github.com> Date: Mon, 8 Apr 2024 13:43:45 +0200 Subject: [PATCH 5/7] Update DbController.cs - empty insert fixed --- code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs index cd3116f..f50ad60 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs @@ -131,7 +131,7 @@ public IActionResult ConvertRawFilesToMySql() { /// /// - [HttpGet] + [HttpPost] [Route("Update")] public IActionResult UpdateCveDatabase() { using (Operation.Time("UpdateCveDatabase")) { @@ -222,6 +222,9 @@ public IActionResult UpdateCveDatabase() { if (product.Length > 500) { product = product[0..500]; } + if (product.Equals("")) { + product = "n/a"; + } } catch { product = "n/a"; From ff51a96e34a997eed108a49060ed0de049db6bbd Mon Sep 17 00:00:00 2001 From: Kretchen001 <83697846+Kretchen001@users.noreply.github.com> Date: Mon, 8 Apr 2024 13:59:06 +0200 Subject: [PATCH 6/7] Fixed comments and application/json --- .../AmIVulnerable/Controllers/DbController.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs index f50ad60..f6215eb 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs @@ -129,7 +129,7 @@ public IActionResult ConvertRawFilesToMySql() { } } - /// + /// Update the Database, if it exists already. /// [HttpPost] [Route("Update")] @@ -149,7 +149,7 @@ public IActionResult UpdateCveDatabase() { connection.Close(); if (count == 0) { - return BadRequest("Table not exist!\nPlease download the cve and create a database before try to update it."); + return BadRequest("Table does not exist!\nPlease download cve data and create the database before trying to update it over the route for that!"); } //start update process @@ -252,7 +252,7 @@ public IActionResult UpdateCveDatabase() { } } - /// + /// Return the full text of a cve, when it is requested. /// /// [HttpGet] @@ -280,7 +280,9 @@ public IActionResult GetFullTextCve([FromHeader] string? cve_number) { return NoContent(); } - return Ok(JsonConvert.SerializeObject(resDataTable.Rows[0]["full_text"].ToString())); + CVEcomp? cVEcomp = JsonConvert.DeserializeObject(resDataTable.Rows[0]["full_text"].ToString()!); + + return Ok(cVEcomp); } catch (Exception ex) { return BadRequest(ex.StackTrace + "\n\n" + ex.Message); From a6806943195e05058a554a839c01b5dc82d13ba0 Mon Sep 17 00:00:00 2001 From: Kretchen001 <83697846+Kretchen001@users.noreply.github.com> Date: Mon, 8 Apr 2024 14:06:41 +0200 Subject: [PATCH 7/7] syntax the third --- code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs index f6215eb..efab323 100644 --- a/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs +++ b/code/AmIVulnerable/AmIVulnerable/Controllers/DbController.cs @@ -149,7 +149,7 @@ public IActionResult UpdateCveDatabase() { connection.Close(); if (count == 0) { - return BadRequest("Table does not exist!\nPlease download cve data and create the database before trying to update it over the route for that!"); + return BadRequest("Table does not exist!\nPlease download cve data and create the database before trying to update it using the update route!"); } //start update process