From 253857fcbfd929a5ec7f788cdfd369c7bb26e991 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 27 Sep 2023 10:52:38 +0300 Subject: [PATCH 01/12] Add `EnumerationHandle`. --- .../SafeHandles/EnumerationHandle.cs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs diff --git a/sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs b/sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs new file mode 100644 index 00000000..91bf5c9f --- /dev/null +++ b/sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs @@ -0,0 +1,55 @@ +using System; +using System.Runtime.InteropServices; +using TileDB.Interop; + +namespace TileDB.CSharp.Marshalling.SafeHandles +{ + internal unsafe sealed class EnumerationHandle : SafeHandle + { + public EnumerationHandle() : base(IntPtr.Zero, true) { } + + public EnumerationHandle(IntPtr handle, bool ownsHandle) : base(IntPtr.Zero, ownsHandle) { SetHandle(handle); } + + public static EnumerationHandle Create(Context context, string name, tiledb_datatype_t datatype, uint cellValNum, bool ordered, byte* data, ulong dataSize, ulong* offsets, ulong offsetsSize) + { + var handle = new EnumerationHandle(); + var successful = false; + tiledb_enumeration_t* enumeration = null; + try + { + using var contextHandle = context.Handle.Acquire(); + using var ms_name = new MarshaledString(name); + context.handle_error(Methods.tiledb_enumeration_alloc(contextHandle, ms_name, datatype, cellValNum, + ordered ? 1 : 0, data, dataSize, offsets, offsetsSize * sizeof(ulong), &enumeration)); + successful = true; + } + finally + { + if (successful) + { + handle.InitHandle(enumeration); + } + else + { + handle.SetHandleAsInvalid(); + } + } + + return handle; + } + + protected override bool ReleaseHandle() + { + fixed (IntPtr* p = &handle) + { + Methods.tiledb_enumeration_free((tiledb_enumeration_t**)p); + } + return true; + } + + internal void InitHandle(tiledb_enumeration_t* h) { SetHandle((IntPtr)h); } + public override bool IsInvalid => handle == IntPtr.Zero; + + public SafeHandleHolder Acquire() => new(this); + } +} From 24a88a35e8275ebe267cf7be003781a5a98e59c4 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 6 Oct 2023 22:08:17 +0300 Subject: [PATCH 02/12] Add an `Enumeration` class. --- sources/TileDB.CSharp/Enumeration.cs | 250 ++++++++++++++++++ .../MarshaledContiguousStringCollection.cs | 67 +++++ .../Marshalling/MarshaledString.cs | 17 ++ .../Marshalling/MarshaledStringCollection.cs | 2 +- sources/TileDB.CSharp/ThrowHelpers.cs | 4 + 5 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 sources/TileDB.CSharp/Enumeration.cs create mode 100644 sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs diff --git a/sources/TileDB.CSharp/Enumeration.cs b/sources/TileDB.CSharp/Enumeration.cs new file mode 100644 index 00000000..cbab3a71 --- /dev/null +++ b/sources/TileDB.CSharp/Enumeration.cs @@ -0,0 +1,250 @@ +using System; +using System.Collections.Generic; +using TileDB.CSharp.Marshalling; +using TileDB.CSharp.Marshalling.SafeHandles; +using TileDB.Interop; + +namespace TileDB.CSharp +{ + /// + /// Represents a TileDB enumeration object. + /// + public sealed unsafe class Enumeration : IDisposable + { + /// + /// This value indicates an enumeration with variable-sized members. + /// It may be returned from . + /// + /// + public const uint VariableSized = Constants.VariableSizedImpl; + + private readonly Context _ctx; + + private readonly EnumerationHandle _handle; + + internal Enumeration(Context ctx, EnumerationHandle handle) + { + _ctx = ctx; + _handle = handle; + } + + internal EnumerationHandle Handle => _handle; + + /// + /// Creates an from a collection of strings. + /// + /// The to use. + /// The name of the enumeration. + /// Whether the enumeration should be considered as ordered. + /// If false, prevents inequality operators in s from + /// being used with this enumeration. + /// The collection of values for the enumeration. + /// The data type of the enumeration. Must be a string type. + /// Optional, defaults to . + public static Enumeration Create(Context ctx, string name, bool ordered, IReadOnlyCollection values, DataType dataType = DataType.StringUtf8) + { + using MarshaledContiguousStringCollection marshaledStrings = new(values, dataType); + var handle = EnumerationHandle.Create(ctx, name, (tiledb_datatype_t)dataType, VariableSized, + ordered, marshaledStrings.Data, marshaledStrings.DataCount, marshaledStrings.Offsets, marshaledStrings.OffsetsCount); + return new Enumeration(ctx, handle); + } + + /// + /// Disposes the . + /// + public void Dispose() + { + _handle.Dispose(); + } + + /// + /// Returns the number of values for each member of the . + /// + /// + /// If this property has a value of , each value has a different size. + /// + public uint ValuesPerMember + { + get + { + uint result; + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + _ctx.handle_error(Methods.tiledb_enumeration_get_cell_val_num(ctxHandle, handle, &result)); + return result; + } + } + + /// + /// The of the . + /// + public DataType DataType + { + get + { + tiledb_datatype_t result; + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + _ctx.handle_error(Methods.tiledb_enumeration_get_type(ctxHandle, handle, &result)); + return (DataType)result; + } + } + + /// + /// Whether the should be considered as ordered. + /// + /// + /// If , prevents inequality operators in + /// s from being used with this enumeration. + /// + public bool IsOrdered + { + get + { + int result; + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + _ctx.handle_error(Methods.tiledb_enumeration_get_ordered(ctxHandle, handle, &result)); + return result != 0; + } + } + + /// + /// Returns the name of the . + /// + public string GetName() + { + using StringHandleHolder nameHolder = new(); + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + _ctx.handle_error(Methods.tiledb_enumeration_get_name(ctxHandle, handle, &nameHolder._handle)); + return nameHolder.ToString(); + } + + /// + /// Returns the values of the . + /// + /// + /// An array whose type depends on the enumeration's + /// and : + /// + /// If is a string datatype, the + /// method will return an array of . + /// If the enumeration's values have a fixed size (i.e. + /// is not equal to ), the method will return an array of + /// values of the underlying data type. + /// If the enumeration's values have a variable size (i.e. + /// is equal to ), the method will return an array of arrays of + /// values of the underlying data type. + /// + /// + public System.Array GetValues() + { + // Keep the handles acquired for the entire method. + // This prevents another thread freeing the data and offsets buffers. + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + + void* dataPtr; + ulong* offsetsPtr; + ulong dataSize, offsetsSize; + _ctx.handle_error(Methods.tiledb_enumeration_get_data(ctxHandle, handle, &dataPtr, &dataSize)); + _ctx.handle_error(Methods.tiledb_enumeration_get_offsets(ctxHandle, handle, (void**)&offsetsPtr, &offsetsSize)); + + DataType datatype; + _ctx.handle_error(Methods.tiledb_enumeration_get_type(ctxHandle, handle, (tiledb_datatype_t*)&datatype)); + + if (EnumUtil.IsStringType(datatype)) + { + ReadOnlySpan offsets = new(offsetsPtr, checked((int)(offsetsSize / sizeof(ulong)))); + string[] result = new string[offsets.Length]; + + for (int i = 0; i < offsets.Length; i++) + { + ulong nextOffset = i == offsets.Length - 1 ? dataSize : offsets[i + 1]; + ReadOnlySpan bytes = new((byte*)dataPtr + offsets[i], checked((int)(nextOffset - offsets[i]))); + result[i] = MarshaledStringOut.GetString(bytes, datatype); + } + + return result; + } + + Type type = EnumUtil.DataTypeToType(datatype); + if (type == typeof(sbyte)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(byte)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(short)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(ushort)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(int)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(uint)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(long)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(ulong)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(float)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + if (type == typeof(double)) + { + return GetValues(dataPtr, dataSize, offsetsPtr, offsetsSize); + } + throw new NotSupportedException($"Unsupported enumeration type: {type}. Please open a bug report."); + + System.Array GetValues(void* dataPtr, ulong dataSize, ulong* offsetsPtr, ulong offsetsSize) + { + uint cellValNum; + _ctx.handle_error(Methods.tiledb_enumeration_get_cell_val_num(ctxHandle, handle, &cellValNum)); + if (cellValNum != VariableSized) + { + return new ReadOnlySpan(dataPtr, checked((int)(dataSize / (ulong)sizeof(T)))).ToArray(); + } + + T[][] result; + if (cellValNum != VariableSized) + { + result = new T[checked((int)offsetsSize)][]; + int i; + ulong j; + for (i = 0, j = 0; i < result.Length; i++, j += cellValNum) + { + result[i] = new ReadOnlySpan((T*)dataPtr + j, checked((int)cellValNum)).ToArray(); + } + + return result; + } + + ReadOnlySpan offsets = new(offsetsPtr, checked((int)(offsetsSize / sizeof(ulong)))); + result = new T[offsets.Length][]; + for (int i = 0; i < offsets.Length; i++) + { + ulong nextOffset = i == offsets.Length - 1 ? dataSize : offsets[i + 1]; + result[i] = new ReadOnlySpan((byte*)dataPtr + offsets[i], checked((int)((nextOffset - offsets[i]) / (ulong)sizeof(T)))).ToArray(); + } + + return result; + } + } + } +} diff --git a/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs b/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs new file mode 100644 index 00000000..bde94b19 --- /dev/null +++ b/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; +using TileDB.Interop; + +namespace TileDB.CSharp.Marshalling +{ + internal unsafe ref struct MarshaledContiguousStringCollection + { + private int _dataCount, _offsetsCount; + + public byte* Data { get; private set; } + + public ulong* Offsets { get; private set; } + + public ulong DataCount { get; private set; } + + public ulong OffsetsCount { get; private set; } + + public MarshaledContiguousStringCollection(IReadOnlyCollection strings, DataType dataType = DataType.StringAscii) + { + Encoding encoding = MarshaledString.GetEncoding(dataType); + try + { + Offsets = (ulong*)Marshal.AllocHGlobal(strings.Count * sizeof(ulong)); + OffsetsCount = (ulong)strings.Count * sizeof(ulong); + ulong currentOffset = 0; + DataCount = 0; + foreach (var str in strings) + { + Offsets[currentOffset++] = DataCount; + DataCount += (ulong)encoding.GetByteCount(str); + } + + Data = (byte*)Marshal.AllocHGlobal((IntPtr)DataCount); + currentOffset = 0; + foreach (var str in strings) + { + encoding.GetBytes(str, new Span(Data + currentOffset, checked((int)(DataCount - currentOffset)))); + currentOffset += Offsets[currentOffset]; + } + } + catch + { + Dispose(); + throw; + } + } + + public void Dispose() + { + if (Data is not null) + { + Marshal.FreeHGlobal((IntPtr)Data); + Data = null; + DataCount = 0; + } + if (Offsets is not null) + { + Marshal.FreeHGlobal((IntPtr)Offsets); + Offsets = null; + OffsetsCount = 0; + } + } + } +} diff --git a/sources/TileDB.CSharp/Marshalling/MarshaledString.cs b/sources/TileDB.CSharp/Marshalling/MarshaledString.cs index 428d437e..549df30a 100644 --- a/sources/TileDB.CSharp/Marshalling/MarshaledString.cs +++ b/sources/TileDB.CSharp/Marshalling/MarshaledString.cs @@ -19,6 +19,23 @@ public MarshaledString(string input) Value = (sbyte*)ptr; } + internal static Encoding GetEncoding(DataType dataType) + { + switch (dataType) + { + case DataType.StringAscii: + return Encoding.ASCII; + case DataType.StringUtf8: + return Encoding.UTF8; + case DataType.StringUtf16: + return Encoding.Unicode; + case DataType.StringUtf32: + return Encoding.UTF32; + } + ThrowHelpers.ThrowInvalidDataType(dataType); + return null; + } + internal static (IntPtr Pointer, int Length) AllocNullTerminated(string str) { if (str is null) diff --git a/sources/TileDB.CSharp/Marshalling/MarshaledStringCollection.cs b/sources/TileDB.CSharp/Marshalling/MarshaledStringCollection.cs index 56e1b9ec..e140de3f 100644 --- a/sources/TileDB.CSharp/Marshalling/MarshaledStringCollection.cs +++ b/sources/TileDB.CSharp/Marshalling/MarshaledStringCollection.cs @@ -5,7 +5,7 @@ namespace TileDB.CSharp.Marshalling { - internal unsafe struct MarshaledStringCollection : IDisposable + internal unsafe ref struct MarshaledStringCollection { public IntPtr* Strings { get; private set; } diff --git a/sources/TileDB.CSharp/ThrowHelpers.cs b/sources/TileDB.CSharp/ThrowHelpers.cs index bdfe9e96..d85ca685 100644 --- a/sources/TileDB.CSharp/ThrowHelpers.cs +++ b/sources/TileDB.CSharp/ThrowHelpers.cs @@ -10,6 +10,10 @@ internal static class ThrowHelpers public static void ThrowArgumentNull(string? paramName) => throw new ArgumentNullException(paramName); + [DoesNotReturn] + public static void ThrowInvalidDataType(DataType dataType, [CallerArgumentExpression(nameof(dataType))] string? paramName = null) => + throw new ArgumentOutOfRangeException(nameof(dataType), dataType, "Invalid data type."); + [DoesNotReturn] public static void ThrowTypeNotSupported() => throw new NotSupportedException("Type is not supported."); From 0d46e944e5255d81521a8cb71d87ee988b54c041 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 2 Oct 2023 19:36:45 +0300 Subject: [PATCH 03/12] Add the other enumeration-related APIs. --- sources/TileDB.CSharp/Array.cs | 48 +++++++++++++++++++ sources/TileDB.CSharp/ArraySchema.cs | 12 +++++ sources/TileDB.CSharp/ArraySchemaEvolution.cs | 26 ++++++++++ sources/TileDB.CSharp/Attribute.cs | 29 +++++++++++ sources/TileDB.CSharp/QueryCondition.cs | 27 +++++++++++ 5 files changed, 142 insertions(+) diff --git a/sources/TileDB.CSharp/Array.cs b/sources/TileDB.CSharp/Array.cs index 7c04c1a4..2f46ac04 100644 --- a/sources/TileDB.CSharp/Array.cs +++ b/sources/TileDB.CSharp/Array.cs @@ -656,6 +656,54 @@ public static EncryptionType EncryptionType(Context ctx, string uri) return (EncryptionType)tiledb_encryption_type; } + /// + /// Gets an from the by name. + /// + /// The enumeration's name. + /// + public Enumeration GetEnumeration(string name) + { + var handle = new EnumerationHandle(); + var successful = false; + tiledb_enumeration_t* enumeration_p = null; + try + { + using (var ctxHandle = _ctx.Handle.Acquire()) + using (var arrayHandle = _handle.Acquire()) + using (var ms_name = new MarshaledString(name)) + { + _ctx.handle_error(Methods.tiledb_array_get_enumeration(ctxHandle, arrayHandle, ms_name, &enumeration_p)); + } + successful = true; + } + finally + { + if (successful) + { + handle.InitHandle(enumeration_p); + } + else + { + handle.SetHandleAsInvalid(); + } + } + + return new Enumeration(_ctx, handle); + } + + /// + /// Loads all enumerations of the . + /// + /// + public void LoadAllEnumerations() + { + using (var ctxHandle = _ctx.Handle.Acquire()) + using (var arrayHandle = _handle.Acquire()) + { + _ctx.handle_error(Methods.tiledb_array_load_all_enumerations(ctxHandle, arrayHandle)); + } + } + /// /// Puts a multi-value metadata to the . /// diff --git a/sources/TileDB.CSharp/ArraySchema.cs b/sources/TileDB.CSharp/ArraySchema.cs index 376d9a60..9d33c061 100644 --- a/sources/TileDB.CSharp/ArraySchema.cs +++ b/sources/TileDB.CSharp/ArraySchema.cs @@ -69,6 +69,18 @@ public void AddAttributes(params Attribute[] attrs) } } + /// + /// Adds an in the . + /// + /// The enumeration to add. + public void AddEnumeration(Enumeration enumeration) + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + using var enumHandle = enumeration.Handle.Acquire(); + _ctx.handle_error(Methods.tiledb_array_schema_add_enumeration(ctxHandle, handle, enumHandle)); + } + /// /// Sets whether cells with duplicate coordinates are allowed in the . /// diff --git a/sources/TileDB.CSharp/ArraySchemaEvolution.cs b/sources/TileDB.CSharp/ArraySchemaEvolution.cs index 76040185..f66b1375 100644 --- a/sources/TileDB.CSharp/ArraySchemaEvolution.cs +++ b/sources/TileDB.CSharp/ArraySchemaEvolution.cs @@ -63,6 +63,32 @@ public void DropAttribute(string attrName) _ctx.handle_error(Methods.tiledb_array_schema_evolution_drop_attribute(ctxHandle, handle, msAttrName)); } + /// + /// Adds an to the . + /// + /// A fully constructed that will be added to the schema. + /// + public void AddEnumeration(Enumeration enumeration) + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + using var enumHandle = enumeration.Handle.Acquire(); + _ctx.handle_error(Methods.tiledb_array_schema_evolution_add_enumeration(ctxHandle, handle, enumHandle)); + } + + /// + /// Drops an enumeration with the given name from the . + /// + /// The name of the attribute that will be dropped from the schema. + /// + public void DropEnumeration(string enumerationName) + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + using var msEnumerationName = new MarshaledString(enumerationName); + _ctx.handle_error(Methods.tiledb_array_schema_evolution_drop_enumeration(ctxHandle, handle, msEnumerationName)); + } + /// /// Drops an from the . /// diff --git a/sources/TileDB.CSharp/Attribute.cs b/sources/TileDB.CSharp/Attribute.cs index 98f478f6..e3c98dd6 100644 --- a/sources/TileDB.CSharp/Attribute.cs +++ b/sources/TileDB.CSharp/Attribute.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using System.Text; using TileDB.Interop; +using TileDB.CSharp.Marshalling; using TileDB.CSharp.Marshalling.SafeHandles; using AttributeHandle = TileDB.CSharp.Marshalling.SafeHandles.AttributeHandle; using FilterListHandle = TileDB.CSharp.Marshalling.SafeHandles.FilterListHandle; @@ -89,6 +90,17 @@ public void SetCellValNum(uint cellValNum) using var handle = _handle.Acquire(); _ctx.handle_error(Methods.tiledb_attribute_set_cell_val_num(ctxHandle, handle, cellValNum)); } + + /// + /// Sets the name of the 's enumeration. + /// + public void SetEnumerationName(string enumerationName) + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + using var ms_enumerationName = new MarshaledString(enumerationName); + _ctx.handle_error(Methods.tiledb_attribute_set_enumeration_name(ctxHandle, handle, ms_enumerationName)); + } /// /// Get name of the attribute. @@ -187,6 +199,23 @@ public ulong CellSize() return cell_size; } + /// + /// Gets the name of the 's enumeration. + /// + /// + /// A string with the name of the attribute's enumeration, or an + /// empty string if the attribute does not have an enumeration. + /// + public string EnumerationName() + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + using var nameHolder = new StringHandleHolder(); + _ctx.handle_error(Methods.tiledb_attribute_get_enumeration_name(ctxHandle, handle, &nameHolder._handle)); + + return nameHolder.ToString(); + } + /// /// Sets the fill value of a nullable multi-valued . /// Used when the fill value is multi-valued. diff --git a/sources/TileDB.CSharp/QueryCondition.cs b/sources/TileDB.CSharp/QueryCondition.cs index f12cc4c0..11250c52 100644 --- a/sources/TileDB.CSharp/QueryCondition.cs +++ b/sources/TileDB.CSharp/QueryCondition.cs @@ -86,6 +86,13 @@ private void Init(string attribute_name, void* condition_value, ulong condition_ _ctx.handle_error(Methods.tiledb_query_condition_init(ctxHandle, handle, ms_attribute_name, condition_value, condition_value_size, (tiledb_query_condition_op_t)optype)); } + private void SetUseEnumeration(bool value) + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + _ctx.handle_error(Methods.tiledb_query_condition_set_use_enumeration(ctxHandle, handle, value ? 1 : 0)); + } + private static QueryCondition Combine(QueryCondition lhs, QueryCondition? rhs, QueryConditionCombinationOperatorType combination_optype) { var ctx = lhs._ctx; @@ -147,9 +154,29 @@ public static QueryCondition Create(Context ctx, string attribute_name, string v /// The type of the relationship between the attribute with /// the name and . public static QueryCondition Create(Context ctx, string attribute_name, T value, QueryConditionOperatorType optype) where T : struct + { + return Create(ctx, attribute_name, value, optype, true); + } + + /// + /// Creates a new query condition with datatype . + /// + /// The associated with the query condition. + /// The name of the attribute the query condition refers to. + /// The value to compare the attribute with. + /// The type of the relationship between the attribute with + /// the name and . + /// Whether to match the query condition on the + /// enumerated value instead of the underlying value. Optional, defaults to + /// . + public static QueryCondition Create(Context ctx, string attribute_name, T value, QueryConditionOperatorType optype, bool useEnumeration = true) where T : struct { var ret = new QueryCondition(ctx); ret.Init(attribute_name, value, optype); + if (!useEnumeration) + { + ret.SetUseEnumeration(false); + } return ret; } From a90b01ee8980dfaf939d00309575a5479b7df62b Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 2 Oct 2023 19:40:08 +0300 Subject: [PATCH 04/12] Suppress Sonar warning S3427. --- .editorconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.editorconfig b/.editorconfig index 7243a02e..6670bfec 100644 --- a/.editorconfig +++ b/.editorconfig @@ -43,3 +43,8 @@ dotnet_diagnostic.S112.severity = None # Native methods should be wrapped # For some reason this warning shows even on auto-generated files. dotnet_diagnostic.S4200.severity = None + +# S3427: Method overloads with default parameter values should not overlap +# If we oblige to this rule and remove the overload without the default parameter, +# the package validator will complain. +dotnet_diagnostic.S3427.severity = none From 001bd269d0032077ff55532ab4448e0835c7c7d7 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 3 Oct 2023 12:50:25 +0300 Subject: [PATCH 05/12] FIx bugs. --- .../Marshalling/MarshaledContiguousStringCollection.cs | 5 +++-- .../Marshalling/SafeHandles/EnumerationHandle.cs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs b/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs index bde94b19..1c2c069f 100644 --- a/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs +++ b/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs @@ -34,11 +34,12 @@ public MarshaledContiguousStringCollection(IReadOnlyCollection strings, } Data = (byte*)Marshal.AllocHGlobal((IntPtr)DataCount); - currentOffset = 0; + int i = 0; foreach (var str in strings) { + currentOffset = Offsets[i]; encoding.GetBytes(str, new Span(Data + currentOffset, checked((int)(DataCount - currentOffset)))); - currentOffset += Offsets[currentOffset]; + i++; } } catch diff --git a/sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs b/sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs index 91bf5c9f..6fdd438b 100644 --- a/sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs +++ b/sources/TileDB.CSharp/Marshalling/SafeHandles/EnumerationHandle.cs @@ -20,7 +20,7 @@ public static EnumerationHandle Create(Context context, string name, tiledb_data using var contextHandle = context.Handle.Acquire(); using var ms_name = new MarshaledString(name); context.handle_error(Methods.tiledb_enumeration_alloc(contextHandle, ms_name, datatype, cellValNum, - ordered ? 1 : 0, data, dataSize, offsets, offsetsSize * sizeof(ulong), &enumeration)); + ordered ? 1 : 0, data, dataSize, offsets, offsetsSize, &enumeration)); successful = true; } finally From 5f35f62148738c170aa5a7801e1bd507770ea879 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 6 Oct 2023 22:19:38 +0300 Subject: [PATCH 06/12] Add some more APIs. --- sources/TileDB.CSharp/Enumeration.cs | 94 ++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/sources/TileDB.CSharp/Enumeration.cs b/sources/TileDB.CSharp/Enumeration.cs index cbab3a71..7199da78 100644 --- a/sources/TileDB.CSharp/Enumeration.cs +++ b/sources/TileDB.CSharp/Enumeration.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Runtime.InteropServices; using TileDB.CSharp.Marshalling; using TileDB.CSharp.Marshalling.SafeHandles; using TileDB.Interop; @@ -30,6 +31,65 @@ internal Enumeration(Context ctx, EnumerationHandle handle) internal EnumerationHandle Handle => _handle; + /// + /// Creates an with fixed-sized members. + /// + /// The type of the enumeration's members. + /// The to use. + /// The name of the enumeration. + /// The value of . + /// A of the enumeration's values. + /// The number of values per member in the enumeration. Optional, + /// defaults to 1. If specified, its value must divide the length of . + /// The data type of the enumeration. Defaults to the + /// default data type of . + /// has a different + /// size from . + public static Enumeration Create(Context ctx, string name, bool ordered, ReadOnlySpan values, uint cellValNum = 1, DataType? dataType = null) where T : struct + { + DataType dataTypeActual = dataType ??= EnumUtil.TypeToDataType(typeof(T)); + if (EnumUtil.DataTypeToType(dataTypeActual) != typeof(T)) + { + ThrowHelpers.ThrowTypeMismatch(dataTypeActual); + } + + fixed (T* valuesPtr = &MemoryMarshal.GetReference(values)) + { + EnumerationHandle handle = EnumerationHandle.Create(ctx, name, (tiledb_datatype_t)dataTypeActual, cellValNum, ordered, (byte*)valuesPtr, (ulong)values.Length * (ulong)sizeof(T), null, 0); + return new Enumeration(ctx, handle); + } + } + + /// + /// Creates an with variable-sized members. + /// + /// The type of the enumeration's members. + /// The to use. + /// The name of the enumeration. + /// The value of . + /// A of all values of each member of the enumeration, concatenated. + /// A of the offsets to the first byte of each member of the enumeration. + /// The data type of the enumeration. Defaults to the + /// default data type of . + /// has a different + /// size from . + public static Enumeration Create(Context ctx, string name, bool ordered, ReadOnlySpan values, ReadOnlySpan offsets, DataType? dataType = null) where T : struct + { + DataType dataTypeActual = dataType ??= EnumUtil.TypeToDataType(typeof(T)); + if (EnumUtil.DataTypeToType(dataTypeActual) != typeof(T)) + { + ThrowHelpers.ThrowTypeMismatch(dataTypeActual); + } + + EnumerationHandle handle; + fixed (T* valuesPtr = &MemoryMarshal.GetReference(values)) + fixed (ulong* offsetsPtr = &MemoryMarshal.GetReference(offsets)) + { + handle = EnumerationHandle.Create(ctx, name, (tiledb_datatype_t)dataTypeActual, VariableSized, ordered, (byte*)valuesPtr, (ulong)values.Length * (ulong)sizeof(T), offsetsPtr, (ulong)offsets.Length * sizeof(ulong)); + } + return new Enumeration(ctx, handle); + } + /// /// Creates an from a collection of strings. /// @@ -246,5 +306,39 @@ System.Array GetValues(void* dataPtr, ulong dataSize, ulong* offsetsPtr, ulon return result; } } + + /// + /// Returns a byte array with the raw data of the 's members. + /// + /// + /// + public byte[] GetRawData() + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + void* data; + ulong dataSize; + _ctx.handle_error(Methods.tiledb_enumeration_get_data(ctxHandle, handle, &data, &dataSize)); + return new ReadOnlySpan(data, checked((int)dataSize)).ToArray(); + } + + /// + /// Returns an array with the offsets to the first byte of each member of the . + /// + /// + /// In enumerations with fixed-size members (i.e. is not equal + /// to , this method will return an empty array, as each member has the same size. + /// + /// + /// + public ulong[] GetRawOffsets() + { + using var ctxHandle = _ctx.Handle.Acquire(); + using var handle = _handle.Acquire(); + void* data; + ulong dataSize; + _ctx.handle_error(Methods.tiledb_enumeration_get_offsets(ctxHandle, handle, &data, &dataSize)); + return new ReadOnlySpan(data, checked((int)(dataSize / sizeof(ulong)))).ToArray(); + } } } From aec03fa305309408d0d83b360a53490f7d22a0c8 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 6 Oct 2023 23:30:44 +0300 Subject: [PATCH 07/12] Always return a flat array on enumerations with fixed-size members. --- sources/TileDB.CSharp/Enumeration.cs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/sources/TileDB.CSharp/Enumeration.cs b/sources/TileDB.CSharp/Enumeration.cs index 7199da78..d54a96a4 100644 --- a/sources/TileDB.CSharp/Enumeration.cs +++ b/sources/TileDB.CSharp/Enumeration.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Runtime.InteropServices; using TileDB.CSharp.Marshalling; @@ -281,22 +281,8 @@ System.Array GetValues(void* dataPtr, ulong dataSize, ulong* offsetsPtr, ulon return new ReadOnlySpan(dataPtr, checked((int)(dataSize / (ulong)sizeof(T)))).ToArray(); } - T[][] result; - if (cellValNum != VariableSized) - { - result = new T[checked((int)offsetsSize)][]; - int i; - ulong j; - for (i = 0, j = 0; i < result.Length; i++, j += cellValNum) - { - result[i] = new ReadOnlySpan((T*)dataPtr + j, checked((int)cellValNum)).ToArray(); - } - - return result; - } - ReadOnlySpan offsets = new(offsetsPtr, checked((int)(offsetsSize / sizeof(ulong)))); - result = new T[offsets.Length][]; + T[][] result = new T[offsets.Length][]; for (int i = 0; i < offsets.Length; i++) { ulong nextOffset = i == offsets.Length - 1 ? dataSize : offsets[i + 1]; From 9dc9c5b2d5e6c7d93b1c8e706f0380e450c75e45 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Sat, 7 Oct 2023 00:22:19 +0300 Subject: [PATCH 08/12] Add some tests. --- tests/TileDB.CSharp.Test/EnumerationTest.cs | 72 +++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/TileDB.CSharp.Test/EnumerationTest.cs diff --git a/tests/TileDB.CSharp.Test/EnumerationTest.cs b/tests/TileDB.CSharp.Test/EnumerationTest.cs new file mode 100644 index 00000000..4160395f --- /dev/null +++ b/tests/TileDB.CSharp.Test/EnumerationTest.cs @@ -0,0 +1,72 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; + +namespace TileDB.CSharp.Test +{ + [TestClass] + public class EnumerationTest + { + [TestMethod] + public void TestString() + { + string[] expectedValues = new[] { "Hello", "World", "👋" }; + using Enumeration e = Enumeration.Create(Context.GetDefault(), "test_string", true, expectedValues); + + System.Array values = e.GetValues(); + byte[] rawData = e.GetRawData(); + ulong[] rawOffsets = e.GetRawOffsets(); + + Assert.AreEqual("test_string", e.GetName()); + Assert.IsTrue(e.IsOrdered); + Assert.AreEqual(DataType.StringUtf8, e.DataType); + Assert.AreEqual(Enumeration.VariableSized, e.ValuesPerMember); + CollectionAssert.AreEqual(expectedValues, values); + CollectionAssert.AreEqual("HelloWorld👋"u8.ToArray(), rawData); + CollectionAssert.AreEqual(new ulong[] { 0, 5, 10 }, rawOffsets); + } + + [TestMethod] + [DataRow(1u)] + [DataRow(2u)] + [DataRow(4u)] + public void TestFixedSize(uint cellValNum) + { + int[] expectedValues = new[] { 1, 2, 3, 4 }; + + using Enumeration e = Enumeration.Create(Context.GetDefault(), "test_fixed", false, expectedValues, cellValNum); + + System.Array values = e.GetValues(); + byte[] rawData = e.GetRawData(); + ulong[] rawOffsets = e.GetRawOffsets(); + + Assert.AreEqual("test_fixed", e.GetName()); + Assert.IsFalse(e.IsOrdered); + Assert.AreEqual(DataType.Int32, e.DataType); + Assert.AreEqual(cellValNum, e.ValuesPerMember); + CollectionAssert.AreEqual(expectedValues, values); + CollectionAssert.AreEqual(new byte[] { 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0 }, rawData); + Assert.AreEqual(0, rawOffsets.Length); + } + + [TestMethod] + public void TestVarSize() + { + int[] expectedValues = new int[] { 1, 2, 2, 3, 3, 3, 4, 4, 4, 4 }; + ulong[] expectedOffsets = new ulong[] { 0, 4, 12, 24, 40 }; + + using Enumeration e = Enumeration.Create(Context.GetDefault(), "test_var", false, expectedValues, expectedOffsets); + + System.Array values = e.GetValues(); + byte[] rawData = e.GetRawData(); + ulong[] rawOffsets = e.GetRawOffsets(); + + Assert.AreEqual("test_var", e.GetName()); + Assert.IsFalse(e.IsOrdered); + Assert.AreEqual(DataType.Int32, e.DataType); + Assert.AreEqual(Enumeration.VariableSized, e.ValuesPerMember); + CollectionAssert.AreEqual(expectedValues, values); + CollectionAssert.AreEqual(new byte[] { 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0 }, rawData); + Assert.AreEqual(0, rawOffsets.Length); + } + } +} From f23b0ba7eeeac9e573810409c6d9ca22c8b1be11 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 10 Oct 2023 11:24:25 +0300 Subject: [PATCH 09/12] Fix tests. --- tests/TileDB.CSharp.Test/EnumerationTest.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/TileDB.CSharp.Test/EnumerationTest.cs b/tests/TileDB.CSharp.Test/EnumerationTest.cs index 4160395f..d57e4a95 100644 --- a/tests/TileDB.CSharp.Test/EnumerationTest.cs +++ b/tests/TileDB.CSharp.Test/EnumerationTest.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Runtime.InteropServices; namespace TileDB.CSharp.Test { @@ -52,11 +53,11 @@ public void TestFixedSize(uint cellValNum) public void TestVarSize() { int[] expectedValues = new int[] { 1, 2, 2, 3, 3, 3, 4, 4, 4, 4 }; - ulong[] expectedOffsets = new ulong[] { 0, 4, 12, 24, 40 }; + ulong[] expectedOffsets = new ulong[] { 0, 4, 12, 24 }; using Enumeration e = Enumeration.Create(Context.GetDefault(), "test_var", false, expectedValues, expectedOffsets); - System.Array values = e.GetValues(); + int[][]? values = e.GetValues() as int[][]; byte[] rawData = e.GetRawData(); ulong[] rawOffsets = e.GetRawOffsets(); @@ -64,9 +65,13 @@ public void TestVarSize() Assert.IsFalse(e.IsOrdered); Assert.AreEqual(DataType.Int32, e.DataType); Assert.AreEqual(Enumeration.VariableSized, e.ValuesPerMember); - CollectionAssert.AreEqual(expectedValues, values); - CollectionAssert.AreEqual(new byte[] { 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0 }, rawData); - Assert.AreEqual(0, rawOffsets.Length); + Assert.IsNotNull(values); + CollectionAssert.AreEqual(new int[] { 1 }, values[0]); + CollectionAssert.AreEqual(new int[] { 2, 2 }, values[1]); + CollectionAssert.AreEqual(new int[] { 3, 3, 3 }, values[2]); + CollectionAssert.AreEqual(new int[] { 4, 4, 4, 4 }, values[3]); + CollectionAssert.AreEqual(MemoryMarshal.AsBytes(expectedValues.AsSpan()).ToArray(), rawData); + Assert.AreEqual(4, rawOffsets.Length); } } } From c1dc425f550ea8387c4ae344314825cfe6fa15d7 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 1 Nov 2023 15:10:26 +0200 Subject: [PATCH 10/12] Remove two unused fields. --- .../Marshalling/MarshaledContiguousStringCollection.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs b/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs index 1c2c069f..c69f4faf 100644 --- a/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs +++ b/sources/TileDB.CSharp/Marshalling/MarshaledContiguousStringCollection.cs @@ -8,8 +8,6 @@ namespace TileDB.CSharp.Marshalling { internal unsafe ref struct MarshaledContiguousStringCollection { - private int _dataCount, _offsetsCount; - public byte* Data { get; private set; } public ulong* Offsets { get; private set; } From 10e5b2c6c05320f8011e08d30f88f104a2e112a1 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 1 Nov 2023 15:41:57 +0200 Subject: [PATCH 11/12] Improve documentation of `ValuesPerMember`. --- sources/TileDB.CSharp/Enumeration.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sources/TileDB.CSharp/Enumeration.cs b/sources/TileDB.CSharp/Enumeration.cs index d54a96a4..3a1159df 100644 --- a/sources/TileDB.CSharp/Enumeration.cs +++ b/sources/TileDB.CSharp/Enumeration.cs @@ -121,7 +121,9 @@ public void Dispose() /// Returns the number of values for each member of the . /// /// - /// If this property has a value of , each value has a different size. + /// If this property has a value of , + /// each value has a different size. + /// This method exposes the tiledb_enumeration_get_cell_val_num function of the TileDB Embedded C API. /// public uint ValuesPerMember { From 457cb39d323db96f73ffa6b2a24526f2b776c982 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 1 Nov 2023 15:59:25 +0200 Subject: [PATCH 12/12] Use the unified type validation API. --- sources/TileDB.CSharp/Enumeration.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/sources/TileDB.CSharp/Enumeration.cs b/sources/TileDB.CSharp/Enumeration.cs index 3a1159df..aa90fe38 100644 --- a/sources/TileDB.CSharp/Enumeration.cs +++ b/sources/TileDB.CSharp/Enumeration.cs @@ -48,10 +48,7 @@ internal Enumeration(Context ctx, EnumerationHandle handle) public static Enumeration Create(Context ctx, string name, bool ordered, ReadOnlySpan values, uint cellValNum = 1, DataType? dataType = null) where T : struct { DataType dataTypeActual = dataType ??= EnumUtil.TypeToDataType(typeof(T)); - if (EnumUtil.DataTypeToType(dataTypeActual) != typeof(T)) - { - ThrowHelpers.ThrowTypeMismatch(dataTypeActual); - } + ErrorHandling.CheckDataType(dataTypeActual); fixed (T* valuesPtr = &MemoryMarshal.GetReference(values)) { @@ -76,10 +73,7 @@ public static Enumeration Create(Context ctx, string name, bool ordered, Read public static Enumeration Create(Context ctx, string name, bool ordered, ReadOnlySpan values, ReadOnlySpan offsets, DataType? dataType = null) where T : struct { DataType dataTypeActual = dataType ??= EnumUtil.TypeToDataType(typeof(T)); - if (EnumUtil.DataTypeToType(dataTypeActual) != typeof(T)) - { - ThrowHelpers.ThrowTypeMismatch(dataTypeActual); - } + ErrorHandling.CheckDataType(dataTypeActual); EnumerationHandle handle; fixed (T* valuesPtr = &MemoryMarshal.GetReference(values))