Skip to content

Commit

Permalink
Multiple enchancements / fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Cinchoo
Cinchoo committed Sep 9, 2023
1 parent 1a93c78 commit 49e26c7
Show file tree
Hide file tree
Showing 38 changed files with 609 additions and 158 deletions.
Binary file modified src/Assets/Db/Northwind.MDF
View file
Binary file not shown.
Binary file modified src/Assets/Db/Northwind_log.ldf
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion src/ChoETL.Avro/ChoAvroReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ private IDataReader AsDataReader(Action<IDictionary<string, Type>> membersDiscov
return this.Select(s =>
{
if (s is IDictionary<string, object>)
return ((IDictionary<string, object>)s).Flatten(Configuration.NestedColumnSeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
return ((IDictionary<string, object>)s).Flatten(Configuration.NestedKeySeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix) as object;
else
Expand Down
2 changes: 1 addition & 1 deletion src/ChoETL.Avro/ChoAvroRecordConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ internal ChoAvroRecordConfiguration(Type recordType) : base(recordType)
{
Init(recordType);
}
NestedColumnSeparator = ChoETLSettings.NestedKeySeparator;
NestedKeySeparator = ChoETLSettings.NestedKeySeparator;
UseAvroSerializer = true;
SyncNumberOfObjects = 24;

Expand Down
2 changes: 1 addition & 1 deletion src/ChoETL.Avro/ChoETL.Avro.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<PackageTags>ETL c# .NET CSV Xml JSON Flat Avro</PackageTags>
<PackageReleaseNotes>.NET Neutral version</PackageReleaseNotes>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<Version>1.0.0.16</Version>
<Version>1.0.0.17</Version>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>ChoETL.Avro.snk</AssemblyOriginatorKeyFile>
<PackageIcon>cinchoo.png</PackageIcon>
Expand Down
2 changes: 1 addition & 1 deletion src/ChoETL.JSON.NETStandard/ChoETL.JSON.NETStandard.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<Authors>Cinchoo</Authors>
<Company>Cinchoo Inc.</Company>
<Product>ChoETL.JSON.NETStandard</Product>
<Version>1.2.1.62</Version>
<Version>1.2.1.63</Version>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>ChoETL.JSON.NETStandard.snk</AssemblyOriginatorKeyFile>
<PackageIcon>cinchoo.png</PackageIcon>
Expand Down
4 changes: 2 additions & 2 deletions src/ChoETL.JSON/ChoJSONReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -366,11 +366,11 @@ private IDataReader AsDataReader(Action<IDictionary<string, Type>> membersDiscov
{
IDictionary<string, object> dict = null;
if (s is IDictionary<string, object>)
dict = ((IDictionary<string, object>)s).Flatten(Configuration.NestedColumnSeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedColumnSeparator,
dict = ((IDictionary<string, object>)s).Flatten(Configuration.NestedKeySeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix);
else
dict = s.ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix).Flatten(Configuration.NestedColumnSeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedColumnSeparator,
dict = s.ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix).Flatten(Configuration.NestedKeySeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix);

Expand Down
2 changes: 1 addition & 1 deletion src/ChoETL.JSON/ChoJSONRecordWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ public override IEnumerable<object> WriteTo(object writer, IEnumerable<object> r
recEnum = GetRecords(recEnum).Select(r => r.ConvertToFlattenObject(Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix)).GetEnumerator();
else
recEnum = GetRecords(recEnum).Select(r => r.ToDynamicObject().ConvertToFlattenObject(Configuration.NestedColumnSeparator,
recEnum = GetRecords(recEnum).Select(r => r.ToDynamicObject().ConvertToFlattenObject(Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix)).GetEnumerator();
}

Expand Down
4 changes: 2 additions & 2 deletions src/ChoETL.JSON/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,6 @@
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.2.1.62")]
[assembly: AssemblyFileVersion("1.2.1.62")]
[assembly: AssemblyVersion("1.2.1.63")]
[assembly: AssemblyFileVersion("1.2.1.63")]
//[assembly: ChoAssemblyBetaVersion("beta3")]
2 changes: 1 addition & 1 deletion src/ChoETL.NETStandard/ChoETL.NETStandard.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<PackageTags>ETL c# .NET CSV Xml JSON Flat</PackageTags>
<PackageReleaseNotes>.NET Standard version</PackageReleaseNotes>
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
<Version>1.2.1.62</Version>
<Version>1.2.1.63</Version>
<AssemblyName>ChoETL.Core</AssemblyName>
<RootNamespace>ChoETL</RootNamespace>
<SignAssembly>true</SignAssembly>
Expand Down
2 changes: 1 addition & 1 deletion src/ChoETL.Parquet/ChoETL.Parquet.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Version>1.0.1.28</Version>
<Version>1.0.1.29</Version>
<Authors>Cinchoo</Authors>
<Company>Cinchoo Inc.</Company>
<Description>Parquet extension to Cinchoo ETL framework</Description>
Expand Down
8 changes: 4 additions & 4 deletions src/ChoETL.Parquet/ChoParquetReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,12 +266,12 @@ private IDataReader AsDataReader(Action<IDictionary<string, Type>> membersDiscov
{
IDictionary<string, object> dict = null;
if (s is IDictionary<string, object>)
dict = ((IDictionary<string, object>)s).Flatten(Configuration.NestedColumnSeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
dict = ((IDictionary<string, object>)s).Flatten(Configuration.NestedKeySeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix);
else
{
dict = s.ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix).Flatten(Configuration.NestedColumnSeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedColumnSeparator,
dict = s.ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix).Flatten(Configuration.NestedKeySeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix);
}
Expand Down Expand Up @@ -434,12 +434,12 @@ public ChoParquetReader<T> ArrayIndexSeparator(char value)
return this;
}

public ChoParquetReader<T> NestedColumnSeparator(char value)
public ChoParquetReader<T> NestedKeySeparator(char value)
{
if (value == ChoCharEx.NUL)
throw new ArgumentException("Invalid nested column separator passed.");

Configuration.NestedColumnSeparator = value;
Configuration.NestedKeySeparator = value;
return this;
}

Expand Down
23 changes: 14 additions & 9 deletions src/ChoETL.Parquet/ChoParquetRecordWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,13 +428,18 @@ private object GetFirstNotNullRecord(IEnumerator<object> recEnum)

private object Flatten(object rec)
{
var dict = Configuration.UseNestedKeyFormat ?
rec?.FlattenToDictionary(Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix)
: rec;
if (Configuration.IsDynamicObjectInternal)
{
var dict = Configuration.UseNestedKeyFormat ?
rec?.FlattenToDictionary(Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix)
: rec;

return dict;
return dict;
}
else
return rec;
}

int rowGroupIndex = 0;
Expand Down Expand Up @@ -684,7 +689,7 @@ private string[] GetFields(List<object> records)

if (Configuration.UseNestedKeyFormat)
{
var kvps = record.Flatten(Configuration.NestedColumnSeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
var kvps = record.Flatten(Configuration.NestedKeySeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToArray();

Expand Down Expand Up @@ -726,7 +731,7 @@ private string[] GetFields(object record)
{
((ChoDynamicObject)dict).DynamicObjectName = ChoDynamicObject.DefaultName;
}
fieldNames = dict.Flatten(Configuration.NestedColumnSeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
fieldNames = dict.Flatten(Configuration.NestedKeySeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix).Keys.ToArray();
}
Expand Down Expand Up @@ -820,7 +825,7 @@ private bool ToText(long index, object rec, ref dynamic recText)
if (Configuration.IsDynamicObjectInternal)
dict = rec.ToDynamicObject() as IDictionary<string, Object>;
if (Configuration.IsDynamicObjectInternal && Configuration.UseNestedKeyFormat)
dict = dict.Flatten(Configuration.NestedColumnSeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
dict = dict.Flatten(Configuration.NestedKeySeparator, Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator,
Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToArray().ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix);
}
Expand Down
4 changes: 2 additions & 2 deletions src/ChoETL.Parquet/ChoParquetWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,12 +216,12 @@ public ChoParquetWriter<T> ArrayIndexSeparator(char value)
return this;
}

public ChoParquetWriter<T> NestedColumnSeparator(char value)
public ChoParquetWriter<T> NestedKeySeparator(char value)
{
if (value == ChoCharEx.NUL)
throw new ArgumentException("Invalid nested column separator passed.");

Configuration.NestedColumnSeparator = value;
Configuration.NestedKeySeparator = value;
return this;
}

Expand Down
2 changes: 1 addition & 1 deletion src/ChoETL.Yaml/ChoETL.Yaml.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<PackageTags>ETL c# .NET CSV Xml JSON Flat Parquet Yaml</PackageTags>
<PackageReleaseNotes>.NET Standard version</PackageReleaseNotes>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<Version>1.0.0.34</Version>
<Version>1.0.0.35</Version>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>ChoETL.Yaml.snk</AssemblyOriginatorKeyFile>
<PackageIcon>cinchoo.png</PackageIcon>
Expand Down
4 changes: 2 additions & 2 deletions src/ChoETL.Yaml/ChoYamlReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,11 +387,11 @@ private IDataReader AsDataReader(Action<IDictionary<string, Type>> membersDiscov
{
IDictionary<string, object> dict = null;
if (s is IDictionary<string, object>)
dict = ((IDictionary<string, object>)s).Flatten(Configuration.NestedColumnSeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedColumnSeparator,
dict = ((IDictionary<string, object>)s).Flatten(Configuration.NestedKeySeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix);
else
dict = s.ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix).Flatten(Configuration.NestedColumnSeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedColumnSeparator,
dict = s.ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix).Flatten(Configuration.NestedKeySeparator == null ? ChoETLSettings.NestedKeySeparator : Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix, Configuration.ArrayValueNamePrefix,
Configuration.IgnoreRootDictionaryFieldPrefix).ToDictionary(valueNamePrefix: Configuration.ArrayValueNamePrefix);

Expand Down
2 changes: 1 addition & 1 deletion src/ChoETL.Yaml/ChoYamlRecordWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ public override IEnumerable<object> WriteTo(object writer, IEnumerable<object> r
recEnum = GetRecords(recEnum).Select(r => r.ConvertToFlattenObject(Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix)).GetEnumerator();
else
recEnum = GetRecords(recEnum).Select(r => r.ToDynamicObject().ConvertToFlattenObject(Configuration.NestedColumnSeparator,
recEnum = GetRecords(recEnum).Select(r => r.ToDynamicObject().ConvertToFlattenObject(Configuration.NestedKeySeparator,
Configuration.ArrayIndexSeparator, Configuration.ArrayEndIndexSeparator, Configuration.IgnoreDictionaryFieldPrefix)).GetEnumerator();
}

Expand Down
73 changes: 47 additions & 26 deletions src/ChoETL/Common/ChoActivator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ public static class ChoActivator
{
private static readonly object _objCachePadLock = new object();
private static readonly Dictionary<Type, object> _objCache = new Dictionary<Type, object>();
public static Func<Type, object[], object> Factory
{
get;
set;
}
public static Func<Type, object[], object> Factory
{
get;
set;
}

static ChoActivator()
{
Expand Down Expand Up @@ -46,7 +46,7 @@ public static object CreateInstanceNCache(Type objType, Func<object> createInsta
public static T CreateInstanceNCache<T>()
{
return (T)CreateInstanceNCache(typeof(T));
}
}

public static object CreateInstanceNCache(Type objType, params object[] args)
{
Expand All @@ -66,14 +66,14 @@ public static object CreateInstanceNCache(Type objType, params object[] args)
}

public static T CreateInstance<T>()
{
return (T)CreateInstance(typeof(T));
}

public static object CreateInstance(Type objType, params object[] args)
{
try
{
{
return (T)CreateInstance(typeof(T));
}

public static object CreateInstance(Type objType, params object[] args)
{
try
{
object obj = null;
Func<Type, object[], object> factory = Factory;
if (factory != null)
Expand Down Expand Up @@ -115,14 +115,14 @@ public static object CreateInstance(Type objType, params object[] args)
}

return obj;
}
catch (TargetInvocationException ex)
{
throw ex.InnerException;
}
}

public static T CreateInstanceAndInit<T>()
}
catch (TargetInvocationException ex)
{
throw ex.InnerException;
}
}

public static T CreateInstanceAndInit<T>()
{
return (T)CreateInstanceAndInit(typeof(T));
}
Expand All @@ -131,10 +131,31 @@ public static object CreateInstanceAndInit(Type objType, params object[] args)
{
try
{
object obj = Factory != null ? Factory(objType, args) : null;
object obj = Factory != null ? Factory(objType, args) : null;
if (obj == null)
{
obj = Activator.CreateInstance(objType, args);
if (Type.GetTypeCode(objType) == TypeCode.String)
return string.Empty;

// get the default constructor and instantiate
Type[] types = new Type[0];
ConstructorInfo info = objType.GetConstructor(types);

if (info == null) //must not have found the constructor
{
if (objType.BaseType.UnderlyingSystemType.FullName.Contains("Enum"))
obj = Activator.CreateInstance(objType);
else
obj = info.Invoke(null);
}

if (obj == null)
{
if (args == null || args.Length == 0)
obj = Activator.CreateInstance(objType);
else
obj = Activator.CreateInstance(objType, args);
}
obj.Initialize();
}
return obj;
Expand Down Expand Up @@ -235,7 +256,7 @@ private static ChoRandomGenerator GetRandomObject(string fieldName, string rgPar

if (rgParams.IsNullOrWhiteSpace())
return null;
//throw new ApplicationException("No random generator defined for {0} field.".FormatString(fieldName));
//throw new ApplicationException("No random generator defined for {0} field.".FormatString(fieldName));

string rgType = rgParams.SplitNTrim().FirstOrDefault();
if (rgType.IsNullOrWhiteSpace())
Expand All @@ -247,7 +268,7 @@ private static ChoRandomGenerator GetRandomObject(string fieldName, string rgPar

string rgObjectParams = String.Join(",", rgParams.SplitNTrim().Skip(1).ToArray());
return Activator.CreateInstance(rgt, (from z in rgObjectParams.SplitNTrim()
select z.ToObject()).ToArray()) as ChoRandomGenerator;
select z.ToObject()).ToArray()) as ChoRandomGenerator;
}
}
}
Loading

0 comments on commit 49e26c7

Please sign in to comment.