From 5f27a4bfc49e6b2be4533a85a3de32deecaeeee8 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 7 Jun 2016 22:17:25 +0100 Subject: [PATCH] Add InListStringSplitCount; when set, causes `in @foo` parameters to be implemented using string_split --- Dapper.Tests/Attributes.cs | 25 ++++- Dapper.Tests/Tests.Parameters.cs | 51 ++++++++++ Dapper.Tests/project.json | 2 +- Dapper/SqlMapper.Settings.cs | 5 + Dapper/SqlMapper.cs | 155 ++++++++++++++++++++++++------- 5 files changed, 200 insertions(+), 38 deletions(-) diff --git a/Dapper.Tests/Attributes.cs b/Dapper.Tests/Attributes.cs index b5261b9..1ed3a1c 100644 --- a/Dapper.Tests/Attributes.cs +++ b/Dapper.Tests/Attributes.cs @@ -28,7 +28,30 @@ public FactLongRunningAttribute() } public string Url { get; private set; } } - public class FactUnlessCaseSensitiveDatabaseAttribute : FactAttribute + public class FactRequiredCompatibilityLevelAttribute : FactAttribute + { + public FactRequiredCompatibilityLevelAttribute(int level) : base() + { + if (DetectedLevel < level) + { + Skip = $"Compatibility level {level} required; detected {DetectedLevel}"; + } + } + public const int SqlServer2016 = 130; + public static readonly int DetectedLevel; + static FactRequiredCompatibilityLevelAttribute() + { + using (var conn = TestSuite.GetOpenConnection()) + { + try + { + DetectedLevel = conn.QuerySingle("SELECT compatibility_level FROM sys.databases where name = DB_NAME()"); + } + catch { } + } + } + } + public class FactUnlessCaseSensitiveDatabaseAttribute : FactAttribute { public FactUnlessCaseSensitiveDatabaseAttribute() : base() { diff --git a/Dapper.Tests/Tests.Parameters.cs b/Dapper.Tests/Tests.Parameters.cs index d5873be..1d1ca09 100644 --- a/Dapper.Tests/Tests.Parameters.cs +++ b/Dapper.Tests/Tests.Parameters.cs @@ -992,5 +992,56 @@ public void SO30156367_DynamicParamsWithoutExec() var value = dbParams.Get("Field1"); value.IsEqualTo(1); } + + [Fact] + public void RunAllStringSplitTestsDisabled() + { + RunAllStringSplitTests(-1, 1500); + } + [FactRequiredCompatibilityLevel(FactRequiredCompatibilityLevelAttribute.SqlServer2016)] + public void RunAllStringSplitTestsEnabled() + { + RunAllStringSplitTests(10, 4500); + } + private void RunAllStringSplitTests(int stringSplit, int max = 150) + { + int oldVal = SqlMapper.Settings.InListStringSplitCount; + try + { + SqlMapper.Settings.InListStringSplitCount = stringSplit; + try { connection.Execute("drop table #splits"); } catch { } + int count = connection.QuerySingle("create table #splits (i int not null);" + + string.Concat(Enumerable.Range(-max, max * 3).Select(i => $"insert #splits (i) values ({i});")) + + "select count(1) from #splits"); + count.IsEqualTo(3 * max); + + for (int i = 0; i < max; Incr(ref i)) + { + try + { + var vals = Enumerable.Range(1, i); + var list = connection.Query("select i from #splits where i in @vals", new { vals }).AsList(); + list.Count.IsEqualTo(i); + list.Sum().IsEqualTo(vals.Sum()); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Error when i={i}: {ex.Message}", ex); + } + } + } + finally + { + SqlMapper.Settings.InListStringSplitCount = oldVal; + } + } + static void Incr(ref int i) + { + if (i <= 15) i++; + else if (i <= 80) i += 5; + else if (i <= 200) i += 10; + else if (i <= 1000) i += 50; + else i += 100; + } } } diff --git a/Dapper.Tests/project.json b/Dapper.Tests/project.json index 1b779f7..15e21ad 100644 --- a/Dapper.Tests/project.json +++ b/Dapper.Tests/project.json @@ -135,7 +135,7 @@ "version": "1.0.0-rc2-3002702", "type": "platform" }, - "Microsoft.Data.Sqlite": "1.0.0-rc2-20597", + "Microsoft.Data.Sqlite": "1.0.0-rc2-final", "xunit": "2.1.0", "dotnet-test-xunit": "1.0.0-rc3-*" } diff --git a/Dapper/SqlMapper.Settings.cs b/Dapper/SqlMapper.Settings.cs index c3036f2..fa17566 100644 --- a/Dapper/SqlMapper.Settings.cs +++ b/Dapper/SqlMapper.Settings.cs @@ -44,6 +44,11 @@ public static void SetDefaults() /// default and must be enabled. /// public static bool PadListExpansions { get; set; } + /// + /// If set (non-negative), when performing in-list expansions of integer types ("where id in @ids", etc), switch to a string_split based + /// operation if there are more than this many elements. Note that this feautre requires SQL Server 2016 / compatibility level 130 (or above). + /// + public static int InListStringSplitCount { get; set; } = -1; } } } diff --git a/Dapper/SqlMapper.cs b/Dapper/SqlMapper.cs index 1ef0a34..96f11dc 100644 --- a/Dapper/SqlMapper.cs +++ b/Dapper/SqlMapper.cs @@ -1831,6 +1831,7 @@ internal static int GetListPaddingExtraCount(int count) return intoBlock == 0 ? 0 : (padFactor - intoBlock); } + private static string GetInListRegex(string name) => @"([?@:]" + Regex.Escape(name) + @")(?!\w)(\s+(?i)unknown(?-i))?"; /// /// Internal use only /// @@ -1858,7 +1859,12 @@ public static void PackListParameters(IDbCommand command, string namePrefix, obj bool isString = value is IEnumerable; bool isDbString = value is IEnumerable; DbType dbType = 0; - if (list != null) + + int splitAt = SqlMapper.Settings.InListStringSplitCount; + bool viaSplit = splitAt >= 0 + && TryStringSplit(ref list, splitAt, namePrefix, command); + + if (list != null && !viaSplit) { object lastValue = null; foreach (var item in list) @@ -1921,57 +1927,134 @@ public static void PackListParameters(IDbCommand command, string namePrefix, obj } } - var regexIncludingUnknown = @"([?@:]" + Regex.Escape(namePrefix) + @")(?!\w)(\s+(?i)unknown(?-i))?"; - if (count == 0) + + if(viaSplit) { - command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match => + // already done + } + else + { + var regexIncludingUnknown = GetInListRegex(namePrefix); + if (count == 0) { - var variableName = match.Groups[1].Value; - if (match.Groups[2].Success) + command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match => { + var variableName = match.Groups[1].Value; + if (match.Groups[2].Success) + { // looks like an optimize hint; leave it alone! return match.Value; - } - else - { - return "(SELECT " + variableName + " WHERE 1 = 0)"; - } - }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant); - var dummyParam = command.CreateParameter(); - dummyParam.ParameterName = namePrefix; - dummyParam.Value = DBNull.Value; - command.Parameters.Add(dummyParam); - } - else - { - command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match => + } + else + { + return "(SELECT " + variableName + " WHERE 1 = 0)"; + } + }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant); + var dummyParam = command.CreateParameter(); + dummyParam.ParameterName = namePrefix; + dummyParam.Value = DBNull.Value; + command.Parameters.Add(dummyParam); + } + else { - var variableName = match.Groups[1].Value; - if (match.Groups[2].Success) + command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match => { + var variableName = match.Groups[1].Value; + if (match.Groups[2].Success) + { // looks like an optimize hint; expand it var suffix = match.Groups[2].Value; - var sb = GetStringBuilder().Append(variableName).Append(1).Append(suffix); - for (int i = 2; i <= count; i++) - { - sb.Append(',').Append(variableName).Append(i).Append(suffix); + var sb = GetStringBuilder().Append(variableName).Append(1).Append(suffix); + for (int i = 2; i <= count; i++) + { + sb.Append(',').Append(variableName).Append(i).Append(suffix); + } + return sb.__ToStringRecycle(); } - return sb.__ToStringRecycle(); - } - else - { - var sb = GetStringBuilder().Append('(').Append(variableName).Append(1); - for (int i = 2; i <= count; i++) + else { - sb.Append(',').Append(variableName).Append(i); + var sb = GetStringBuilder().Append('(').Append(variableName).Append(1); + for (int i = 2; i <= count; i++) + { + sb.Append(',').Append(variableName).Append(i); + } + return sb.Append(')').__ToStringRecycle(); } - return sb.Append(')').__ToStringRecycle(); - } - }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant); + }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant); + } } } + } + + private static bool TryStringSplit(ref IEnumerable list, int splitAt, string namePrefix, IDbCommand command) + { + if (list == null || splitAt < 0) return false; + if (list is IEnumerable) return TryStringSplit(ref list, splitAt, namePrefix, command, "int not null", + (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture))); + if (list is IEnumerable) return TryStringSplit(ref list, splitAt, namePrefix, command, "bigint not null", + (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture))); + if (list is IEnumerable) return TryStringSplit(ref list, splitAt, namePrefix, command, "smallint not null", + (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture))); + if (list is IEnumerable) return TryStringSplit(ref list, splitAt, namePrefix, command, "tinyint not null", + (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture))); + return false; + } + private static bool TryStringSplit(ref IEnumerable list, int splitAt, string namePrefix, IDbCommand command, string colType, + Action append) + { + ICollection typed = list as ICollection; + if(typed == null) + { + typed = ((IEnumerable)list).ToList(); + list = typed; // because we still need to be able to iterate it, even if we fail here + } + if (typed.Count < splitAt) return false; + + string varName = null; + var regexIncludingUnknown = GetInListRegex(namePrefix); + var sql = Regex.Replace(command.CommandText, regexIncludingUnknown, match => + { + var variableName = match.Groups[1].Value; + if (match.Groups[2].Success) + { + // looks like an optimize hint; leave it alone! + return match.Value; + } + else + { + varName = variableName; + return $"(SELECT val from {variableName}_TSS)"; + } + }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant); + if (varName == null) return false; // couldn't resolve the var! + command.CommandText = $"declare {varName}_TSS table(val {colType});insert {varName}_TSS (val) select value from string_split({varName},',');" + sql; + var concatenatedParam = command.CreateParameter(); + concatenatedParam.ParameterName = namePrefix; + concatenatedParam.DbType = DbType.AnsiString; + concatenatedParam.Size = -1; + string val; + using (var iter = typed.GetEnumerator()) + { + if(iter.MoveNext()) + { + var sb = GetStringBuilder(); + append(sb, iter.Current); + while(iter.MoveNext()) + { + append(sb.Append(','), iter.Current); + } + val = sb.ToString(); + } + else + { + val = ""; + } + } + concatenatedParam.Value = val; + command.Parameters.Add(concatenatedParam); + return true; } /// -- GitLab