Add InListStringSplitCount; when set, causes `in @foo` parameters to be...

Add InListStringSplitCount; when set, causes `in @foo` parameters to be implemented using string_split

Dapper.Tests/Attributes.cs

@@ -28,7 +28,30 @@ public FactLongRunningAttribute()
         }
         public string Url { get; private set; }
     }
-    public class FactUnlessCaseSensitiveDatabaseAttribute : FactAttribute
+    public class FactRequiredCompatibilityLevelAttribute : FactAttribute
+    {
+        public FactRequiredCompatibilityLevelAttribute(int level) : base()
+        {
+            if (DetectedLevel < level)
+            {
+                Skip = $"Compatibility level {level} required; detected {DetectedLevel}";
+            }
+        }
+        public const int SqlServer2016 = 130;
+        public static readonly int DetectedLevel;
+        static FactRequiredCompatibilityLevelAttribute()
+        {
+            using (var conn = TestSuite.GetOpenConnection())
+            {
+                try
+                {
+                    DetectedLevel = conn.QuerySingle<int>("SELECT compatibility_level FROM sys.databases where name = DB_NAME()");
+                }
+                catch { }
+            }
+        }
+    }
+        public class FactUnlessCaseSensitiveDatabaseAttribute : FactAttribute
     {
         public FactUnlessCaseSensitiveDatabaseAttribute() : base()
         {

Dapper.Tests/Tests.Parameters.cs

@@ -992,5 +992,56 @@ public void SO30156367_DynamicParamsWithoutExec()
             var value = dbParams.Get<int>("Field1");
             value.IsEqualTo(1);
         }
+
+        [Fact]
+        public void RunAllStringSplitTestsDisabled()
+        {
+            RunAllStringSplitTests(-1, 1500);
+        }
+        [FactRequiredCompatibilityLevel(FactRequiredCompatibilityLevelAttribute.SqlServer2016)]
+        public void RunAllStringSplitTestsEnabled()
+        {
+            RunAllStringSplitTests(10, 4500);
+        }
+        private void RunAllStringSplitTests(int stringSplit, int max = 150)
+        {
+            int oldVal = SqlMapper.Settings.InListStringSplitCount;
+            try
+            {
+                SqlMapper.Settings.InListStringSplitCount = stringSplit;
+                try { connection.Execute("drop table #splits"); } catch { }
+                int count = connection.QuerySingle<int>("create table #splits (i int not null);"
+                    + string.Concat(Enumerable.Range(-max, max * 3).Select(i => $"insert #splits (i) values ({i});"))
+                    + "select count(1) from #splits");
+                count.IsEqualTo(3 * max);
+
+                for (int i = 0; i < max; Incr(ref i))
+                {
+                    try
+                    {
+                        var vals = Enumerable.Range(1, i);
+                        var list = connection.Query<int>("select i from #splits where i in @vals", new { vals }).AsList();
+                        list.Count.IsEqualTo(i);
+                        list.Sum().IsEqualTo(vals.Sum());
+                    }
+                    catch (Exception ex)
+                    {
+                        throw new InvalidOperationException($"Error when i={i}: {ex.Message}", ex);
+                    }
+                }
+            }
+            finally
+            {
+                SqlMapper.Settings.InListStringSplitCount = oldVal;
+            }
+        }
+        static void Incr(ref int i)
+        {
+            if (i <= 15) i++;
+            else if (i <= 80) i += 5;
+            else if (i <= 200) i += 10;
+            else if (i <= 1000) i += 50;
+            else i += 100;
+        }
     }
 }

Dapper.Tests/project.json

@@ -135,7 +135,7 @@
           "version": "1.0.0-rc2-3002702",
           "type": "platform"
         },
-        "Microsoft.Data.Sqlite": "1.0.0-rc2-20597",
+        "Microsoft.Data.Sqlite": "1.0.0-rc2-final",
         "xunit": "2.1.0",
         "dotnet-test-xunit": "1.0.0-rc3-*"
       }

Dapper/SqlMapper.Settings.cs

@@ -44,6 +44,11 @@ public static void SetDefaults()
             /// default and must be enabled.
             /// </remarks>
             public static bool PadListExpansions { get; set; }
+            /// <summary>
+            /// If set (non-negative), when performing in-list expansions of integer types ("where id in @ids", etc), switch to a string_split based
+            /// operation if there are more than this many elements. Note that this feautre requires SQL Server 2016 / compatibility level 130 (or above).
+            /// </summary>
+            public static int InListStringSplitCount { get; set; } = -1;
         }
     }
 }

Dapper/SqlMapper.cs

@@ -1831,6 +1831,7 @@ internal static int GetListPaddingExtraCount(int count)
             return intoBlock == 0 ? 0 : (padFactor - intoBlock);
         }
 
+        private static string GetInListRegex(string name) => @"([?@:]" + Regex.Escape(name) + @")(?!\w)(\s+(?i)unknown(?-i))?";
         /// <summary>
         /// Internal use only
         /// </summary>
@@ -1858,7 +1859,12 @@ public static void PackListParameters(IDbCommand command, string namePrefix, obj
                 bool isString = value is IEnumerable<string>;
                 bool isDbString = value is IEnumerable<DbString>;
                 DbType dbType = 0;
-                if (list != null)
+
+                int splitAt = SqlMapper.Settings.InListStringSplitCount;
+                bool viaSplit = splitAt >= 0
+                    && TryStringSplit(ref list, splitAt, namePrefix, command);
+
+                if (list != null && !viaSplit)
                 {
                     object lastValue = null;
                     foreach (var item in list)
@@ -1921,57 +1927,134 @@ public static void PackListParameters(IDbCommand command, string namePrefix, obj
                     }
                 }
 
-                var regexIncludingUnknown = @"([?@:]" + Regex.Escape(namePrefix) + @")(?!\w)(\s+(?i)unknown(?-i))?";
-                if (count == 0)
+                
+                if(viaSplit)
                 {
-                    command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match =>
+                    // already done
+                }
+                else
+                {
+                    var regexIncludingUnknown = GetInListRegex(namePrefix);
+                    if (count == 0)
                     {
-                        var variableName = match.Groups[1].Value;
-                        if (match.Groups[2].Success)
+                        command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match =>
                         {
+                            var variableName = match.Groups[1].Value;
+                            if (match.Groups[2].Success)
+                            {
                             // looks like an optimize hint; leave it alone!
                             return match.Value;
-                        }
-                        else
-                        {
-                            return "(SELECT " + variableName + " WHERE 1 = 0)";
-                        }
-                    }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant);
-                    var dummyParam = command.CreateParameter();
-                    dummyParam.ParameterName = namePrefix;
-                    dummyParam.Value = DBNull.Value;
-                    command.Parameters.Add(dummyParam);
-                }
-                else
-                {
-                    command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match =>
+                            }
+                            else
+                            {
+                                return "(SELECT " + variableName + " WHERE 1 = 0)";
+                            }
+                        }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant);
+                        var dummyParam = command.CreateParameter();
+                        dummyParam.ParameterName = namePrefix;
+                        dummyParam.Value = DBNull.Value;
+                        command.Parameters.Add(dummyParam);
+                    }
+                    else
                     {
-                        var variableName = match.Groups[1].Value;
-                        if (match.Groups[2].Success)
+                        command.CommandText = Regex.Replace(command.CommandText, regexIncludingUnknown, match =>
                         {
+                            var variableName = match.Groups[1].Value;
+                            if (match.Groups[2].Success)
+                            {
                             // looks like an optimize hint; expand it
                             var suffix = match.Groups[2].Value;
 
-                            var sb = GetStringBuilder().Append(variableName).Append(1).Append(suffix);
-                            for (int i = 2; i <= count; i++)
-                            {
-                                sb.Append(',').Append(variableName).Append(i).Append(suffix);
+                                var sb = GetStringBuilder().Append(variableName).Append(1).Append(suffix);
+                                for (int i = 2; i <= count; i++)
+                                {
+                                    sb.Append(',').Append(variableName).Append(i).Append(suffix);
+                                }
+                                return sb.__ToStringRecycle();
                             }
-                            return sb.__ToStringRecycle();
-                        }
-                        else
-                        {
-                            var sb = GetStringBuilder().Append('(').Append(variableName).Append(1);
-                            for (int i = 2; i <= count; i++)
+                            else
                             {
-                                sb.Append(',').Append(variableName).Append(i);
+                                var sb = GetStringBuilder().Append('(').Append(variableName).Append(1);
+                                for (int i = 2; i <= count; i++)
+                                {
+                                    sb.Append(',').Append(variableName).Append(i);
+                                }
+                                return sb.Append(')').__ToStringRecycle();
                             }
-                            return sb.Append(')').__ToStringRecycle();
-                        }
-                    }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant);
+                        }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant);
+                    }
                 }
             }
+        }
+
+        private static bool TryStringSplit(ref IEnumerable list, int splitAt, string namePrefix, IDbCommand command)
+        {
+            if (list == null || splitAt < 0) return false;
+            if (list is IEnumerable<int>) return TryStringSplit<int>(ref list, splitAt, namePrefix, command, "int not null",
+                (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture)));
+            if (list is IEnumerable<long>) return TryStringSplit<long>(ref list, splitAt, namePrefix, command, "bigint not null",
+                (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture)));
+            if (list is IEnumerable<short>) return TryStringSplit<short>(ref list, splitAt, namePrefix, command, "smallint not null",
+                (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture)));            
+            if (list is IEnumerable<byte>) return TryStringSplit<byte>(ref list, splitAt, namePrefix, command, "tinyint not null",
+                (sb, i) => sb.Append(i.ToString(CultureInfo.InvariantCulture)));
+            return false;
+        }
+        private static bool TryStringSplit<T>(ref IEnumerable list, int splitAt, string namePrefix, IDbCommand command, string colType,
+            Action<StringBuilder, T> append)
+        {
+            ICollection<T> typed = list as ICollection<T>; 
+            if(typed == null)
+            {
+                typed = ((IEnumerable<T>)list).ToList();
+                list = typed; // because we still need to be able to iterate it, even if we fail here
+            }
+            if (typed.Count < splitAt) return false;
+            
+            string varName = null;
+            var regexIncludingUnknown = GetInListRegex(namePrefix);
+            var sql = Regex.Replace(command.CommandText, regexIncludingUnknown, match =>
+            {
+                var variableName = match.Groups[1].Value;
+                if (match.Groups[2].Success)
+                {
+                    // looks like an optimize hint; leave it alone!
+                    return match.Value;
+                }
+                else
+                {
+                    varName = variableName;
+                    return $"(SELECT val from {variableName}_TSS)";
+                }
+            }, RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.CultureInvariant);
+            if (varName == null) return false; // couldn't resolve the var!
 
+            command.CommandText = $"declare {varName}_TSS table(val {colType});insert {varName}_TSS (val) select value from string_split({varName},',');" + sql;
+            var concatenatedParam = command.CreateParameter();
+            concatenatedParam.ParameterName = namePrefix;
+            concatenatedParam.DbType = DbType.AnsiString;
+            concatenatedParam.Size = -1;
+            string val;
+            using (var iter = typed.GetEnumerator())
+            {
+                if(iter.MoveNext())
+                {
+                    var sb = GetStringBuilder();
+                    append(sb, iter.Current);
+                    while(iter.MoveNext())
+                    {
+                        append(sb.Append(','), iter.Current);
+                    }
+                    val = sb.ToString();
+                }
+                else
+                {
+                    val = "";
+                }
+            }
+            concatenatedParam.Value = val;
+            command.Parameters.Add(concatenatedParam);
+            return true;
         }
 
         /// <summary>