C# benchmark tool

After the last post on .Net 4.0 StringBuilder vs .Net 2.0 StringBuilder I realized I often ended-up testing different strategies for efficiency and I'm a little bit tired of rewriting the same .Net benchmarking framework, so here it is.

I use this when I need to check the execution speed and memory consumption of a piece of C# code, usually with pretty consistent results. The idea is to have a piece of code running in an environment where timing and memory use are monitored, and making sure the data collected isn't skewed by the JIT kicking-in or the GC stalling everything.

Here's the StringBuilder post rewritten to use this framework. Each method in the loop will be passed the object the initer fired, and the current iteration/max iter values.




using System;
using System.Text;

namespace OneKStrongOxen
{
    internal class StringBuilderDemo
    {
        private static void Main(string[] args)
        {
            var bench = new Benchmark(100, 20,10,
                string.Format("StringBuilder v{0} in .Net {1}", 
typeof(StringBuilder).Assembly.GetName().Version, 
Environment.Version));

            StringBuilder bob = null; // use this guy for the tostring test
            bench.Run("Append End", (sb, i, l) => sb.Append("abcdefgh"), 
() => bob = new StringBuilder());
            bench.Run("Insert Beg", (sb, i, l) => sb.Insert(0, "abcdefgh"), 
() => new StringBuilder());
            bench.Run("Insert Mid", (sb, i, l) => sb.Insert(sb.Length / 2, "abcdefgh"), 
() => new StringBuilder());
            bench.Run("ToString", (sb, i, l) => sb.ToString(), () => bob);

            Console.Out.WriteLine(bench);
        }
    }
}
And here's the results
--------------------
Test: StringBuilder v2.0.0.0 in .Net 2.0.50727.3082
 DryRuns: 100   Loops: 20    Repeats:10
 -Append End Total:0.0943999999999998ms deltaB:327680  avgDeltaB:1638.4
  Avg:0.000471999999999999  Min:0.0003   Max:0.0022
 -Insert Beg Total:0.1183 ms deltaB:228752  avgDeltaB:1143.76
  Avg:0.0005915  Min:0.0004   Max:0.0022
 -Insert Mid Total:0.119  ms deltaB:327680  avgDeltaB:1638.4
  Avg:0.000595  Min:0.0004   Max:0.0025
 -ToString Total:0.1843 ms deltaB:1638400 avgDeltaB:8192
  Avg:0.000921500000000002  Min:0.0009   Max:0.0016
And now for the bench code itself...

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;

namespace OneKStrongOxen
{
    /// <summary>
    /// A benchmark suite trackign min/max/avg execution times as well as memory use
    /// </summary>
    public class Benchmark : IEnumerable<Benchmark.Result>
    {
        private readonly int _repeats;
        public int DryRuns { get; private set; }
        public int Loops { get; private set; }
        public string BenchName { get; private set; }
        public int Repeats { get; private set; }

        readonly List<Result> results = new List<Result>();
        public IEnumerable<Result> SortedResults(IComparer<Result> sortOrder)
        {
            lock (results)
            {
                var r = results.ToArray();
                Array.Sort(r, sortOrder);
                return r;
            }
        }

        /// <summary>
        /// Create a benchmark suite with consistent parameters for different tests
        /// </summary>
        /// <param name="dryRuns">Number of dry loops before perf is measured</param>
        /// <param name="loops">Number of test iterations</param>
        /// <param name="repeats">Number of time this bench should be repeated</param>
        /// <param name="benchName">Name of this benchmark</param>
        public Benchmark(int dryRuns, int loops, int repeats, string benchName)
        {
            
            DryRuns = dryRuns;
            Loops = loops;
            BenchName = benchName;
            Repeats = repeats;
        }

        /// <summary>
        /// Enumerate over the results
        /// </summary>
        /// <returns></returns>
        public IEnumerator<Result> GetEnumerator()
        {
            Result[] ar;

            lock (results)
            {
                ar = results.ToArray();
            }
            for (int i = 0; i < ar.Length; i++)
            {
                yield return ar[i];
            }
        }

        public override string ToString()
        {
            var sb = new StringBuilder(new string('-', 20)).AppendLine()
                .Append("Test: ").AppendLine(this.BenchName)
                .AppendFormat(" DryRuns: {0,-5} Loops: {1,-5} Repeats:{2,-5}",
DryRuns, Loops,Repeats).AppendLine();

            lock (results)
            {
                foreach (var res in results)
                {
                    sb.AppendLine(res.ToString());
                }
            }

            return sb.ToString();
        }

        /// <summary>
        /// Enumerate over the results
        /// </summary>
        /// <returns></returns>
        IEnumerator IEnumerable.GetEnumerator()
        {
            return GetEnumerator();
        }

        /// <summary>
        /// Loops a benchmark
        /// </summary>
        /// <param name="name"></param>
        /// <param name="test"></param>
        /// <param name="initer"></param>
        /// <returns></returns>
        public Result Run<T>(string name, Action<T, int, int> test, Func<T> initer)
        {
            T val = initer();
            for (int i = 0; i < DryRuns; i++)
                test(val, i, DryRuns);

            GC.Collect();GC.WaitForPendingFinalizers();
            double min = double.MaxValue, max = double.MinValue, total = 0d;
            long deltaMem = 0;
            long totalRuns = Repeats*Loops;
            for (int repeat = 0; repeat < Repeats; repeat++)
            {
                val = initer();
                Stopwatch watch = new Stopwatch();
                for (int i = 0; i < Loops; i++)
                {
                    long startMem = GC.GetTotalMemory(true);
                    watch.Start();
                    test(val, i, Loops); // run the test
                    watch.Stop();
                    deltaMem += GC.GetTotalMemory(false) - startMem;
                    // collate the data
                    var el = watch.Elapsed.TotalMilliseconds;
                    total += el;
                    min = Math.Min(min, el);
                    max = Math.Max(max, el);
                    watch.Reset();
                }
            }
            // store results
            var r = new Result(name, total, total / totalRuns, min, max, deltaMem, deltaMem / (double)totalRuns);
            lock (results)
            {
                results.Add(r);
            }
            return r;
        }



        #region Nested type: Result

        /// <summary>
        /// Bench results
        /// </summary>
        public struct Result
        {
            internal Result(string name, double totalTime, double averageTime, double minTime, double maxTime, long totalDeltaMem, double avgDeltaMem)
                : this()
            {
                Name = name;
                TotalTimeMs = totalTime;
                AverageTimeMs = averageTime;
                MinTimeMs = minTime;
                MaxTimeMs = maxTime;
                TotalDeltaMem = totalDeltaMem;
                AvgDeltaMem = avgDeltaMem;
            }


            public string Name { get; private set; }
            public double TotalTimeMs { get; private set; }
            public double AverageTimeMs { get; private set; }
            public double MinTimeMs { get; private set; }
            public double MaxTimeMs { get; private set; }
            public long TotalDeltaMem { get; private set; }
            public double AvgDeltaMem { get; private set; }

            public override string ToString()
            {
                return string.Format(" -{0} Total:{1,-7}ms deltaB:{6,-7} avgDeltaB:{7,-7} {5}  Avg:{2,-7}  Min:{3,-7}  Max:{4,-7}", Name, TotalTimeMs, AverageTimeMs, MinTimeMs, MaxTimeMs, Environment.NewLine, TotalDeltaMem, AvgDeltaMem);
            }
        }

        #endregion
    }
}

The meat of the thing is in the fact that you need two delegates, one is the inner loop body and the other is the loop initializer.

You can then use the results from the Benchmark instance.

No comments:

Post a Comment

Please leave your comments in English or French and I will be pleased to answer them if you have any questions.

Spammers will be walked down the plank matey. Arrr!