diff --git a/source/C#/UniTraX/Core/Bookkeeper/BKQueryable.cs b/source/C#/UniTraX/Core/Bookkeeper/BKQueryable.cs new file mode 100644 index 0000000000000000000000000000000000000000..57c54035a17e3f53875d98cb2ac9e5e779c55976 --- /dev/null +++ b/source/C#/UniTraX/Core/Bookkeeper/BKQueryable.cs @@ -0,0 +1,87 @@ +namespace Qbb.Core.Bookkeeper +{ + using PINQ; + using Qbb.Core.Partition; + using System; + using System.Collections.Generic; + using System.Linq; + + public class BKQueryable + { + public IQueryable Queryable { get; private set; } + public Bookkeeper Bookkeeper { get; private set; } + public Partition Current { get; private set; } + + public BKQueryable(IQueryable queryable, Bookkeeper bookkeeper, Partition current) + { + Queryable = queryable ?? throw new ArgumentNullException(nameof(queryable)); + Bookkeeper = bookkeeper ?? throw new ArgumentNullException(nameof(bookkeeper)); + Current = current ?? throw new ArgumentNullException(nameof(current)); + } + + public BKQueryable(IQueryable queryable, Bookkeeper bookkeeper, ContiguousPartition current) : + this(queryable, bookkeeper, new Partition(new HashSet() { current })) { } + + public long MaxUsage() + { + return Bookkeeper.MaxUsage(Current).Epsilon; + } + + public void WriteDetailedBudgetUseToFile(string fileName) + { + if (fileName == null) throw new ArgumentNullException(nameof(fileName)); + Bookkeeper.WriteDetailedBudgetUseToFile(fileName); + } + + public int BudgetIndex() => Bookkeeper.BudgetIndex(); + + public Boolean CoversDataspace() + { + return Current.Parts.Count > 0; + } + + public PINQueryable AsPinq(long epsilon) + { + return Bookkeeper.CreatePinQueryable(Queryable, Current, epsilon); + } + + public void CleanHistory() => Bookkeeper.CleanHistory(); + + public BKQueryable Materialize() + { + return new BKQueryable(Bookkeeper.Materialize(Queryable, Current), Bookkeeper, Current); + } + + public BKQueryable Apply(Func function) + { + if (function == null) throw new ArgumentNullException(nameof(function)); + return new BKQueryable(Queryable, Bookkeeper, function(Current)); + } + + public BKQueryable Except(BKQueryable other) + { + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Bookkeeper != Bookkeeper) throw new ArgumentException("bookkeepers do not match"); + return new BKQueryable(Queryable, Bookkeeper, Current.RemainderWithout(other.Current)); + } + + public BKQueryable Intersect(BKQueryable other) + { + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Bookkeeper != Bookkeeper) throw new ArgumentException("bookkeepers do not match"); + return new BKQueryable(Queryable, Bookkeeper, Current.IntersectionWith(other.Current)); + } + + public BKQueryable Union(BKQueryable other) + { + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Bookkeeper != Bookkeeper) throw new ArgumentException("bookkeepers do not match"); + return new BKQueryable(Queryable, Bookkeeper, Current.UnionWith(other.Current)); + } + + public override string ToString() + { + return "BKQ:\n" + Bookkeeper + "\n" + Current; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Bookkeeper/Bookkeeper.cs b/source/C#/UniTraX/Core/Bookkeeper/Bookkeeper.cs new file mode 100644 index 0000000000000000000000000000000000000000..799c74f2e5230160bdb1fa177a61036d148354f4 --- /dev/null +++ b/source/C#/UniTraX/Core/Bookkeeper/Bookkeeper.cs @@ -0,0 +1,159 @@ +namespace Qbb.Core.Bookkeeper +{ + using PINQ; + using Qbb.Core.BoundedData; + using Qbb.Core.History; + using Qbb.Core.Partition; + using System; + using System.Collections.Generic; + using System.Linq; + + public class Bookkeeper + { + private BookkeeperState State; + private BoundedDataAccess BoundedDataAccess; + + public Bookkeeper(BookkeeperState state, BoundedDataAccess boundedDataAccess) + { +#if DEBUG + if (state == null) throw new ArgumentNullException(nameof(state)); + if (boundedDataAccess == null) throw new ArgumentNullException(nameof(boundedDataAccess)); +#endif + State = state; + BoundedDataAccess = boundedDataAccess; + } + + public Bookkeeper(Partition focus, BoundedDataAccess boundedDataAccess, IHistory history) + { +#if DEBUG + if (focus == null) throw new ArgumentNullException(nameof(focus)); + if (boundedDataAccess == null) throw new ArgumentNullException(nameof(boundedDataAccess)); + if (history == null) throw new ArgumentNullException(nameof(history)); +#endif + State = new BookkeeperState + { + Forbidden = new Partition(new HashSet()), + Focus = focus ?? throw new ArgumentNullException(nameof(focus)), + History = history ?? throw new ArgumentNullException(nameof(history)), + NumberPartitionsAfterCleanCount = 0 + }; + BoundedDataAccess = boundedDataAccess ?? throw new ArgumentNullException(nameof(boundedDataAccess)); + } + + public Bookkeeper(ContiguousPartition focus, BoundedDataAccess boundedDataAccess, IHistory history) : + this(new Partition(new HashSet() { focus }), boundedDataAccess, history) { } + + private Partition GetFocus() + { + return State.Focus; + } + + private void SetFocus(Partition focus) + { + if (focus == null) throw new ArgumentNullException(nameof(focus)); +#if DEBUG + if (focus.Parts.Any() && BoundedDataAccess.Record.Fields.Length != focus.Parts.First().Intervals.Length) + throw new ArgumentException("intervals do not match"); +#endif + if (focus.Intersects(State.Forbidden)) throw new ArgumentException("focus cannot be expanded into previous focus space"); + var boundary = new Partition(new HashSet() { BoundedDataAccess.Record.Boundary }); + if (!boundary.Encloses(focus)) throw new ArgumentException("focus cannot be expanded into space that does not exist in the database"); + if (State.Focus != null && !focus.Encloses(State.Focus)) State.Forbidden = State.Forbidden.UnionWith(focus.Intersects(State.Focus) ? State.Focus.RemainderWithout(focus) : State.Focus); + State.Focus = focus; + + } + + public int NumberPartitions() + { + return State.History.Count; + } + + public int NumberPartitionsAfterClean() + { + return State.NumberPartitionsAfterCleanCount; + } + + public int BudgetIndex() => BoundedDataAccess.Record.GetBudgetIndex(); + + private bool BudgetGreaterEquals(Partition partition, long minBudget) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); +#endif + if (minBudget < 0) throw new ArgumentException("minBudget is negative"); + if (minBudget == 0) return true; + var b = minBudget - 1; + var newIntervals = new Interval[BoundedDataAccess.Record.Fields.Length]; + for (var i = 0; i < newIntervals.Length; i++) newIntervals[i] = i != BoundedDataAccess.Record.GetBudgetIndex() ? + new Interval(long.MinValue,long.MaxValue) : new Interval(0L, b); + return !partition.Intersects(new ContiguousPartition(newIntervals)); + } + + public PINQueryable CreatePinQueryable(IQueryable queryable, Partition partition, long epsilon) + { + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (partition.Count() == 0) throw new ArgumentException("partition contains no parts"); + if (!GetFocus().Encloses(partition)) throw new ArgumentException("partition is not fully in focus"); + if (!BudgetGreaterEquals(partition, epsilon)) throw new ArgumentException("not enough budget"); + var intersectionToBudget = new Dictionary(); + var overlapInformation = State.History.Overlapping(partition); + for (int i = 0; i < overlapInformation.Overlappings.Count; i++) + { + var intersection = overlapInformation.Intersections[i]; + var budget = overlapInformation.Overlappings[i].Epsilon + epsilon; + if (!BudgetGreaterEquals(intersection, budget)) throw new ArgumentException("not enough budget"); + if (intersectionToBudget.ContainsKey(intersection) && intersectionToBudget[intersection] > budget) budget = intersectionToBudget[intersection]; + intersectionToBudget[intersection] = budget; + } + State.History.Add(new HistoryEntry(partition, epsilon)); + foreach (var kv in intersectionToBudget) State.History.Add(new HistoryEntry(kv.Key, kv.Value)); + return new PINQueryable(BoundedDataAccess.BoundQueryable(queryable, partition), + new PINQAgentBudget(BoundedDataAccess.Record.Fields[BoundedDataAccess.Record.GetBudgetIndex()].FieldType.Convert(epsilon, true))); + } + + public EpsilonInformation MaxUsage(Partition partition) + { + if (partition == null) throw new ArgumentNullException(nameof(partition)); + var o = State.History.Overlapping(partition); + return !o.Overlappings.Any() ? new EpsilonInformation(null, 0L) : new EpsilonInformation(o.MaxPartition, o.MaxEpsilon); + } + + public void CleanHistory() + { + State.History.Clean(); + State.NumberPartitionsAfterCleanCount = State.History.Count; + } + + public IQueryable Materialize(IQueryable queryable, Partition partition) + { + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (partition.Count() == 0) throw new ArgumentException("partition contains no parts"); + return BoundedDataAccess.BoundQueryable(queryable, partition).ToArray().AsQueryable(); + } + + public void WriteDetailedBudgetUseToFile(string fileName) + { +#if DEBUG + if (fileName == null) throw new ArgumentNullException(nameof(fileName)); +#endif + State.History.WriteDetailedBudgetUseToFile(fileName, BoundedDataAccess, BoundedDataAccess.Record.Fields[BoundedDataAccess.Record.GetBudgetIndex()].FieldType); + } + + public void ExtendFocus(Partition extension) + { + if (extension == null) throw new ArgumentNullException(nameof(extension)); + SetFocus(GetFocus().UnionWith(extension)); + GetFocus().Clean(); + } + + public override string ToString() + { + var ret = "[BK: ...\n"; + ret += State.History.ToString(); + ret += "]\n"; + return ret; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Bookkeeper/BookkeeperState.cs b/source/C#/UniTraX/Core/Bookkeeper/BookkeeperState.cs new file mode 100644 index 0000000000000000000000000000000000000000..0c578b3cc796529a0e61e3592c60468a81232881 --- /dev/null +++ b/source/C#/UniTraX/Core/Bookkeeper/BookkeeperState.cs @@ -0,0 +1,13 @@ +namespace Qbb.Core.Bookkeeper +{ + using Qbb.Core.History; + using Qbb.Core.Partition; + + public class BookkeeperState + { + public Partition Focus; + public Partition Forbidden; + public IHistory History; + public int NumberPartitionsAfterCleanCount; + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Bookkeeper/EpsilonInformation.cs b/source/C#/UniTraX/Core/Bookkeeper/EpsilonInformation.cs new file mode 100644 index 0000000000000000000000000000000000000000..0dfdc3464ac18e64d42cc19bad95db02cee902f3 --- /dev/null +++ b/source/C#/UniTraX/Core/Bookkeeper/EpsilonInformation.cs @@ -0,0 +1,30 @@ +namespace Qbb.Core.Bookkeeper +{ + using Qbb.Core.Partition; + using System; + + public class EpsilonInformation + { + public Partition Partition { get; private set; } + public long Epsilon { get; private set; } + + public EpsilonInformation(Partition partition, long epsiplon) + { + // WARNING: Partition can be null if there is no partition + Partition = partition; + Epsilon = epsiplon; + } + + public override bool Equals(object obj) + { + if (!(obj is EpsilonInformation o)) return false; + if (!Partition.Equals(o.Partition)) return false; + return Epsilon == o.Epsilon; + } + + public override int GetHashCode() + { + throw new NotSupportedException("cannot get perfect hash of partition"); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/BoundedData/BoundedDataAccess.cs b/source/C#/UniTraX/Core/BoundedData/BoundedDataAccess.cs new file mode 100644 index 0000000000000000000000000000000000000000..abffab84e91b65c54ec670992dc558ce202704d5 --- /dev/null +++ b/source/C#/UniTraX/Core/BoundedData/BoundedDataAccess.cs @@ -0,0 +1,61 @@ +namespace Qbb.Core.BoundedData +{ + using Qbb.Core.History; + using Qbb.Core.Partition; + using Qbb.Core.Specification; + using System; + using System.Collections.Generic; + using System.Linq; + + public class BoundedDataAccess : ICounting + { + public Record Record { get; private set; } + public IQueryableFactory QueryableFactory { get; private set; } + private ExpressionProvider ExpressionProvider { get; set; } + + public BoundedDataAccess(Record record, IQueryableFactory queryableFactory, ExpressionProvider expressionProvider) + { +#if DEBUG + if (record == null) throw new ArgumentNullException(nameof(record)); + if (queryableFactory == null) throw new ArgumentNullException(nameof(queryableFactory)); + if (expressionProvider == null) throw new ArgumentNullException(nameof(expressionProvider)); +#endif + Record = record; + QueryableFactory = queryableFactory; + ExpressionProvider = expressionProvider; + } + + public IQueryable GetQueryable() + { + return QueryableFactory.Create(); + } + + public IQueryable BoundQueryable(IQueryable queryable, Partition partition) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (partition.Parts.Count == 0) throw new ArgumentException("partition contains no parts"); +#endif + return queryable.Where(ExpressionProvider.EnclosedIn(partition, Record, null)); + } + + public override string ToString() + { + return "[BDA: ...]"; + } + + public int CountWithin(ContiguousPartition contiguousPartition) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); +#endif + return BoundQueryable(GetQueryable(), new Partition(new HashSet() { contiguousPartition })).Count(); + } + + public int CountAll() + { + return CountWithin(Record.Boundary); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/BoundedData/ExpressionProvider.cs b/source/C#/UniTraX/Core/BoundedData/ExpressionProvider.cs new file mode 100644 index 0000000000000000000000000000000000000000..f8d5f38762152893054b294d1cb7afffff1ac745 --- /dev/null +++ b/source/C#/UniTraX/Core/BoundedData/ExpressionProvider.cs @@ -0,0 +1,61 @@ +namespace Qbb.Core.BoundedData +{ + using Qbb.Core.Partition; + using Qbb.Core.Specification; + using Qbb.Core.Utils; + using System; + using System.Linq; + using System.Linq.Expressions; + + public class ExpressionProvider + { + public Expression> EnclosedIn(Partition partition, Record record, ParameterExpression parameter) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (record == null) throw new ArgumentNullException(nameof(record)); +#endif + ParameterExpression currentParameter = parameter; + if (currentParameter == null) currentParameter = Expression.Parameter(typeof(R)); + return partition.Parts.Aggregate>>(null, (e, p) => + { + if (e == null) return EnclosedIn(p, currentParameter, record); + else return Expression.Lambda>(Expression.OrElse(e.Body, EnclosedIn(p, currentParameter, record).Body), currentParameter); + }); + } + + public Expression> EnclosedIn(ContiguousPartition contiguousPartition, ParameterExpression parameter, Record record) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (record == null) throw new ArgumentNullException(nameof(record)); + if (contiguousPartition.Intervals.Length != record.Fields.Length) throw new ArgumentException("intervals do not match"); +#endif + Expression expression = null; + ParameterExpression currentParameter = parameter; + if (currentParameter == null) currentParameter = Expression.Parameter(typeof(R)); + for (var i = 0; i < contiguousPartition.Intervals.Length; i++) + { + var field = record.Fields[i]; + var range = contiguousPartition.Intervals[i]; + var accessVisitor = new ReplaceExpressionVisitor(field.Access.Parameters[0], currentParameter); + var accessBody = accessVisitor.Visit(field.Access.Body); + + if (field.FieldType.Bounds.Low < range.Low) + { + var exp = Expression.GreaterThanOrEqual(accessBody, Expression.Constant(field.FieldType.Convert(range.Low, false))); + if (expression == null) expression = exp; + else expression = Expression.AndAlso(expression, exp); + } + if (range.High < field.FieldType.Bounds.High) + { + var exp = Expression.LessThanOrEqual(accessBody, Expression.Constant(field.FieldType.Convert(range.High, true))); + if (expression == null) expression = exp; + else expression = Expression.AndAlso(expression, exp); + } + } + if (expression == null) expression = Expression.Constant(true); + return Expression.Lambda>(expression, currentParameter); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/BoundedData/IQueryableFactory.cs b/source/C#/UniTraX/Core/BoundedData/IQueryableFactory.cs new file mode 100644 index 0000000000000000000000000000000000000000..b30a474bbfd40d2bf39ef828f893ea27b9125911 --- /dev/null +++ b/source/C#/UniTraX/Core/BoundedData/IQueryableFactory.cs @@ -0,0 +1,9 @@ +namespace Qbb.Core.BoundedData +{ + using System.Linq; + + public interface IQueryableFactory + { + IQueryable Create(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/History/HistoryEntry.cs b/source/C#/UniTraX/Core/History/HistoryEntry.cs new file mode 100644 index 0000000000000000000000000000000000000000..bf1a94048a8fce354b9ed6322bc6091f21b4eeb0 --- /dev/null +++ b/source/C#/UniTraX/Core/History/HistoryEntry.cs @@ -0,0 +1,31 @@ +namespace Qbb.Core.History +{ + using Qbb.Core.Partition; + using System; + + public class HistoryEntry + { + public Partition Partition { get; private set; } + public long Epsilon { get; private set; } + + public HistoryEntry(Partition partition, long epsilon) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); +#endif + Partition = partition; + Epsilon = epsilon; + } + + public override bool Equals(object obj) + { + if (!(obj is HistoryEntry o)) return false; + return Partition.Equals(o.Partition) && Epsilon == o.Epsilon; + } + + public override int GetHashCode() + { + throw new NotSupportedException("cannot get perfect hash of partition"); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/History/ICounting.cs b/source/C#/UniTraX/Core/History/ICounting.cs new file mode 100644 index 0000000000000000000000000000000000000000..d8b46729437bde98586ae6098f458ee1ac58f4eb --- /dev/null +++ b/source/C#/UniTraX/Core/History/ICounting.cs @@ -0,0 +1,10 @@ +namespace Qbb.Core.History +{ + using Qbb.Core.Partition; + + public interface ICounting + { + int CountWithin(ContiguousPartition contiguousPartition); + int CountAll(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/History/IHistory.cs b/source/C#/UniTraX/Core/History/IHistory.cs new file mode 100644 index 0000000000000000000000000000000000000000..1f5aca35e0f7cd93822629b8cb5d4446d2adf7b3 --- /dev/null +++ b/source/C#/UniTraX/Core/History/IHistory.cs @@ -0,0 +1,15 @@ +namespace Qbb.Core.History +{ + using Qbb.Core.Partition; + using Qbb.Core.Specification; + + public interface IHistory + { + int Count { get; } + void Add(HistoryEntry historyEntry); + void Clean(); + OverlapInformation Overlapping(Partition partition); + void WriteDetailedBudgetUseToFile(string fileName, ICounting counting, IFieldType budgetType); + string ToString(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/History/InMemoryHistory.cs b/source/C#/UniTraX/Core/History/InMemoryHistory.cs new file mode 100644 index 0000000000000000000000000000000000000000..e333f56c51e60902e4adf6975dd8b4297d32fd33 --- /dev/null +++ b/source/C#/UniTraX/Core/History/InMemoryHistory.cs @@ -0,0 +1,150 @@ +namespace Qbb.Core.History +{ + using Qbb.Core.Partition; + using Qbb.Core.Specification; + using System; + using System.Collections.Generic; + using System.Linq; + + public class InMemoryHistory : IHistory + { + public int BudgetIndex { get; private set; } + public int Count { get; private set; } + private SortedDictionary History; + private SortedDictionary WasExpanded; + private SortedDictionary NeedsCleaning; + + public InMemoryHistory(int budgetIndex) + { + BudgetIndex = budgetIndex; + Count = 0; + History = new SortedDictionary(); + WasExpanded = new SortedDictionary(); + NeedsCleaning = new SortedDictionary(); + } + + public void Add(HistoryEntry historyEntry) + { +#if DEBUG + if (historyEntry == null) throw new ArgumentNullException(nameof(historyEntry)); +#endif + var contains = History.TryGetValue(historyEntry.Epsilon, out Partition partition); + var previousPartitionSize = contains ? partition.Count() : 0; + var newPartition = contains ? partition.UnionWith(historyEntry.Partition) : historyEntry.Partition; + var newPartitionSize = newPartition.Count(); + History[historyEntry.Epsilon] = newPartition; + WasExpanded[historyEntry.Epsilon] = newPartition; + Count = Count + newPartitionSize - previousPartitionSize; + } + + public void Clean() + { + foreach (var kv in WasExpanded) + { + var preCount = kv.Value.Count(); + kv.Value.Clean(); + var postCount = kv.Value.Count(); + Count += postCount - preCount; + } + var wasExpanded = WasExpanded.ToArray(); + WasExpanded.Clear(); + foreach (var kv in wasExpanded) + { + var epsilon = kv.Key; + var partition = kv.Value; + var keysSmaller = History.Keys.ToList().Where(k => k < epsilon); + foreach (var k in keysSmaller) + { + var other = History[k]; + var preCount = other.Count(); + if (partition.Encloses(other)) + { + History.Remove(k); + if (NeedsCleaning.ContainsKey(k)) NeedsCleaning.Remove(k); + Count -= preCount; + continue; + } + if (!partition.Intersects(other)) continue; + other = other.RemainderWithout(partition); + var postCount = other.Count(); + if (postCount < 1) + { + History.Remove(k); + if (NeedsCleaning.ContainsKey(k)) NeedsCleaning.Remove(k); + Count -= preCount; + continue; + } + History[k] = other; + NeedsCleaning[k] = other; + Count += postCount - preCount; + } + NeedsCleaning[epsilon] = partition; + } + var needsCleaning = NeedsCleaning.ToArray(); + NeedsCleaning.Clear(); + foreach (var kv in needsCleaning) + { + var preCount = kv.Value.Count(); + kv.Value.Clean(); + var postCount = kv.Value.Count(); + Count += postCount - preCount; + } + } + + public OverlapInformation Overlapping(Partition partition) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); +#endif + var overlappings = new List(); + var intersections = new List(); + var maxEpsilon = -1L; + Partition maxPartition = null; + foreach (var kv in History) + { + if (!kv.Value.Intersects(partition)) continue; + overlappings.Add(new HistoryEntry(kv.Value, kv.Key)); + if (kv.Key == maxEpsilon) throw new Exception("history has same epsilon key twice. should never happen."); + var intersection = kv.Value.IntersectionWith(partition); + intersections.Add(intersection); + if (kv.Key < maxEpsilon) continue; + maxEpsilon = kv.Key; + maxPartition = intersection; + } + if (maxEpsilon < 0L) maxEpsilon = 0L; + return new OverlapInformation(overlappings, intersections, maxEpsilon, maxPartition); + } + + public void WriteDetailedBudgetUseToFile(string fileName, ICounting counting, IFieldType budgetType) + { +#if DEBUG + if (fileName == null) throw new ArgumentNullException(nameof(fileName)); + if (counting == null) throw new ArgumentNullException(nameof(counting)); + if (budgetType == null) throw new ArgumentNullException(nameof(budgetType)); +#endif + Clean(); + using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName)) + { + var totalCount = 0; + foreach(KeyValuePair kvp in History) + { + var count = 0; + foreach (ContiguousPartition cp in kvp.Value.Parts) count += counting.CountWithin(cp); + file.WriteLine(budgetType.ConvertUnchecked(kvp.Key, true) + "," + count); + totalCount += count; + } + var dataCount = counting.CountAll(); + if (dataCount > totalCount) file.WriteLine(budgetType.ConvertUnchecked(0L, true) + "," + (dataCount - totalCount)); + } + } + + public override string ToString() + { + var ret = History.Aggregate("[IMH: ...\n", + (current, kv) => current + ("\t[Budget:" + kv.Key + "; #ContP:" + kv.Value.Count() + "]\n")); + // + _recordSpec; + ret += "]\n"; + return ret; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/History/OverlapInformation.cs b/source/C#/UniTraX/Core/History/OverlapInformation.cs new file mode 100644 index 0000000000000000000000000000000000000000..1937bbe7d486f633c3445e543317cd306eef1208 --- /dev/null +++ b/source/C#/UniTraX/Core/History/OverlapInformation.cs @@ -0,0 +1,52 @@ +namespace Qbb.Core.History +{ + using Qbb.Core.Partition; + using System; + using System.Collections.Generic; + using System.Linq; + + public class OverlapInformation + { + public List Overlappings { get; private set; } + public List Intersections { get; private set; } + public long MaxEpsilon { get; private set; } + public Partition MaxPartition { get; private set; } + + public OverlapInformation(List overlappings, List intersections, long maxEpsilon, Partition maxPartition) + { +#if DEBUG + if (overlappings == null) throw new ArgumentNullException(nameof(overlappings)); + if (intersections == null) throw new ArgumentNullException(nameof(intersections)); + if (overlappings.Count != intersections.Count) throw new ArgumentException("must be an intersection for every overlap and vice versa"); + if (maxPartition != null && maxPartition.Parts.Any()) + { + foreach(HistoryEntry h in overlappings) + { + if (h.Partition != null && h.Partition.Parts.Any()) + { + if (h.Partition.Parts.First().Intervals.Length != maxPartition.Parts.First().Intervals.Length) + throw new ArgumentException("intervals do not match"); + } + } + } +#endif + Overlappings = overlappings; + Intersections = intersections; + MaxEpsilon = maxEpsilon; + MaxPartition = maxPartition; + } + + public override bool Equals(object obj) + { + if (!(obj is OverlapInformation o)) return false; + if (Overlappings.Any(overlap => !o.Overlappings.Contains(overlap))) return false; + if (Intersections.Any(intersection => !o.Intersections.Contains(intersection))) return false; + return MaxEpsilon == o.MaxEpsilon && MaxPartition.Equals(o.MaxPartition); + } + + public override int GetHashCode() + { + throw new NotSupportedException("cannot get perfect hash of partition"); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Partition/ContiguousPartition.cs b/source/C#/UniTraX/Core/Partition/ContiguousPartition.cs new file mode 100644 index 0000000000000000000000000000000000000000..d38e199b3ff47a56e28430b91cad34ea89376592 --- /dev/null +++ b/source/C#/UniTraX/Core/Partition/ContiguousPartition.cs @@ -0,0 +1,426 @@ +namespace Qbb.Core.Partition +{ + using Qbb.Core.Utils; + using System; + using System.Collections.Generic; + using System.Linq; + + public struct IndexedValue + { + public int Index { get; private set; } + public long Value { get; private set; } + + public IndexedValue(int index, long value) + { + Index = index; + Value = value; + } + } + + public class ContiguousPartitionComparer : Comparer + { + public override int Compare(ContiguousPartition x, ContiguousPartition y) + { +#if DEBUG + if (x == null) throw new ArgumentNullException(nameof(x)); + if (y == null) throw new ArgumentNullException(nameof(y)); + if (x.Intervals == null) throw new ArgumentNullException(nameof(x.Intervals)); + if (y.Intervals == null) throw new ArgumentNullException(nameof(y.Intervals)); + if (x.Intervals.Length != y.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + for (int i = 0; i < x.Intervals.Length; i++) + { + int c = x.Intervals[i].Low.CompareTo(y.Intervals[i].Low); + if (c != 0) return c; + c = x.Intervals[i].High.CompareTo(y.Intervals[i].High); + if (c != 0) return c; + } + return 0; + } + } + + public class ContiguousPartition + { + public Interval[] Intervals { get; private set; } + public int HashCode { get; private set; } +#if DEBUG + public int[] HashCodes { get; private set; } +#endif + + + public ContiguousPartition(Interval[] intervals) + { +#if DEBUG + if (intervals == null) throw new ArgumentNullException(nameof(intervals)); + if (!intervals.Any()) throw new ArgumentException("no intervals defined"); + HashCodes = new int[intervals.Length]; + for (int i = 0; i < intervals.Length; i++) HashCodes[i] = intervals[i].GetHashCode(); +#endif + Intervals = intervals; + HashCode = GenerateHashCode(); + } + + private bool IsZeroSize() + { + return Intervals.All(f => !(f.Length() > 0)); + } + + #region ContainsData + public bool ContainsDataLessThan(IndexedValue iVal) + { +#if DEBUG + if (iVal.Index < 0 || iVal.Index > Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + return Intervals[iVal.Index].ContainsDataLessThan(iVal.Value); + } + + public bool ContainsDataLessEqual(IndexedValue iVal) + { +#if DEBUG + if (iVal.Index < 0 || iVal.Index > Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + return Intervals[iVal.Index].ContainsDataLessEqual(iVal.Value); + } + + public bool ContainsDataGreaterThan(IndexedValue iVal) + { +#if DEBUG + if (iVal.Index < 0 || iVal.Index > Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + return Intervals[iVal.Index].ContainsDataGreaterThan(iVal.Value); + } + + public bool ContainsDataGreaterEqual(IndexedValue iVal) + { +#if DEBUG + if (iVal.Index < 0 || iVal.Index > Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + return Intervals[iVal.Index].ContainsDataGreaterEqual(iVal.Value); + } + #endregion + + #region Enclosure + public bool Encloses(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (Intervals.Length != other.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + return !Intervals.Where((t, i) => !t.Encloses(other.Intervals[i])).Any(); + } + + public bool Encloses(IndexedValue iVal) + { +#if DEBUG + if (iVal.Index < 0 || iVal.Index > Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + return Intervals[iVal.Index].Encloses(iVal.Value); + } + #endregion + + #region Intersection + public bool Intersects(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (Intervals.Length != other.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + return !Intervals.Where((t, i) => !t.Intersects(other.Intervals[i])).Any(); + } + + public ContiguousPartition IntersectionWith(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (Intervals.Length != other.Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!Intersects(other)) throw new ArgumentException("partitions do not intersect"); +#endif + var newIntervals = new Interval[Intervals.Length]; + for (int i = 0; i < Intervals.Length; i++) newIntervals[i] = Intervals[i].IntersectionWith(other.Intervals[i]); + return new ContiguousPartition(newIntervals); + } + #endregion + + #region Split & Remainder + public ContiguousPartition[] Halve() + { + Interval longest = null; + var index = -1; + for (var i = 0; i < Intervals.Length; i++) + if (longest == null || Intervals[i].Length() > longest.Length()) + { + longest = Intervals[i]; + index = i; + } + if (longest != null) return SplitAt(new IndexedValue(index, longest.SmallerMiddle())); + throw new InvalidOperationException("should never happen"); + } + + public ContiguousPartition[] Halve(ICollection ignore) + { + Interval longest = null; + var index = -1; + for (var i = 0; i < Intervals.Length; i++) + if (!ignore.Contains(i) && (longest == null || Intervals[i].Length() > longest.Length())) + { + longest = Intervals[i]; + index = i; + } + if (longest != null) return SplitAt(new IndexedValue(index, longest.SmallerMiddle())); + throw new InvalidOperationException("should never happen"); + } + + public ContiguousPartition[] SplitAt(IndexedValue iVal) + { +#if DEBUG + if (iVal.Index < 0 || iVal.Index > Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); + if (!Encloses(iVal)) throw new ArgumentException("value not inside partition"); + if (IsZeroSize()) throw new InvalidOperationException("cannot split zero size partition"); +#endif + var intervalsLeft = new Interval[Intervals.Length]; + var intervalsRight = new Interval[Intervals.Length]; + Array.Copy(Intervals, intervalsLeft, Intervals.Length); + Array.Copy(Intervals, intervalsRight, Intervals.Length); + + var intervalsSplit = Intervals[iVal.Index].SplitAt(iVal.Value); + intervalsLeft[iVal.Index] = intervalsSplit[0]; + intervalsRight[iVal.Index] = intervalsSplit[1]; + + return new[] + { + new ContiguousPartition(intervalsLeft), + new ContiguousPartition(intervalsRight) + }; + } + + public bool HasSingleRemainderWithout(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (Intervals.Length != other.Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!Intersects(other)) throw new ArgumentException("partitions do not intersect"); + if (other.Encloses(this)) throw new ArgumentException("other encloses this; no remainder possible"); +#endif + var countSingle = 0; + for (int i = 0; i < Intervals.Length; i++) + { + if (Intervals[i].HasSingleRemainderWithout(other.Intervals[i])) + { + countSingle++; + if (countSingle > 1) return false; + } else if (!Intervals[i].Encloses(other.Intervals[i]) && other.Intervals[i].Encloses(Intervals[i])) return false; + } + return true; + } + + public IndexedValue SplitPointForLargestRemainderWithout(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (Intervals.Length != other.Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!Intersects(other)) throw new ArgumentException("partitions do not intersect"); + if (other.Encloses(this)) throw new ArgumentException("other encloses this; no remainder possible"); +#endif + var maxLength = 0.0; + var maxIndex = -1; + for (var i = 0; i < Intervals.Length; i++) + { + if (other.Intervals[i].Encloses(Intervals[i]) || !(Intervals[i].LengthForLargestRemainderWithout(other.Intervals[i]) >= maxLength)) continue; + maxLength = Intervals[i].LengthForLargestRemainderWithout(other.Intervals[i]); + maxIndex = i; + } + + return new IndexedValue(maxIndex, Intervals[maxIndex].SplitPointForLargestRemainderWithout(other.Intervals[maxIndex])); + } + #endregion + + #region Merge + public bool IsMergeableWith(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (Intervals.Length != other.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + var countMergeable = 0; + for (int i = 0; i < Intervals.Length; i++) + { + if (!Intervals[i].Equals(other.Intervals[i])) + { + if (!Intervals[i].IsMergeableWith(other.Intervals[i])) return false; + countMergeable++; + if (countMergeable > 1) return false; + } + } + return true; + } + + public ContiguousPartition MergeWith(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (Intervals.Length != other.Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!IsMergeableWith(other)) throw new ArgumentException("partitions are not mergeable"); +#endif + if (Encloses(other)) return this; + if (other.Encloses(this)) return other; + var intervals = new Interval[Intervals.Length]; + Array.Copy(Intervals, intervals, Intervals.Length); + for (var i = 0; i < Intervals.Length; i++) + { + if (Intervals[i].Equals(other.Intervals[i])) continue; + intervals[i] = Intervals[i].MergeWith(other.Intervals[i]); + break; + } + return new ContiguousPartition(intervals); + } + #endregion + + #region Standard functions + public override bool Equals(object obj) + { + return obj is ContiguousPartition o && Equals(o); + } + + public bool Equals(ContiguousPartition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); +#endif + if (HashCode != other.HashCode) return false; + if (Intervals.Length != other.Intervals.Length) return false; + for (int i = 0; i < Intervals.Length; i++) if (!Intervals[i].Equals(other.Intervals[i])) return false; + return true; + } + + public override int GetHashCode() + { + return HashCode; + } + + private int GenerateHashCode() + { + return Intervals.Aggregate(Hash.GenerateHash(), (current, i) => Hash.MergeHashes(current, i.GetHashCode())); + } + + public override string ToString() + { + var ret = "[CP:"; + var first = true; + foreach (var interval in Intervals) + { + if (!first) ret = ret + ";"; + ret = ret + "\n" + interval; + first = false; + } + ret = ret + "\n]"; + return ret; + } + #endregion + + #region k-means + + public static IDictionary> Kmeans( + ContiguousPartition contiguousPartition, ISet centroids, + IDictionary ignoreAndReplace) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (centroids == null) throw new ArgumentNullException(nameof(centroids)); + if (!centroids.Any()) throw new ArgumentException("need at least one centroid"); +#endif + var todo = new List { contiguousPartition }; + return Kmeans(todo, centroids, null, ignoreAndReplace); + } + + public static IDictionary> Kmeans( + IList todo, ISet centroids, + IDictionary> done, + IDictionary ignoreAndReplace) + { + while (true) + { + if (todo == null || !todo.Any()) + return done; +#if DEBUG + if (centroids == null) throw new ArgumentNullException(nameof(centroids)); + if (!centroids.Any()) throw new ArgumentException("need at least one centroid"); +#endif + if (done == null) done = new Dictionary>(); + var next = todo.First(); + todo.RemoveAt(0); + var closest = ClosestTo(next, centroids, ignoreAndReplace); + if (closest != null) + { + if (!done.ContainsKey(closest)) done.Add(closest, new HashSet()); + done[closest].Add(next); + } + else + { + var halves = next.Halve(ignoreAndReplace.Keys); + todo.Add(halves[0]); + todo.Add(halves[1]); + } + } + } + + private static Point ClosestTo(ContiguousPartition contiguousPartition, ISet centroids, IDictionary ignoreAndReplace) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (centroids == null) throw new ArgumentNullException(nameof(centroids)); + if (!centroids.Any()) throw new ArgumentException("need at least one centroid"); +#endif + var corners = CornersOf(contiguousPartition, ignoreAndReplace); + Point closest = null; + foreach (var c in corners) + { + var cl = Point.ClosestTo(c, centroids); + if (closest == null) closest = cl; + if (!closest.Equals(cl)) return null; + } + return closest; + } + + private static ISet CornersOf(ContiguousPartition contiguousPartition, IDictionary ignoreAndReplace) + { + var unconfigured = new List(); + for (var i = 0; i < contiguousPartition.Intervals.Length; i++) + { + unconfigured.Add(ignoreAndReplace.ContainsKey(i) + ? ignoreAndReplace[i] + : contiguousPartition.Intervals[i]); + } + return CornersOf(null, unconfigured); + } + + private static ISet CornersOf(IList configured, IList unconfigured) + { +#if DEBUG + if ((configured == null || !configured.Any()) && (unconfigured == null || !unconfigured.Any())) + throw new ArgumentException("neither configured nor unconfigured present"); +#endif + if (unconfigured == null || !unconfigured.Any()) + return new HashSet() { new Point(configured.ToArray()) }; + var next = unconfigured.First(); + unconfigured.RemoveAt(0); + var a = new List(); + if (configured != null) a.AddRange(configured); + a.Add(new Interval(next.Low, next.Low)); + var au = new List(); + au.AddRange(unconfigured); + var s = CornersOf(a, au); + if (next.Length() == 0) return s; + var b = new List(); + if (configured != null) b.AddRange(configured); + b.Add(new Interval(next.High, next.High)); + var bu = new List(); + bu.AddRange(unconfigured); + var s2 = CornersOf(b, bu); + var ret = new HashSet(); + ret.UnionWith(s); + ret.UnionWith(s2); + return ret; + } + #endregion + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Partition/Interval.cs b/source/C#/UniTraX/Core/Partition/Interval.cs new file mode 100644 index 0000000000000000000000000000000000000000..adee9de54b81ac3663b91c0fa53328bc00818e05 --- /dev/null +++ b/source/C#/UniTraX/Core/Partition/Interval.cs @@ -0,0 +1,196 @@ +namespace Qbb.Core.Partition +{ + using Qbb.Core.Utils; + using System; + + public class Interval + { + public long Low { get; private set; } + public long High { get; private set; } + public int HashCode { get; private set; } + +#if DEBUG + public int LowHash { get; private set; } + public int HighHash { get; private set; } +#endif + + public Interval(long low, long high) + { +#if DEBUG + if (low > high) throw new ArgumentException("low is greater than high"); + LowHash = low.GetHashCode(); + HighHash = high.GetHashCode(); +#endif + Low = low; + High = high; + HashCode = GenerateHashCode(); + } + + public long Length() + { + return High - Low; + } + + public long SmallerMiddle() + { + return Low + Length() / 2; + } + + #region ContainsData + public bool ContainsDataLessThan(long value) + { + return Low < value; + } + + public bool ContainsDataLessEqual(long value) + { + return Low <= value; + } + + public bool ContainsDataGreaterThan(long value) + { + return value < High; + } + + public bool ContainsDataGreaterEqual(long value) + { + return value <= High; + } + #endregion + + #region Enclosure + public bool Encloses(Interval other) + { + return Low <= other.Low && other.High <= High; + } + + public bool Encloses(long value) + { + return Low <= value && value <= High; + } + #endregion + + #region Intersection + public bool Intersects(Interval other) + { + return !(other.High < Low || High < other.Low); + } + + public Interval IntersectionWith(Interval other) + { +#if DEBUG + if (!Intersects(other)) throw new ArgumentException("intervals do not intersect"); +#endif + var newLow = Low; + var newHigh = High; + if (newLow < other.Low) newLow = other.Low; + if (other.High < newHigh) newHigh = other.High; + return new Interval(newLow, newHigh); + } + #endregion + + #region Split & Remainder + public Interval[] SplitAt(long value) + { +#if DEBUG + if (!Encloses(value)) throw new ArgumentException("value is outside the interval"); + if (value == High) throw new ArgumentException("value must be below high"); +#endif + return new[] { new Interval(Low, value), new Interval(value + 1, High) }; + } + + public bool HasSingleRemainderWithout(Interval other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (!Intersects(other)) throw new ArgumentException("intervals do not intersect"); + if (other.Encloses(this)) throw new ArgumentException("other encloses this; no remainder possible"); +#endif + return !Encloses(other) || Low == other.Low || other.High == High; + } + + public long SplitPointForLargestRemainderWithout(Interval other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (!Intersects(other)) throw new ArgumentException("intervals do not intersect"); + if (other.Encloses(this)) throw new ArgumentException("other encloses this; no remainder possible"); +#endif + if (Low < other.Low) + { + if (other.High < High) return other.Low - 1 - Low > High - other.High - 1 ? other.Low - 1 : other.High; + return other.Low - 1; + } + return other.High; + } + + public double LengthForLargestRemainderWithout(Interval other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (!Intersects(other)) throw new ArgumentException("intervals do not intersect"); + if (other.Encloses(this)) throw new ArgumentException("other encloses this; no remainder possible"); +#endif + if (Low < other.Low) + { + if (other.High < High) return other.Low - 1 - Low > High - other.High - 1 ? other.Low - 1 - Low : High - other.High - 1; + return other.Low - 1 - Low; + } + return High - other.High - 1; + } + #endregion + + #region Merge + public bool IsMergeableWith(Interval other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); +#endif + if (Intersects(other)) return true; + return other.High == Low - 1 || High + 1 == other.Low; + } + + public Interval MergeWith(Interval other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (!IsMergeableWith(other)) throw new ArgumentException("intervals are not mergeable"); +#endif + var newLow = Low; + var newHigh = High; + if (other.Low < Low) newLow = other.Low; + if (High < other.High) newHigh = other.High; + return new Interval(newLow, newHigh); + } + #endregion + + public override bool Equals(object obj) + { + return obj is Interval o && Equals(o); + } + + public bool Equals(Interval other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); +#endif + if (HashCode != other.HashCode) return false; + return Low == other.Low && High == other.High; + } + + private int GenerateHashCode() + { + return Hash.GenerateHash(new int[] { (Low + 4611686018427387776L).GetHashCode(), (High + 4611686018427387776L).GetHashCode() }); + } + + public override int GetHashCode() + { + return HashCode; + } + + public override string ToString() + { + return "[I:" + Low + ";" + High + "]"; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Partition/Partition.cs b/source/C#/UniTraX/Core/Partition/Partition.cs new file mode 100644 index 0000000000000000000000000000000000000000..dd64da69bee5e8832d639d7a718c2c3c38ac4088 --- /dev/null +++ b/source/C#/UniTraX/Core/Partition/Partition.cs @@ -0,0 +1,410 @@ +namespace Qbb.Core.Partition +{ + using Qbb.Core.Utils; + using System; + using System.Collections.Generic; + using System.Linq; + + public class Partition + { + public ISet Parts { get; private set; } + private ISet Changed { get; set; } + + public Partition(ISet parts) + { +#if DEBUG + if (parts == null) throw new ArgumentNullException(nameof(parts)); + if (parts.Any(part => part.Intervals.Length != parts.First().Intervals.Length)) + throw new ArgumentException("intervals of parts do not match"); +#endif + Parts = parts; + Changed = new HashSet(); + Changed.UnionWith(Parts); + } + + public int Count() + { + return Parts.Count; + } + + #region Enclosure + public bool Encloses(Partition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Parts.Any() && Parts.First().Intervals.Length != other.Parts.First().Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + return other.Parts.All(Encloses); + } + + private bool Encloses(ContiguousPartition contiguousPartition) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (Parts.First().Intervals.Length != contiguousPartition.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + return Encloses(new List { contiguousPartition }); + } + + private bool Encloses(IList parts) + { +#if DEBUG + if (parts == null) throw new ArgumentNullException(nameof(parts)); +#endif + if (!parts.Any()) return true; + while (true) + { + var last = parts.Last(); + parts.RemoveAt(parts.Count - 1); + var intersects = new List(); + foreach (var p in Parts) + { + if (p.Encloses(last)) return Encloses(parts); + if (p.Intersects(last)) intersects.Add(p); + } + if (intersects.Count < 2) return false; + var splitPoint = last.SplitPointForLargestRemainderWithout(intersects[0]); + var splits = last.SplitAt(splitPoint); + parts.Add(splits[0]); + parts.Add(splits[1]); + } + } + #endregion + + #region Intersection + public bool Intersects(Partition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Parts.Any() && Parts.First().Intervals.Length != other.Parts.First().Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + return other.Parts.Any(Intersects); + } + + public bool Intersects(ContiguousPartition contiguousPartition) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (Parts.First().Intervals.Length != contiguousPartition.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + return Parts.Any(p => p.Intersects(contiguousPartition)); + } + + public Partition IntersectionWith(Partition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Parts.Any() && Parts.First().Intervals.Length != other.Parts.First().Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!Intersects(other)) throw new ArgumentException("partitions do not intersect"); +#endif + var newParts = new HashSet(); + foreach (var p in Parts) + { + foreach (var op in other.Parts) + { + if (p.Intersects(op)) newParts.Add(p.IntersectionWith(op)); + } + } + var newPartition = new Partition(newParts); + return newPartition; + } + + private Partition IntersectionWith(ContiguousPartition contiguousPartition) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (Parts.First().Intervals.Length != contiguousPartition.Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!Intersects(contiguousPartition)) throw new ArgumentException("partitions do not intersect"); +#endif + var newParts = new HashSet(); + foreach (var p in Parts) + { + if (p.Intersects(contiguousPartition)) newParts.Add(p.IntersectionWith(contiguousPartition)); + } + var newPartition = new Partition(newParts); + return newPartition; + } + #endregion + + #region Union + public Partition UnionWith(Partition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Parts.Any() && Parts.First().Intervals.Length != other.Parts.First().Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + var newParts = new HashSet(Parts); + newParts.UnionWith(other.Parts); + return new Partition(newParts); + } + + private void Add(ContiguousPartition contiguousPartition) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (Parts.First().Intervals.Length != contiguousPartition.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + if (Parts.Contains(contiguousPartition)) return; + + /** + * too expensive + if (Encloses(contiguousPartition)) return; + + Partition toAdd = new Partition(new HashSet() { contiguousPartition }); + if (Intersects(toAdd)) toAdd = toAdd.RemainderWithout(this); + if (toAdd.Count() == 0) return; + foreach (ContiguousPartition part in toAdd.Parts) + { + Parts.Add(part); + Changed.Add(part); + } + **/ + Parts.Add(contiguousPartition); + Changed.Add(contiguousPartition); + } + + public void Clean() + { + while (true) + { + if (Parts.Count < 2) + { + Changed.Clear(); + return; + } + var merged = false; + var intersected = false; + ContiguousPartition part = null; + ContiguousPartition changed = null; + HashSet toAdd = null; + while (!merged && !intersected && Changed.Count > 0) + { + changed = Changed.First(); + if (!Changed.Remove(changed)) throw new Exception("shouldn't happen"); + foreach (var p in Parts) + { + if (p.Equals(changed)) continue; + if (p.IsMergeableWith(changed)) + { + part = p; + toAdd = new HashSet() { p.MergeWith(changed) }; + merged = true; + break; + } + else if (p.Intersects(changed)) + { + part = p; + var newParts = new HashSet(); + var oldParts = new List(); + oldParts.Add(changed); + RemainderOf(oldParts, newParts, p); + toAdd = newParts; + intersected = true; + break; + } + } + } + if (!merged && !intersected) + { + if (Changed.Count > 0) throw new Exception("shouldn't happen"); + return; + } + if (merged) + { + Parts.Remove(part); + Changed.Remove(part); + } + Parts.Remove(changed); + Parts.UnionWith(toAdd); + Changed.UnionWith(toAdd); + } + } + #endregion + + #region Split & Remainder + public Partition RemainderWithout(Partition other) + { +#if DEBUG + if (other == null) throw new ArgumentNullException(nameof(other)); + if (other.Parts.Any() && Parts.First().Intervals.Length != other.Parts.First().Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!Intersects(other)) throw new ArgumentException("partitions do not intersect"); + if (other.Encloses(this)) throw new ArgumentException("other encloses this partition; no remainder possible"); +#endif + var ret = this; + foreach (var p in other.Parts) if (ret.Intersects(p)) ret = ret.RemainderWithout(p); + return ret; + } + + private Partition RemainderWithout(ContiguousPartition contiguousPartition) + { +#if DEBUG + if (contiguousPartition == null) throw new ArgumentNullException(nameof(contiguousPartition)); + if (Parts.First().Intervals.Length != contiguousPartition.Intervals.Length) throw new ArgumentException("intervals do not match"); + if (!Intersects(contiguousPartition)) throw new ArgumentException("partitions do not intersect"); + var enc = true; + foreach (var p in Parts) if (!contiguousPartition.Encloses(p)) enc = false; + if (enc) throw new ArgumentException("contiguousPartition encloses this partition; no remainder possible"); +#endif + var newParts = new HashSet(); + var oldParts = new List(); + oldParts.AddRange(Parts); + RemainderOf(oldParts, newParts, contiguousPartition); + return new Partition(newParts); + } + + private void RemainderOf(List inParts, ISet outParts, ContiguousPartition withoutPartition) + { +#if DEBUG + if (inParts == null) throw new ArgumentNullException(nameof(inParts)); + if (outParts == null) throw new ArgumentNullException(nameof(outParts)); + if (withoutPartition == null) throw new ArgumentNullException(nameof(withoutPartition)); +#endif + while (true) + { + if (!inParts.Any()) return; + var last = inParts.Last(); + inParts.RemoveAt(inParts.Count - 1); + if (withoutPartition.Encloses(last)) continue; + if (!last.Intersects(withoutPartition)) + { + outParts.Add(last); + continue; + } + var splitPoint = last.SplitPointForLargestRemainderWithout(withoutPartition); + var splits = last.SplitAt(splitPoint); + if (!splits[0].Intersects(withoutPartition)) + { + outParts.Add(splits[0]); + inParts.Add(splits[1]); + continue; + } + if (splits[1].Intersects(withoutPartition)) + throw new InvalidOperationException("this should never happen"); + outParts.Add(splits[1]); + inParts.Add(splits[0]); + } + } + #endregion + + #region Standard functions + public override bool Equals(object obj) + { + if (!(obj is Partition o)) return false; + return Encloses(o) && o.Encloses(this); + } + + // WARNING: One cannot get a perfect hash for a partition. Some partitions with different hashes can still be equal. + public override int GetHashCode() + { + var hash = Hash.GenerateHash(); + return Parts.Aggregate(hash, (current, p) => Hash.MergeHashes(current, p.GetHashCode())); + } + + public override string ToString() + { + var ret = "[P: ...\n"; // + Record; + ret = Parts.Aggregate(ret, (current, p) => current + ";\n" + p); + ret = ret + "\n]"; + return ret; + } + #endregion + + #region Projection + public static Partition Equals(Partition partition, IndexedValue iVal, Interval bounds) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (bounds == null) throw new ArgumentNullException(nameof(bounds)); + if (!partition.Parts.Any()) throw new ArgumentException("partition is empty"); + if (iVal.Index > partition.Parts.First().Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + return GreaterEqual(LessEqual(partition, iVal), iVal, bounds); + } + + public static Partition LessThan(Partition partition, IndexedValue iVal, Interval bounds) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (bounds == null) throw new ArgumentNullException(nameof(bounds)); + if (!partition.Parts.Any()) throw new ArgumentException("partition is empty"); + if (iVal.Index > partition.Parts.First().Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + var newParts = new HashSet(); + foreach (var p in partition.Parts) + { + if (!p.ContainsDataLessThan(iVal)) continue; + var bound = bounds.Low < iVal.Value ? iVal.Value - 1 : iVal.Value; + var newIVal = new IndexedValue(iVal.Index, bound); + newParts.Add(!p.ContainsDataGreaterEqual(iVal) ? p : p.SplitAt(newIVal)[0]); + } + return new Partition(newParts); + } + + public static Partition LessEqual(Partition partition, IndexedValue iVal) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (!partition.Parts.Any()) throw new ArgumentException("partition is empty"); + if (iVal.Index > partition.Parts.First().Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + var newParts = new HashSet(); + foreach (var p in partition.Parts) + { + if (!p.ContainsDataLessEqual(iVal)) continue; + newParts.Add(!p.ContainsDataGreaterThan(iVal) ? p : p.SplitAt(iVal)[0]); + } + return new Partition(newParts); + } + + public static Partition GreaterThan(Partition partition, IndexedValue iVal) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (!partition.Parts.Any()) throw new ArgumentException("partition is empty"); + if (iVal.Index > partition.Parts.First().Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + var newParts = new HashSet(); + foreach (var p in partition.Parts) + { + if (!p.ContainsDataGreaterThan(iVal)) continue; + newParts.Add(!p.ContainsDataLessEqual(iVal) ? p : p.SplitAt(iVal)[1]); + } + return new Partition(newParts); + } + + public static Partition GreaterEqual(Partition partition, IndexedValue iVal, Interval bounds) + { +#if DEBUG + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (bounds == null) throw new ArgumentNullException(nameof(bounds)); + if (!partition.Parts.Any()) throw new ArgumentException("partition is empty"); + if (iVal.Index > partition.Parts.First().Intervals.Length - 1) throw new IndexOutOfRangeException("index out of range"); +#endif + var newParts = new SortedSet(new ContiguousPartitionComparer()); + foreach (var p in partition.Parts) + { + if (!p.ContainsDataGreaterEqual(iVal)) continue; + var bound = bounds.Low < iVal.Value ? iVal.Value - 1 : iVal.Value; + var newIVal = new IndexedValue(iVal.Index, bound); + newParts.Add(!p.ContainsDataLessThan(iVal) ? p : p.SplitAt(newIVal)[1]); + } + return new Partition(newParts); + } + #endregion + + #region k-means + + public static IDictionary Kmeans(Partition partition, ISet centroids, IDictionary ignoreAndReplace) + { + if (partition == null) throw new ArgumentNullException(nameof(partition)); + if (centroids == null) throw new ArgumentNullException(nameof(centroids)); + if (!centroids.Any()) throw new ArgumentException("need at least one centroid"); + var todo = new List(); + todo.AddRange(partition.Parts); + var split = ContiguousPartition.Kmeans(todo, centroids, null, ignoreAndReplace); + var d = split.ToDictionary(s => s.Key, s => new Partition(s.Value)); + return d; + } + #endregion + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Partition/Point.cs b/source/C#/UniTraX/Core/Partition/Point.cs new file mode 100644 index 0000000000000000000000000000000000000000..913990aa2ab78118920c97823166273fca93c7cc --- /dev/null +++ b/source/C#/UniTraX/Core/Partition/Point.cs @@ -0,0 +1,36 @@ +namespace Qbb.Core.Partition +{ + using System; + using System.Collections.Generic; + using System.Linq; + + public class Point : ContiguousPartition + { + public Point(Interval[] intervals) : base(intervals) + { + if (intervals.Any(f => f.Length() > 0)) throw new ArgumentException("intervals must be of zero step size"); + } + + public static Point ClosestTo(Point a, ISet others) + { + if (a == null) throw new ArgumentNullException(nameof(a)); + if (others == null) throw new ArgumentNullException(nameof(others)); + if (!others.Any()) throw new ArgumentException("need at least one other point"); + Point closest = null; + foreach (var o in others) if (closest == null || DistanceBetween(a, o) < DistanceBetween(a, closest)) closest = o; + return closest; + } + + public static double DistanceBetween(Point a, Point b) + { + if (a == null) throw new ArgumentNullException(nameof(a)); + if (b == null) throw new ArgumentNullException(nameof(b)); +#if DEBUG + if (a.Intervals.Length != b.Intervals.Length) throw new ArgumentException("intervals do not match"); +#endif + var sum = 0.0; + for (var i = 0; i < a.Intervals.Length; i++) sum += Math.Pow((double)a.Intervals[i].Low - b.Intervals[i].Low, 2.0); + return Math.Sqrt(sum); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Specification/Field.cs b/source/C#/UniTraX/Core/Specification/Field.cs new file mode 100644 index 0000000000000000000000000000000000000000..a260cbf53e59cf4833e3134067bdeabc89aca83a --- /dev/null +++ b/source/C#/UniTraX/Core/Specification/Field.cs @@ -0,0 +1,36 @@ +namespace Qbb.Core.Specification +{ + using System; + using System.Linq.Expressions; + + public class Field + { + public int Index { get; private set; } + public string Name { get; private set; } + public IFieldType FieldType { get; private set; } + public Expression> Access { get; private set; } + + public Field(int index, string name, IFieldType fieldType, Expression> access) + { +#if DEBUG + if (index < 0) throw new ArgumentException("index below 0"); + if (name == null) throw new ArgumentNullException(nameof(name)); + if (fieldType == null) throw new ArgumentNullException(nameof(fieldType)); +#endif + Index = index; + Name = name; + FieldType = fieldType; + Access = access; + } + + public Field Derive(Expression> access) + { + return new Field(Index, Name, FieldType, access); + } + + public override string ToString() + { + return "[F:" + Index + ":" + Name + ";" + FieldType.Bounds.ToString() + "]"; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Specification/IFieldType.cs b/source/C#/UniTraX/Core/Specification/IFieldType.cs new file mode 100644 index 0000000000000000000000000000000000000000..dc4a455a908e608ea2435a4bcb123d8bcb3873aa --- /dev/null +++ b/source/C#/UniTraX/Core/Specification/IFieldType.cs @@ -0,0 +1,12 @@ +namespace Qbb.Core.Specification +{ + using Qbb.Core.Partition; + using System; + + public interface IFieldType + { + Interval Bounds { get; } + Func Convert { get; } + Func ConvertUnchecked { get; } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Specification/Record.cs b/source/C#/UniTraX/Core/Specification/Record.cs new file mode 100644 index 0000000000000000000000000000000000000000..28c4e3450b9a4d2c982e90d20e42db6188c7606b --- /dev/null +++ b/source/C#/UniTraX/Core/Specification/Record.cs @@ -0,0 +1,77 @@ +namespace Qbb.Core.Specification +{ + using Qbb.Core.Partition; + using System; + using System.Linq; + using System.Linq.Expressions; + + public class Record + { + public static ContiguousPartition GenerateBoundary(Field[] fields) + { +#if DEBUG + if (fields == null) throw new ArgumentNullException(nameof(fields)); + if (!fields.Any()) throw new ArgumentException("no fields specified"); +#endif + var intervals = new Interval[fields.Length]; + for (int i = 0; i < fields.Length; i++) intervals[i] = fields[i].FieldType.Bounds; + return new ContiguousPartition(intervals); + } + + private readonly int BudgetIndex; + public Field[] Fields { get; private set; } + public ContiguousPartition Boundary { get; private set; } + + public Record(int budgetIndex, Field[] fields) + { +#if DEBUG + if (fields == null) throw new ArgumentNullException(nameof(fields)); + if (!fields.Any()) throw new ArgumentException("no fields specified"); + if (budgetIndex < 0 || fields.Length - 1 < budgetIndex) throw new ArgumentException("budgetIndex is out of bounds"); +#endif + BudgetIndex = budgetIndex; + Fields = fields; + Boundary = GenerateBoundary(Fields); + } + + public Record(Field[] fields) + { +#if DEBUG + if (fields == null) throw new ArgumentNullException(nameof(fields)); + if (!fields.Any()) throw new ArgumentException("no fields specified"); +#endif + BudgetIndex = -1; + Fields = fields; + Boundary = GenerateBoundary(Fields); + } + + public int GetBudgetIndex() + { + if (BudgetIndex < 0) throw new InvalidOperationException("no budget specified on this record"); + return BudgetIndex; + } + + public Record Derive() + { + var newFields = new Field[Fields.Length]; + var ptParam = Expression.Parameter(typeof(PT), "x"); + for (int i = 0; i < Fields.Length; i++) newFields[i] = Fields[i].Derive(Expression.Lambda>(Expression.Field(ptParam, Fields[i].Name), new ParameterExpression[] { ptParam })); + if (BudgetIndex < 0) return new Record(newFields); + return new Record(BudgetIndex, newFields); + } + + public override string ToString() + { + var ret = "[R:"; + var first = true; + foreach (var field in Fields) + { + if (!first) ret = ret + ";"; + ret = ret + field; + first = false; + } + ret = ret + "]"; + return ret; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Utils/FileUtils.cs b/source/C#/UniTraX/Core/Utils/FileUtils.cs new file mode 100644 index 0000000000000000000000000000000000000000..a5b33e7f0f6818862128fbe2427ddb0acedc0a0e --- /dev/null +++ b/source/C#/UniTraX/Core/Utils/FileUtils.cs @@ -0,0 +1,19 @@ +namespace Qbb.Core.Utils +{ + using System; + + public class FileUtils + { + public static void WriteLineToFile(string line, string fileName) + { +#if DEBUG + if (line == null) throw new ArgumentNullException(nameof(line)); + if (fileName == null) throw new ArgumentNullException(nameof(fileName)); +#endif + using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName)) + { + file.WriteLine(line); + } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Utils/Hash.cs b/source/C#/UniTraX/Core/Utils/Hash.cs new file mode 100644 index 0000000000000000000000000000000000000000..cdbdbbf492cdb7c56447661880e5201df2837615 --- /dev/null +++ b/source/C#/UniTraX/Core/Utils/Hash.cs @@ -0,0 +1,27 @@ +namespace Qbb.Core.Utils +{ + using System; + using System.Linq; + + public class Hash + { + public static int GenerateHash() + { + return GenerateHash(null); + } + + public static int GenerateHash(int[] hashes) + { + if (hashes == null || !hashes.Any()) return 17; + return hashes.Aggregate(17, MergeHashes); + } + + public static int MergeHashes(int hashA, int hashB) + { + unchecked + { + return hashA * 23 + hashB; + } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Core/Utils/ReplaceExpressionVisitor.cs b/source/C#/UniTraX/Core/Utils/ReplaceExpressionVisitor.cs new file mode 100644 index 0000000000000000000000000000000000000000..ada79cb4754c63492648561f5e9772544b6680fa --- /dev/null +++ b/source/C#/UniTraX/Core/Utils/ReplaceExpressionVisitor.cs @@ -0,0 +1,22 @@ +namespace Qbb.Core.Utils +{ + using System.Linq.Expressions; + + public class ReplaceExpressionVisitor : ExpressionVisitor + { + private readonly Expression _oldValue; + private readonly Expression _newValue; + + public ReplaceExpressionVisitor(Expression oldValue, Expression newValue) + { + _oldValue = oldValue; + _newValue = newValue; + } + + public override Expression Visit(Expression node) + { + if (node == _oldValue) return _newValue; + return base.Visit(node); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/Account.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/Account.cs new file mode 100644 index 0000000000000000000000000000000000000000..5066eef9fb430d5764e210a8d23ef6904941fb38 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/Account.cs @@ -0,0 +1,62 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "account")] + public class Account + { + [Column(Name = "account_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double AccountId; + [Column(Name = "account_district_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountDistrictId; + [Column(Name = "account_frequency", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountFrequency; + [Column(Name = "account_date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountDate; + [Column(Name = "account_balance", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountBalance; + [Column(Name = "owner_birthdate", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerBirthdate; + [Column(Name = "owner_sex", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerSex; + [Column(Name = "owner_district_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerDistrictId; + [Column(Name = "owner_card_type", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerCardType; + [Column(Name = "owner_card_issued", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerCardIssued; + [Column(Name = "user_birthdate", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UserBirthdate; + [Column(Name = "user_sex", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UserSex; + [Column(Name = "user_district_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UserDistrictId; + [Column(Name = "loan_date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanDate; + [Column(Name = "loan_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanAmount; + [Column(Name = "loan_duration", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanDuration; + [Column(Name = "loan_payments", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanPayments; + [Column(Name = "loan_status", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanStatus; + private EntitySet _orders = new EntitySet(); + [Association(Name = "FK_order_account_id", Storage = "_orders", OtherKey = "AccountId", ThisKey = "AccountId")] + public ICollection Orders { get { return _orders; } set { _orders.Assign(value); } } + private EntitySet _transactions = new EntitySet(); + [Association(Name = "FK_transaction_account_id", Storage = "_transactions", OtherKey = "AccountId", ThisKey = "AccountId")] + public ICollection Transactions { get { return _transactions; } set { _transactions.Assign(value); } } + private EntityRef _accountDistrict; + [Association(Name = "FK_account_account_district_id", IsForeignKey = true, Storage = "_accountDistrict", ThisKey = "AccountDistrictId")] + public District AccountDistrict { get { return _accountDistrict.Entity; } set { _accountDistrict.Entity = value; } } + private EntityRef _ownerDistrict; + [Association(Name = "FK_account_owner_district_id", IsForeignKey = true, Storage = "_ownerDistrict", ThisKey = "OwnerDistrictId")] + public District OwnerDistrict { get { return _ownerDistrict.Entity; } set { _ownerDistrict.Entity = value; } } + private EntityRef _userDistrict; + [Association(Name = "FK_account_user_district_id", IsForeignKey = true, Storage = "_userDistrict", ThisKey = "UserDistrictId")] + public District UserDistrict { get { return _userDistrict.Entity; } set { _userDistrict.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/AccountB.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/AccountB.cs new file mode 100644 index 0000000000000000000000000000000000000000..027a53aed425f6dc017bdf437f894353bc722e9e --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/AccountB.cs @@ -0,0 +1,64 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "account")] + public class AccountB + { + [Column(Name = "account_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double AccountId; + [Column(Name = "account_district_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountDistrictId; + [Column(Name = "account_frequency", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountFrequency; + [Column(Name = "account_date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountDate; + [Column(Name = "account_balance", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountBalance; + [Column(Name = "owner_birthdate", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerBirthdate; + [Column(Name = "owner_sex", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerSex; + [Column(Name = "owner_district_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerDistrictId; + [Column(Name = "owner_card_type", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerCardType; + [Column(Name = "owner_card_issued", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double OwnerCardIssued; + [Column(Name = "user_birthdate", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UserBirthdate; + [Column(Name = "user_sex", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UserSex; + [Column(Name = "user_district_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UserDistrictId; + [Column(Name = "loan_date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanDate; + [Column(Name = "loan_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanAmount; + [Column(Name = "loan_duration", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanDuration; + [Column(Name = "loan_payments", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanPayments; + [Column(Name = "loan_status", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double LoanStatus; + [Column(Name = "budget", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Budget; + private EntitySet _orders = new EntitySet(); + [Association(Name = "FK_order_account_id", Storage = "_orders", OtherKey = "AccountId", ThisKey = "AccountId")] + public ICollection Orders { get { return _orders; } set { _orders.Assign(value); } } + private EntitySet _transactions = new EntitySet(); + [Association(Name = "FK_transaction_account_id", Storage = "_transactions", OtherKey = "AccountId", ThisKey = "AccountId")] + public ICollection Transactions { get { return _transactions; } set { _transactions.Assign(value); } } + private EntityRef _accountDistrict; + [Association(Name = "FK_account_account_district_id", IsForeignKey = true, Storage = "_accountDistrict", ThisKey = "AccountDistrictId")] + public DistrictB AccountDistrict { get { return _accountDistrict.Entity; } set { _accountDistrict.Entity = value; } } + private EntityRef _ownerDistrict; + [Association(Name = "FK_account_owner_district_id", IsForeignKey = true, Storage = "_ownerDistrict", ThisKey = "OwnerDistrictId")] + public DistrictB OwnerDistrict { get { return _ownerDistrict.Entity; } set { _ownerDistrict.Entity = value; } } + private EntityRef _userDistrict; + [Association(Name = "FK_account_user_district_id", IsForeignKey = true, Storage = "_userDistrict", ThisKey = "UserDistrictId")] + public DistrictB UserDistrict { get { return _userDistrict.Entity; } set { _userDistrict.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/District.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/District.cs new file mode 100644 index 0000000000000000000000000000000000000000..fc4c4c20f9c24953f4bbedbc6ac547c8a3fed1d9 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/District.cs @@ -0,0 +1,52 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "district")] + public class District + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "name", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Name; + [Column(Name = "region", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Region; + [Column(Name = "no_inhabitants", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoInhabitants; + [Column(Name = "no_municipalities_inhabitants_0", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants0; + [Column(Name = "no_municipalities_inhabitants_500", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants500; + [Column(Name = "no_municipalities_inhabitants_2000", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants2000; + [Column(Name = "no_municipalities_inhabitants_10000", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants10000; + [Column(Name = "no_cities", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoCities; + [Column(Name = "urban_ratio", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UrbanRatio; + [Column(Name = "avg_salary", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AvgSalary; + [Column(Name = "unemployment_95", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Unemployment95; + [Column(Name = "unemployment_96", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Unemployment96; + [Column(Name = "entrepreneurs_per_1000", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double EntrepreneursPer1000; + [Column(Name = "crimes_95", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Crimes95; + [Column(Name = "crimes_96", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Crimes96; + private EntitySet _accountsAccountDistrict = new EntitySet(); + [Association(Name = "FK_District_Accounts_AD", Storage = "_accountsAccountDistrict", OtherKey = "AccountDistrictId", ThisKey = "Id")] + public ICollection AccountsAccountDistrict { get { return _accountsAccountDistrict; } set { _accountsAccountDistrict.Assign(value); } } + private EntitySet _accountsOwnerDistrict = new EntitySet(); + [Association(Name = "FK_District_Accounts_OD", Storage = "_accountsOwnerDistrict", OtherKey = "OwnerDistrictId", ThisKey = "Id")] + public ICollection AccountsOwnerDistrict { get { return _accountsOwnerDistrict; } set { _accountsOwnerDistrict.Assign(value); } } + private EntitySet _accountsUserDistrict = new EntitySet(); + [Association(Name = "FK_District_Accounts_UD", Storage = "_accountsUserDistrict", OtherKey = "UserDistrictId", ThisKey = "Id")] + public ICollection AccountsUserDistrict { get { return _accountsUserDistrict; } set { _accountsUserDistrict.Assign(value); } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/DistrictB.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/DistrictB.cs new file mode 100644 index 0000000000000000000000000000000000000000..31d096a747910eb58f47803e5dbd8c36bba5d270 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/DistrictB.cs @@ -0,0 +1,52 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "district")] + public class DistrictB + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "name", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Name; + [Column(Name = "region", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Region; + [Column(Name = "no_inhabitants", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoInhabitants; + [Column(Name = "no_municipalities_inhabitants_0", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants0; + [Column(Name = "no_municipalities_inhabitants_500", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants500; + [Column(Name = "no_municipalities_inhabitants_2000", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants2000; + [Column(Name = "no_municipalities_inhabitants_10000", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoMunicipalitiesInhabitants10000; + [Column(Name = "no_cities", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double NoCities; + [Column(Name = "urban_ratio", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double UrbanRatio; + [Column(Name = "avg_salary", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AvgSalary; + [Column(Name = "unemployment_95", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Unemployment95; + [Column(Name = "unemployment_96", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Unemployment96; + [Column(Name = "entrepreneurs_per_1000", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double EntrepreneursPer1000; + [Column(Name = "crimes_95", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Crimes95; + [Column(Name = "crimes_96", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Crimes96; + private EntitySet _accountsAccountDistrict = new EntitySet(); + [Association(Name = "FK_District_Accounts_AD", Storage = "_accountsAccountDistrict", OtherKey = "AccountDistrictId", ThisKey = "Id")] + public ICollection AccountsAccountDistrict { get { return _accountsAccountDistrict; } set { _accountsAccountDistrict.Assign(value); } } + private EntitySet _accountsOwnerDistrict = new EntitySet(); + [Association(Name = "FK_District_Accounts_OD", Storage = "_accountsOwnerDistrict", OtherKey = "OwnerDistrictId", ThisKey = "Id")] + public ICollection AccountsOwnerDistrict { get { return _accountsOwnerDistrict; } set { _accountsOwnerDistrict.Assign(value); } } + private EntitySet _accountsUserDistrict = new EntitySet(); + [Association(Name = "FK_District_Accounts_UD", Storage = "_accountsUserDistrict", OtherKey = "UserDistrictId", ThisKey = "Id")] + public ICollection AccountsUserDistrict { get { return _accountsUserDistrict; } set { _accountsUserDistrict.Assign(value); } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/Financial.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/Financial.cs new file mode 100644 index 0000000000000000000000000000000000000000..e17b81d09534abb714bbaeb18e4f2017a27326df --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/Financial.cs @@ -0,0 +1,36 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Data.Linq.Mapping; + + [Database(Name = "financial")] + public class Financial : System.Data.Linq.DataContext + { + private static readonly MappingSource MappingSource = new AttributeMappingSource(); + + public Financial(string connectionString) : base(connectionString, MappingSource) + { + ObjectTrackingEnabled = false; + CommandTimeout = 300; + } + + public System.Data.Linq.Table Accounts + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Districts + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Orders + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Transactions + { + get { return GetTable(); } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/FinancialB.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/FinancialB.cs new file mode 100644 index 0000000000000000000000000000000000000000..bc20788438cbb74aee6cadd7ea7ff77adb4cf257 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/FinancialB.cs @@ -0,0 +1,36 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Data.Linq.Mapping; + + [Database(Name = "financial")] + public class FinancialB : System.Data.Linq.DataContext + { + private static readonly MappingSource MappingSource = new AttributeMappingSource(); + + public FinancialB(string connectionString) : base(connectionString, MappingSource) + { + ObjectTrackingEnabled = false; + CommandTimeout = 300; + } + + public System.Data.Linq.Table Accounts + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Districts + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Orders + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Transactions + { + get { return GetTable(); } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/Metadata.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/Metadata.cs new file mode 100644 index 0000000000000000000000000000000000000000..f33786075dffe18878edd6818c73f3279f2baf58 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/Metadata.cs @@ -0,0 +1,100 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using Qbb.Core.BoundedData; + using Qbb.Core.Specification; + using Qbb.Infrastructure.DataConversion; + using System; + using System.Linq; + + public class Metadata : IQueryableFactory + { + private static DoubleTypeConverter TypeFinancialDistrictId = new DoubleTypeConverter(new DoubleType(1.0, 77.0, 1.0)); + private static DoubleTypeConverter TypeFinancialDate = new DoubleTypeConverter(new DoubleType(631148400.0, 946681199.0, 1.0)); + private static DoubleTypeConverter TypeFinancialBirthdate = new DoubleTypeConverter(new DoubleType(-2208992400.0, 946681199.0, 1.0)); + private static DoubleTypeConverter TypeFinancialSex = new DoubleTypeConverter(new DoubleType(1.0, 2.0, 1.0)); + private static DoubleTypeConverter TypeFinancialAccountId = new DoubleTypeConverter(new DoubleType(1.0, 11382.0, 1.0)); + private static DoubleTypeConverter TypeFinancialAccountFrequency = new DoubleTypeConverter(new DoubleType(1.0, 3.0, 1.0)); + private static DoubleTypeConverter TypeFinancialAccountBalance = new DoubleTypeConverter(new DoubleType(-30000.0, 150000.0, 0.01)); + private static DoubleTypeConverter TypeFinancialOwnerCardType = new DoubleTypeConverter(new DoubleType(0.0, 3.0, 1.0)); + private static DoubleTypeConverter TypeFinancialLoanAmount = new DoubleTypeConverter(new DoubleType(0.0, 600000.0, 0.01)); + private static DoubleTypeConverter TypeFinancialLoanDuration = new DoubleTypeConverter(new DoubleType(0.0, 60.0, 1.0)); + private static DoubleTypeConverter TypeFinancialLoanPayments = new DoubleTypeConverter(new DoubleType(0.0, 10000.0, 0.01)); + private static DoubleTypeConverter TypeFinancialLoanStatus = new DoubleTypeConverter(new DoubleType(0.0, 4.0, 1.0)); + private static DoubleTypeConverter TypeFinancialBudget = new DoubleTypeConverter(new DoubleType(0.0, 10000000000.0, 0.01)); + + private static Record GenerateRecord() + { + var fields = new[] + { + new Field(0, "AccountId", TypeFinancialAccountId, accessRecord => accessRecord.AccountId), + new Field(1, "AccountDistrictId", TypeFinancialDistrictId, accessRecord => accessRecord.AccountDistrictId), + new Field(2, "AccountFrequency", TypeFinancialAccountFrequency, accessRecord => accessRecord.AccountFrequency), + new Field(3, "AccountDate", TypeFinancialDate, accessRecord => accessRecord.AccountDate), + new Field(4, "AccountBalance", TypeFinancialAccountBalance, accessRecord => accessRecord.AccountBalance), + new Field(5, "OwnerBirthdate", TypeFinancialBirthdate, accessRecord => accessRecord.OwnerBirthdate), + new Field(6, "OwnerSex", TypeFinancialSex, accessRecord => accessRecord.OwnerSex), + new Field(7, "OwnerDistrictId", TypeFinancialDistrictId, accessRecord => accessRecord.OwnerDistrictId), + new Field(8, "OwnerCardType", TypeFinancialOwnerCardType, accessRecord => accessRecord.OwnerCardType), + new Field(9, "OwnerCardIssued", TypeFinancialDate, accessRecord => accessRecord.OwnerCardIssued), + new Field(10, "UserBirthdate", TypeFinancialBirthdate, accessRecord => accessRecord.UserBirthdate), + new Field(11, "UserSex", TypeFinancialSex, accessRecord => accessRecord.UserSex), + new Field(12, "UserDistrictId", TypeFinancialDistrictId, accessRecord => accessRecord.UserDistrictId), + new Field(13, "LoanDate", TypeFinancialDate, accessRecord => accessRecord.LoanDate), + new Field(14, "LoanAmount", TypeFinancialLoanAmount, accessRecord => accessRecord.LoanAmount), + new Field(15, "LoanDuration", TypeFinancialLoanDuration, accessRecord => accessRecord.LoanDuration), + new Field(16, "LoanPayments", TypeFinancialLoanPayments, accessRecord => accessRecord.LoanPayments), + new Field(17, "LoanStatus", TypeFinancialLoanStatus, accessRecord => accessRecord.LoanStatus) + }; + return new Record(fields); + } + + private static Record GenerateRecordB() + { + var fields = new[] + { + new Field(0, "AccountId", TypeFinancialAccountId, accessRecord => accessRecord.AccountId), + new Field(1, "AccountDistrictId", TypeFinancialDistrictId, accessRecord => accessRecord.AccountDistrictId), + new Field(2, "AccountFrequency", TypeFinancialAccountFrequency, accessRecord => accessRecord.AccountFrequency), + new Field(3, "AccountDate", TypeFinancialDate, accessRecord => accessRecord.AccountDate), + new Field(4, "AccountBalance", TypeFinancialAccountBalance, accessRecord => accessRecord.AccountBalance), + new Field(5, "OwnerBirthdate", TypeFinancialBirthdate, accessRecord => accessRecord.OwnerBirthdate), + new Field(6, "OwnerSex", TypeFinancialSex, accessRecord => accessRecord.OwnerSex), + new Field(7, "OwnerDistrictId", TypeFinancialDistrictId, accessRecord => accessRecord.OwnerDistrictId), + new Field(8, "OwnerCardType", TypeFinancialOwnerCardType, accessRecord => accessRecord.OwnerCardType), + new Field(9, "OwnerCardIssued", TypeFinancialDate, accessRecord => accessRecord.OwnerCardIssued), + new Field(10, "UserBirthdate", TypeFinancialBirthdate, accessRecord => accessRecord.UserBirthdate), + new Field(11, "UserSex", TypeFinancialSex, accessRecord => accessRecord.UserSex), + new Field(12, "UserDistrictId", TypeFinancialDistrictId, accessRecord => accessRecord.UserDistrictId), + new Field(13, "LoanDate", TypeFinancialDate, accessRecord => accessRecord.LoanDate), + new Field(14, "LoanAmount", TypeFinancialLoanAmount, accessRecord => accessRecord.LoanAmount), + new Field(15, "LoanDuration", TypeFinancialLoanDuration, accessRecord => accessRecord.LoanDuration), + new Field(16, "LoanPayments", TypeFinancialLoanPayments, accessRecord => accessRecord.LoanPayments), + new Field(17, "LoanStatus", TypeFinancialLoanStatus, accessRecord => accessRecord.LoanStatus), + new Field(18, "Budget", TypeFinancialBudget, accessRecord => accessRecord.Budget) + }; + return new Record(18, fields); + } + + public static readonly Record Record = GenerateRecord(); + public static readonly Record RecordB = GenerateRecordB(); + + public string ConnectionString { get; private set; } + public BoundedDataAccess BoundedDataAccess { get; private set; } + public IQueryable Accounts; + + public Metadata(string connectionString) + { +#if DEBUG + if (connectionString == null) throw new ArgumentNullException(nameof(connectionString)); +#endif + ConnectionString = connectionString; + BoundedDataAccess = new BoundedDataAccess(RecordB, this, new ExpressionProvider()); + Accounts = new FinancialB(ConnectionString).Accounts; + } + + public IQueryable Create() + { + return Accounts; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/Order.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/Order.cs new file mode 100644 index 0000000000000000000000000000000000000000..eec91e0aba51151f57b21853c2cec7bdada2caa9 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/Order.cs @@ -0,0 +1,25 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "order")] + public class Order + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "account_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountId; + [Column(Name = "bank_to", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double BankTo; + [Column(Name = "account_to", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountTo; + [Column(Name = "amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Amount; + [Column(Name = "k_symbol", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double KSymbol; + private EntityRef _account; + [Association(Name = "FK_order_account_id", IsForeignKey = true, Storage = "_account", OtherKey = "AccountId", ThisKey = "AccountId")] + public Account Account { get { return _account.Entity; } set { _account.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/OrderB.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/OrderB.cs new file mode 100644 index 0000000000000000000000000000000000000000..8e6f3a29005da63c2795f4b55a5726cedf13d833 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/OrderB.cs @@ -0,0 +1,25 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "order")] + public class OrderB + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "account_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountId; + [Column(Name = "bank_to", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double BankTo; + [Column(Name = "account_to", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountTo; + [Column(Name = "amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Amount; + [Column(Name = "k_symbol", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double KSymbol; + private EntityRef _account; + [Association(Name = "FK_order_account_id", IsForeignKey = true, Storage = "_account", OtherKey = "AccountId", ThisKey = "AccountId")] + public AccountB Account { get { return _account.Entity; } set { _account.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/Transaction.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/Transaction.cs new file mode 100644 index 0000000000000000000000000000000000000000..ac7d1766877f743f0b1a9461b02daf2161402ccd --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/Transaction.cs @@ -0,0 +1,33 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "transaction")] + public class Transaction + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "account_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountId; + [Column(Name = "date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Date; + [Column(Name = "type", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Type; + [Column(Name = "operation", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Operation; + [Column(Name = "amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Amount; + [Column(Name = "balance", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Balance; + [Column(Name = "k_symbol", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double KSymbol; + [Column(Name = "bank", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double BankTo; + [Column(Name = "account", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountTo; + private EntityRef _account; + [Association(Name = "FK_transaction_account_id", IsForeignKey = true, Storage = "_account", OtherKey = "AccountId", ThisKey = "AccountId")] + public Account Account { get { return _account.Entity; } set { _account.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataAccess/TransactionB.cs b/source/C#/UniTraX/Datasets/Financial/DataAccess/TransactionB.cs new file mode 100644 index 0000000000000000000000000000000000000000..c5e57b08549ccce8f08def4f4dc328cf5ba53984 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataAccess/TransactionB.cs @@ -0,0 +1,33 @@ +namespace Qbb.Datasets.Financial.DataAccess +{ + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "transaction")] + public class TransactionB + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "account_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountId; + [Column(Name = "date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Date; + [Column(Name = "type", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Type; + [Column(Name = "operation", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Operation; + [Column(Name = "amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Amount; + [Column(Name = "balance", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Balance; + [Column(Name = "k_symbol", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double KSymbol; + [Column(Name = "bank", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double BankTo; + [Column(Name = "account", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AccountTo; + private EntityRef _account; + [Association(Name = "FK_transaction_account_id", IsForeignKey = true, Storage = "_account", OtherKey = "AccountId", ThisKey = "AccountId")] + public AccountB Account { get { return _account.Entity; } set { _account.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataLoading/FinancialDisk.cs b/source/C#/UniTraX/Datasets/Financial/DataLoading/FinancialDisk.cs new file mode 100644 index 0000000000000000000000000000000000000000..95e548f02641a8dc00e9246a76b7120478fb71b6 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataLoading/FinancialDisk.cs @@ -0,0 +1,194 @@ +namespace Qbb.Datasets.Financial.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using Qbb.Infrastructure.DataLoading; + using System; + + public class FinancialDisk + { + public static string[] AccountColumnNames = new string[] { + "account_id", + "account_district_id", + "account_frequency", + "account_date", + "account_balance", + "owner_birthdate", + "owner_sex", + "owner_district_id", + "owner_card_type", + "owner_card_issued", + "user_birthdate", + "user_sex", + "user_district_id", + "loan_date", + "loan_amount", + "loan_duration", + "loan_payments", + "loan_status", + "budget" + }; + public static string[] DistrictColumnNames = new string[] { + "id", + "name", + "region", + "no_inhabitants", + "no_municipalities_inhabitants_0", + "no_municipalities_inhabitants_500", + "no_municipalities_inhabitants_2000", + "no_municipalities_inhabitants_10000", + "no_cities", + "urban_ratio", + "avg_salary", + "unemployment_95", + "unemployment_96", + "entrepreneurs_per_1000", + "crimes_95", + "crimes_96" + }; + public static string[] OrderColumnNames = new string[] { + "id", + "account_id", + "bank_to", + "account_to", + "amount", + "k_symbol" + }; + public static string[] TransactionColumnNames = new string[] { + "id", + "account_id", + "date", + "type", + "operation", + "amount", + "balance", + "k_symbol", + "bank", + "account" + }; + + public static void CreateTable(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Table districtTable = TableDisk.CreateTable(database, false, "financial-disk", "district", DistrictColumnNames, "id", "budget", null); + Table accountTable = TableDisk.CreateTable(database, withBudget, "financial-disk", "account", AccountColumnNames, "account_id", "budget", null); + ForeignKey accountDistrictIdFK = new ForeignKey(accountTable, "FK_account_account_district_id"); + ForeignKeyColumn accountDistrictIdFKC = new ForeignKeyColumn(accountDistrictIdFK, "account_district_id", "id"); + accountDistrictIdFK.Columns.Add(accountDistrictIdFKC); + accountDistrictIdFK.ReferencedTable = "district"; + accountDistrictIdFK.Create(); + ForeignKey ownerDistrictIdFK = new ForeignKey(accountTable, "FK_account_owner_district_id"); + ForeignKeyColumn ownerDistrictIdFKC = new ForeignKeyColumn(ownerDistrictIdFK, "owner_district_id", "id"); + ownerDistrictIdFK.Columns.Add(ownerDistrictIdFKC); + ownerDistrictIdFK.ReferencedTable = "district"; + ownerDistrictIdFK.Create(); + ForeignKey userDistrictIdFK = new ForeignKey(accountTable, "FK_account_user_district_id"); + ForeignKeyColumn userDistrictIdFKC = new ForeignKeyColumn(userDistrictIdFK, "user_district_id", "id"); + userDistrictIdFK.Columns.Add(userDistrictIdFKC); + userDistrictIdFK.ReferencedTable = "district"; + userDistrictIdFK.Create(); + accountTable.Alter(); + Table orderTable = TableDisk.CreateTable(database, false, "financial-disk", "order", OrderColumnNames, "id", "budget", null); + ForeignKey orderAccountFK = new ForeignKey(orderTable, "FK_order_account_id"); + ForeignKeyColumn orderAccountFKC = new ForeignKeyColumn(orderAccountFK, "account_id", "account_id"); + orderAccountFK.Columns.Add(orderAccountFKC); + orderAccountFK.ReferencedTable = "account"; + orderAccountFK.Create(); + orderTable.Alter(); + Table transactionTable = TableDisk.CreateTable(database, false, "financial-disk", "transaction", TransactionColumnNames, "id", "budget", null); + ForeignKey transactionAccountFK = new ForeignKey(transactionTable, "FK_transaction_account_id"); + ForeignKeyColumn transactionAccountFKC = new ForeignKeyColumn(transactionAccountFK, "account_id", "account_id"); + transactionAccountFK.Columns.Add(transactionAccountFKC); + transactionAccountFK.ReferencedTable = "account"; + transactionAccountFK.Create(); + transactionTable.Alter(); + } + + public static void CreateIndexes(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Console.WriteLine("FinancialDisk: Creating indexes on tables account, order, and transaction " + (withBudget ? "with" : "without") + " budget."); + var accountTable = database.Tables["account"]; + var accountIndexOS = TableDisk.CreateDefaultIndex(accountTable, "index_nc_financial-disk_account_owner_sex"); + accountIndexOS.IndexedColumns.Add(new IndexedColumn(accountIndexOS, "owner_sex", false)); + if (withBudget) accountIndexOS.IndexedColumns.Add(new IndexedColumn(accountIndexOS, "budget", false)); + accountTable.Indexes.Add(accountIndexOS); + accountIndexOS.Create(); + accountIndexOS.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + accountIndexOS.Alter(); + var accountIndexOSADI = TableDisk.CreateDefaultIndex(accountTable, "index_nc_financial-disk_account_owner_sex_account_district_id"); + accountIndexOSADI.IndexedColumns.Add(new IndexedColumn(accountIndexOSADI, "owner_sex", false)); + accountIndexOSADI.IndexedColumns.Add(new IndexedColumn(accountIndexOSADI, "account_district_id", false)); + if (withBudget) accountIndexOSADI.IndexedColumns.Add(new IndexedColumn(accountIndexOSADI, "budget", false)); + accountIndexOSADI.IndexedColumns.Add(new IndexedColumn(accountIndexOSADI, "owner_birthdate") { IsIncluded = true }); + accountIndexOSADI.IndexedColumns.Add(new IndexedColumn(accountIndexOSADI, "account_balance") { IsIncluded = true }); + accountTable.Indexes.Add(accountIndexOSADI); + accountIndexOSADI.Create(); + accountIndexOSADI.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + accountIndexOSADI.Alter(); + var accountIndexOSAD = TableDisk.CreateDefaultIndex(accountTable, "index_nc_financial-disk_owner_sex_account_account_date"); + accountIndexOSAD.IndexedColumns.Add(new IndexedColumn(accountIndexOSAD, "owner_sex", false)); + accountIndexOSAD.IndexedColumns.Add(new IndexedColumn(accountIndexOSAD, "account_date", false)); + if (withBudget) accountIndexOSAD.IndexedColumns.Add(new IndexedColumn(accountIndexOSAD, "budget", false)); + accountTable.Indexes.Add(accountIndexOSAD); + accountIndexOSAD.Create(); + accountIndexOSAD.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + accountIndexOSAD.Alter(); + var accountIndexOSAB = TableDisk.CreateDefaultIndex(accountTable, "index_nc_financial-disk_account_owner_sex_account_balance"); + accountIndexOSAB.IndexedColumns.Add(new IndexedColumn(accountIndexOSAB, "owner_sex", false)); + accountIndexOSAB.IndexedColumns.Add(new IndexedColumn(accountIndexOSAB, "account_balance", false)); + if (withBudget) accountIndexOSAB.IndexedColumns.Add(new IndexedColumn(accountIndexOSAB, "budget", false)); + accountTable.Indexes.Add(accountIndexOSAB); + accountIndexOSAB.Create(); + accountIndexOSAB.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + accountIndexOSAB.Alter(); + var accountIndexOSOB = TableDisk.CreateDefaultIndex(accountTable, "index_nc_financial-disk_account_owner_sex_owner_birthdate"); + accountIndexOSOB.IndexedColumns.Add(new IndexedColumn(accountIndexOSOB, "owner_sex", false)); + accountIndexOSOB.IndexedColumns.Add(new IndexedColumn(accountIndexOSOB, "owner_birthdate", false)); + if (withBudget) accountIndexOSOB.IndexedColumns.Add(new IndexedColumn(accountIndexOSOB, "budget", false)); + accountTable.Indexes.Add(accountIndexOSOB); + accountIndexOSOB.Create(); + accountIndexOSOB.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + accountIndexOSOB.Alter(); + var accountIndexOSLA = TableDisk.CreateDefaultIndex(accountTable, "index_nc_financial-disk_account_owner_sex_loan_amount"); + accountIndexOSLA.IndexedColumns.Add(new IndexedColumn(accountIndexOSLA, "owner_sex", false)); + accountIndexOSLA.IndexedColumns.Add(new IndexedColumn(accountIndexOSLA, "loan_amount", false)); + if (withBudget) accountIndexOSLA.IndexedColumns.Add(new IndexedColumn(accountIndexOSLA, "budget", false)); + accountTable.Indexes.Add(accountIndexOSLA); + accountIndexOSLA.Create(); + accountIndexOSLA.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + accountIndexOSLA.Alter(); + if (withBudget) + { + var accountIndexB = TableDisk.CreateDefaultIndex(accountTable, "index_nc_financial-disk_account_budget"); + accountIndexB.IndexedColumns.Add(new IndexedColumn(accountIndexB, "budget", false)); + accountTable.Indexes.Add(accountIndexB); + accountIndexB.Create(); + accountIndexB.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + accountIndexB.Alter(); + } + accountTable.Alter(); + var orderTable = database.Tables["order"]; + var orderIndexAI = TableDisk.CreateDefaultIndex(orderTable, "index_nc_financial-disk_order_account_id"); + orderIndexAI.IndexedColumns.Add(new IndexedColumn(orderIndexAI, "account_id", false)); + orderTable.Indexes.Add(orderIndexAI); + orderIndexAI.Create(); + orderIndexAI.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + orderIndexAI.Alter(); + orderTable.Alter(); + var transactionTable = database.Tables["transaction"]; + var transactionIndexAI = TableDisk.CreateDefaultIndex(transactionTable, "index_nc_financial-disk_transaction_account_id"); + transactionIndexAI.IndexedColumns.Add(new IndexedColumn(transactionIndexAI, "account_id", false)); + transactionIndexAI.IndexedColumns.Add(new IndexedColumn(transactionIndexAI, "amount") { IsIncluded = true }); + transactionTable.Indexes.Add(transactionIndexAI); + transactionIndexAI.Create(); + transactionIndexAI.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + transactionIndexAI.Alter(); + transactionTable.Alter(); + Console.WriteLine("FinancialDisk: Created indexes on tables account, order, and transaction."); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/DataLoading/FinancialMem.cs b/source/C#/UniTraX/Datasets/Financial/DataLoading/FinancialMem.cs new file mode 100644 index 0000000000000000000000000000000000000000..163a9b1365dbd748dab0b7c3f434a4e56f8ad1ee --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/DataLoading/FinancialMem.cs @@ -0,0 +1,88 @@ +namespace Qbb.Datasets.Financial.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using Qbb.Infrastructure.DataLoading; + using System; + + public class FinancialMem + { + public static void CreateTable(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Table districtTable = TableMem.CreateTableInit(database, withBudget, "financial-mem", "district", FinancialDisk.DistrictColumnNames, "id", "budget", null); + TableMem.CreateTableFinish(districtTable, "district"); + Table accountTable = TableMem.CreateTableInit(database, withBudget, "financial-mem", "account", FinancialDisk.AccountColumnNames, "account_id", "budget", null); + var accountIndexADI = TableMem.CreateDefaultIndex(accountTable, "index_nc_financial-mem_account_account_district_id"); + accountIndexADI.IndexedColumns.Add(new IndexedColumn(accountIndexADI, "account_district_id", false)); + if (withBudget) accountIndexADI.IndexedColumns.Add(new IndexedColumn(accountIndexADI, "budget", false)); + accountTable.Indexes.Add(accountIndexADI); + var accountIndexAD = TableMem.CreateDefaultIndex(accountTable, "index_nc_financial-mem_account_account_date"); + accountIndexAD.IndexedColumns.Add(new IndexedColumn(accountIndexAD, "account_date", false)); + if (withBudget) accountIndexAD.IndexedColumns.Add(new IndexedColumn(accountIndexAD, "budget", false)); + accountTable.Indexes.Add(accountIndexAD); + var accountIndexAB = TableMem.CreateDefaultIndex(accountTable, "index_nc_financial-mem_account_account_balance"); + accountIndexAB.IndexedColumns.Add(new IndexedColumn(accountIndexAB, "account_balance", false)); + if (withBudget) accountIndexAB.IndexedColumns.Add(new IndexedColumn(accountIndexAB, "budget", false)); + accountTable.Indexes.Add(accountIndexAB); + var accountIndexOB = TableMem.CreateDefaultIndex(accountTable, "index_nc_financial-mem_account_owner_birthdate"); + accountIndexOB.IndexedColumns.Add(new IndexedColumn(accountIndexOB, "owner_birthdate", false)); + if (withBudget) accountIndexOB.IndexedColumns.Add(new IndexedColumn(accountIndexOB, "budget", false)); + accountTable.Indexes.Add(accountIndexOB); + var accountIndexOS = TableMem.CreateDefaultIndex(accountTable, "index_nc_financial-mem_account_owner_sex"); + accountIndexOS.IndexedColumns.Add(new IndexedColumn(accountIndexOS, "owner_sex", false)); + if (withBudget) accountIndexOS.IndexedColumns.Add(new IndexedColumn(accountIndexOS, "budget", false)); + accountTable.Indexes.Add(accountIndexOS); + var accountIndexLA = TableMem.CreateDefaultIndex(accountTable, "index_nc_financial-mem_account_loan_amount"); + accountIndexLA.IndexedColumns.Add(new IndexedColumn(accountIndexLA, "loan_amount", false)); + if (withBudget) accountIndexLA.IndexedColumns.Add(new IndexedColumn(accountIndexLA, "budget", false)); + accountTable.Indexes.Add(accountIndexLA); + if (withBudget) + { + var accountIndexB = TableMem.CreateDefaultIndex(accountTable, "index_nc_financial-mem_account_budget"); + accountIndexB.IndexedColumns.Add(new IndexedColumn(accountIndexB, "budget", false)); + accountTable.Indexes.Add(accountIndexB); + } + TableMem.CreateTableFinish(accountTable, "account"); + ForeignKey accountDistrictIdFK = new ForeignKey(accountTable, "FK_account_account_district_id"); + ForeignKeyColumn accountDistrictIdFKC = new ForeignKeyColumn(accountDistrictIdFK, "account_district_id", "id"); + accountDistrictIdFK.Columns.Add(accountDistrictIdFKC); + accountDistrictIdFK.ReferencedTable = "district"; + accountDistrictIdFK.Create(); + ForeignKey ownerDistrictIdFK = new ForeignKey(accountTable, "FK_account_owner_district_id"); + ForeignKeyColumn ownerDistrictIdFKC = new ForeignKeyColumn(ownerDistrictIdFK, "owner_district_id", "id"); + ownerDistrictIdFK.Columns.Add(ownerDistrictIdFKC); + ownerDistrictIdFK.ReferencedTable = "district"; + ownerDistrictIdFK.Create(); + ForeignKey userDistrictIdFK = new ForeignKey(accountTable, "FK_account_user_district_id"); + ForeignKeyColumn userDistrictIdFKC = new ForeignKeyColumn(userDistrictIdFK, "user_district_id", "id"); + userDistrictIdFK.Columns.Add(userDistrictIdFKC); + userDistrictIdFK.ReferencedTable = "district"; + userDistrictIdFK.Create(); + accountTable.Alter(); + Table orderTable = TableMem.CreateTableInit(database, withBudget, "financial-mem", "order", FinancialDisk.OrderColumnNames, "id", "budget", null); + var orderIndexAI = TableMem.CreateDefaultIndex(orderTable, "index_nc_financial-mem_order_account_id"); + orderIndexAI.IndexedColumns.Add(new IndexedColumn(orderIndexAI, "account_id", false)); + orderTable.Indexes.Add(orderIndexAI); + TableMem.CreateTableFinish(orderTable, "order"); + ForeignKey orderAccountFK = new ForeignKey(orderTable, "FK_order_account_id"); + ForeignKeyColumn orderAccountFKC = new ForeignKeyColumn(orderAccountFK, "account_id", "account_id"); + orderAccountFK.Columns.Add(orderAccountFKC); + orderAccountFK.ReferencedTable = "account"; + orderAccountFK.Create(); + orderTable.Alter(); + Table transactionTable = TableMem.CreateTableInit(database, withBudget, "financial-mem", "transaction", FinancialDisk.TransactionColumnNames, "id", "budget", null); + var transactionIndexAI = TableMem.CreateDefaultIndex(transactionTable, "index_nc_financial-mem_transaction_account_id"); + transactionIndexAI.IndexedColumns.Add(new IndexedColumn(transactionIndexAI, "account_id", false)); + transactionTable.Indexes.Add(transactionIndexAI); + TableMem.CreateTableFinish(transactionTable, "transaction"); + ForeignKey transactionAccountFK = new ForeignKey(transactionTable, "FK_transaction_account_id"); + ForeignKeyColumn transactionAccountFKC = new ForeignKeyColumn(transactionAccountFK, "account_id", "account_id"); + transactionAccountFK.Columns.Add(transactionAccountFKC); + transactionAccountFK.ReferencedTable = "account"; + transactionAccountFK.Create(); + transactionTable.Alter(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/Experiment/FinancialExperiment.cs b/source/C#/UniTraX/Datasets/Financial/Experiment/FinancialExperiment.cs new file mode 100644 index 0000000000000000000000000000000000000000..40f8e07bb2cd87e1d6c201960202627b690c6063 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/Experiment/FinancialExperiment.cs @@ -0,0 +1,419 @@ +namespace Qbb.Datasets.Financial.Experiment +{ + using Qbb.Datasets.Financial.DataAccess; + using Qbb.Datasets.Financial.DataLoading; + using Qbb.Datasets.Financial.Queries; + using Qbb.Infrastructure.BudgetProvider; + using Qbb.Infrastructure.DataConversion; + using Qbb.Infrastructure.DataLoading; + using Qbb.Infrastructure.ExperimentUtils; + using Qbb.Infrastructure.Provisioning; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + + public class FinancialQueries + { + public AnalyticBaseProvider Provider { get; private set; } + + public FinancialQueries(long baseId, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long defaultEpsilon) + { +#if DEBUG + if (defaultEpsilon < 0L) throw new ArgumentException("defaultEpsilon is below zero"); +#endif + var factories = new List>(8) + { + new AccountsPerOwnerSex(baseId + 1L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new FemaleAccountsPerAccountDistrict(baseId + 2L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new FemaleAccountsPerAccountDate(baseId + 3L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new FemaleAccountsPerAccountBalance(baseId + 4L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new FemaleAccountsPerOwnerBirthdate(baseId + 5L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new FemaleAccountsPerLoanAmount(baseId + 6L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new LocalInvestigation(baseId + 7L, useEvenly, budgetOverRuntime, cleanThreshold, new long[] { defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon }), + new FemaleAccountsPerTransactionsAmount(baseId + 8L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon) + }; + Provider = new AnalyticBaseProvider(factories); + } + } + + public class FinancialExperiment + { + public static bool DiskOnly = false; + public static bool RuntimeOnly = false; + public static int Repeats = 8; + public static int CleanThreshold = 500; + public static long BaseId = 2200L; + public static long DefaultEpsilon = DoubleTypeConverter.TwoDecimal.ToSystem(10000.0); + public static string BaseDirName = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "experiments", "financial", "experiment"); + public static string InstanceName = BaseId + "_" + DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss-fff", CultureInfo.InvariantCulture); + public static string InstanceDirName = Path.Combine(BaseDirName, InstanceName); + public static string DataFileNameDistrict = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "financial", "original", "district.csv"); + public static string DataFileNameAccount = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "financial", "original", "account.csv"); + public static string DataFileNameOrder = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "financial", "original", "order.csv"); + public static string DataFileNameTransaction = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "financial", "original", "transaction.csv"); + public static double DefaultTotalBudget = 100000000.0; + public static IBudgetProvider BudgetProvider = new ConstantBudgetProvider(DefaultTotalBudget); + + public static void SetupDatabaseMem(IExperimentServer server, IBudgetProvider budgetProvider) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); +#endif + server.Setup(); + server.CreateDatabaseInMem("financial-mem"); + FinancialMem.CreateTable(server.GetDatabase("financial-mem"), budgetProvider != null); + var dataTableDistrict = new DataTableProvider(DataFileNameDistrict, budgetProvider, FinancialDisk.DistrictColumnNames, "budget", null).GetDataTable(); + var dataTableAccount = new DataTableProvider(DataFileNameAccount, budgetProvider, FinancialDisk.AccountColumnNames, "budget", null).GetDataTable(); + var dataTableOrder = new DataTableProvider(DataFileNameOrder, budgetProvider, FinancialDisk.OrderColumnNames, "budget", null).GetDataTable(); + var dataTableTransaction = new DataTableProvider(DataFileNameTransaction, budgetProvider, FinancialDisk.TransactionColumnNames, "budget", null).GetDataTable(); + var connection = server.GetSqlConnection("financial-mem"); + connection.Open(); + TableMem.FillTable(dataTableDistrict, connection, "district"); + TableMem.FillTable(dataTableAccount, connection, "account"); + TableMem.FillTable(dataTableOrder, connection, "order"); + TableMem.FillTable(dataTableTransaction, connection, "transaction"); + connection.Close(); + dataTableDistrict.Dispose(); + dataTableAccount.Dispose(); + dataTableOrder.Dispose(); + dataTableTransaction.Dispose(); + } + + public static void SetupDatabaseDisk(IExperimentServer server, IBudgetProvider budgetProvider) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); +#endif + server.Setup(); + server.CreateDatabaseOnDisk("financial-disk"); + FinancialDisk.CreateTable(server.GetDatabase("financial-disk"), budgetProvider != null); + var dataTableDistrict = new DataTableProvider(DataFileNameDistrict, budgetProvider, FinancialDisk.DistrictColumnNames, "budget", null).GetDataTable(); + var dataTableAccount = new DataTableProvider(DataFileNameAccount, budgetProvider, FinancialDisk.AccountColumnNames, "budget", null).GetDataTable(); + var dataTableOrder = new DataTableProvider(DataFileNameOrder, budgetProvider, FinancialDisk.OrderColumnNames, "budget", null).GetDataTable(); + var dataTableTransaction = new DataTableProvider(DataFileNameTransaction, budgetProvider, FinancialDisk.TransactionColumnNames, "budget", null).GetDataTable(); + var connection = server.GetSqlConnection("financial-disk"); + connection.Open(); + TableDisk.FillTable(dataTableDistrict, connection, "district"); + TableDisk.FillTable(dataTableAccount, connection, "account"); + TableDisk.FillTable(dataTableOrder, connection, "order"); + TableDisk.FillTable(dataTableTransaction, connection, "transaction"); + connection.Close(); + dataTableDistrict.Dispose(); + dataTableAccount.Dispose(); + dataTableOrder.Dispose(); + dataTableTransaction.Dispose(); + FinancialDisk.CreateIndexes(server.GetDatabase("financial-disk"), budgetProvider != null); + server.MakeDatabaseReadonly("financial-disk"); + } + + public static void Main(string[] args) + { +#if RECORD + var timerT = Recorder.StartTimer(); +#endif + Console.WriteLine("Experiment Financial: STARTED"); + var server = new ExampleDBServer(); + + var queriesAlwaysClean = new FinancialQueries(BaseId, false, false, -1, DefaultEpsilon); + var queriesAlwaysCleanEven = new FinancialQueries(BaseId, true, false, -1, DefaultEpsilon); + var queriesThresholdClean = new FinancialQueries(BaseId, false, false, CleanThreshold, DefaultEpsilon); + var queriesThresholdCleanEven = new FinancialQueries(BaseId, true, false, CleanThreshold, DefaultEpsilon); + + var queriesAlwaysCleanBudget = new FinancialQueries(BaseId, false, true, -1, DefaultEpsilon); + var queriesAlwaysCleanEvenBudget = new FinancialQueries(BaseId, true, true, -1, DefaultEpsilon); + var queriesThresholdCleanBudget = new FinancialQueries(BaseId, false, true, CleanThreshold, DefaultEpsilon); + var queriesThresholdCleanEvenBudget = new FinancialQueries(BaseId, true, true, CleanThreshold, DefaultEpsilon); + + Metadata metadataMem = null; + if (!DiskOnly) metadataMem = new Metadata(server.GetDatabaseConnectionString("financial-mem")); + var metadataDisk = new Metadata(server.GetDatabaseConnectionString("financial-disk")); + + Directory.CreateDirectory(InstanceDirName); + + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysClean.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanEven.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Even-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Direct-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Disk-Direct-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Disk-Direct-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Direct-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Disk-Direct-Runtime", server, + new Financial(server.GetDatabaseConnectionString("financial-disk")).Accounts, + queriesAlwaysClean.Provider.GetQueriesDirect(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Direct-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-PINQ-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Disk-PINQ-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Disk-PINQ-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-PINQ-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Disk-PINQ-Runtime", server, + new PINQueryableFactory(new Financial(server.GetDatabaseConnectionString("financial-disk")).Accounts, DefaultTotalBudget), + queriesAlwaysClean.Provider.GetQueriesPINQ(), DefaultTotalBudget, "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-PINQ-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdClean.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanEven.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Even-Runtime: FINISHED"); + + if (!RuntimeOnly) + { + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Budget"); + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanBudget.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Financial Disk-Always-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanEvenBudget.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Always-BK-Even-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Direct-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Disk-Direct-Budget"); + Console.WriteLine("Setup SubExperiment Financial Disk-Direct-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Direct-Budget: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Disk-Direct-Budget", server, + new Financial(server.GetDatabaseConnectionString("financial-disk")).Accounts, + queriesAlwaysCleanBudget.Provider.GetQueriesDirect(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Direct-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-PINQ-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Disk-PINQ-Budget"); + Console.WriteLine("Setup SubExperiment Financial Disk-PINQ-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-PINQ-Budget: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Disk-PINQ-Budget", server, + new PINQueryableFactory(new Financial(server.GetDatabaseConnectionString("financial-disk")).Accounts, DefaultTotalBudget), + queriesAlwaysCleanBudget.Provider.GetQueriesPINQ(), DefaultTotalBudget, "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-PINQ-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Budget"); + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanBudget.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Financial Disk-Threshold-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanEvenBudget.Provider.GetQueriesBK(), "financial-disk", Repeats); + Console.WriteLine("SubExperiment Financial Disk-Threshold-BK-Even-Budget: FINISHED"); + } + + if (!DiskOnly) + { + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysClean.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanEven.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Even-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Direct-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Mem-Direct-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Mem-Direct-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Direct-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Mem-Direct-Runtime", server, + new Financial(server.GetDatabaseConnectionString("financial-mem")).Accounts, + queriesAlwaysClean.Provider.GetQueriesDirect(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Direct-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-PINQ-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Mem-PINQ-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Mem-PINQ-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-PINQ-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Mem-PINQ-Runtime", server, + new PINQueryableFactory(new Financial(server.GetDatabaseConnectionString("financial-mem")).Accounts, DefaultTotalBudget), + queriesAlwaysClean.Provider.GetQueriesPINQ(), DefaultTotalBudget, "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-PINQ-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdClean.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanEven.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Even-Runtime: FINISHED"); + + if (!RuntimeOnly) + { + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Budget"); + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanBudget.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Financial Mem-Always-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanEvenBudget.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Always-BK-Even-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Direct-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Mem-Direct-Budget"); + Console.WriteLine("Setup SubExperiment Financial Mem-Direct-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Direct-Budget: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Mem-Direct-Budget", server, + new Financial(server.GetDatabaseConnectionString("financial-mem")).Accounts, + queriesAlwaysCleanBudget.Provider.GetQueriesDirect(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Direct-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-PINQ-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Mem-PINQ-Budget"); + Console.WriteLine("Setup SubExperiment Financial Mem-PINQ-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-PINQ-Budget: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Mem-PINQ-Budget", server, + new PINQueryableFactory(new Financial(server.GetDatabaseConnectionString("financial-mem")).Accounts, DefaultTotalBudget), + queriesAlwaysCleanBudget.Provider.GetQueriesPINQ(), DefaultTotalBudget, "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-PINQ-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Budget"); + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanBudget.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Financial Mem-Threshold-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanEvenBudget.Provider.GetQueriesBK(), "financial-mem", Repeats); + Console.WriteLine("SubExperiment Financial Mem-Threshold-BK-Even-Budget: FINISHED"); + } + } + +#if RECORD + var timeT = Recorder.StopTimer(timerT); + Console.WriteLine("Took time ms: " + timeT); +#endif + Console.WriteLine("Experiment Financial: FINISHED"); + Console.Read(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/Queries/AccountsPer.cs b/source/C#/UniTraX/Datasets/Financial/Queries/AccountsPer.cs new file mode 100644 index 0000000000000000000000000000000000000000..da66267cd5de337720285f2dbc1182d6dcef241f --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/Queries/AccountsPer.cs @@ -0,0 +1,199 @@ +namespace Qbb.Datasets.Financial.Queries +{ + using Qbb.Datasets.Financial.DataAccess; + using Qbb.Queries.Aggregation; + using Qbb.Queries.Core; + using System.Linq; + + public class AccountsPerOwnerSex : AnalyticBaseFactory + { + public AccountsPerOwnerSex(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "AccountsPerOwnerSex", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new PerSex(Metadata.Record.Fields[6], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerSex(Metadata.RecordB.Fields[6], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } + + public class FemaleAccountsPerAccountDistrict : AnalyticBaseFactory + { + public FemaleAccountsPerAccountDistrict(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "FemaleAccountsPerAccountDistrict", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[6], 2L, 2L, + new PerDistrict(Metadata.Record.Fields[1], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new WhereRange(Metadata.RecordB.Fields[6], 2L, 2L, + new PerDistrict(Metadata.RecordB.Fields[1], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class FemaleAccountsPerAccountDate : AnalyticBaseFactory + { + public FemaleAccountsPerAccountDate(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "FemaleAccountsPerAccountDate", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[6], 2L, 2L, + new PerYear1990to2000(Metadata.Record.Fields[3], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new WhereRange(Metadata.RecordB.Fields[6], 2L, 2L, + new PerYear1990to2000(Metadata.RecordB.Fields[3], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class FemaleAccountsPerAccountBalance : AnalyticBaseFactory + { + public FemaleAccountsPerAccountBalance(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "FemaleAccountsPerAccountBalance", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[6], 2L, 2L, + new PerAccountBalance(Metadata.Record.Fields[4], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new WhereRange(Metadata.RecordB.Fields[6], 2L, 2L, + new PerAccountBalance(Metadata.RecordB.Fields[4], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class FemaleAccountsPerOwnerBirthdate : AnalyticBaseFactory + { + public FemaleAccountsPerOwnerBirthdate(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "FemaleAccountsPerOwnerBirthdate", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[6], 2L, 2L, + new PerDecade1900to2000(Metadata.Record.Fields[5], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new WhereRange(Metadata.RecordB.Fields[6], 2L, 2L, + new PerDecade1900to2000(Metadata.RecordB.Fields[5], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class FemaleAccountsPerLoanAmount : AnalyticBaseFactory + { + public FemaleAccountsPerLoanAmount(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "FemaleAccountsPerLoanAmount", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[6], 2L, 2L, + new PerLoanAmount(Metadata.Record.Fields[14], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new WhereRange(Metadata.RecordB.Fields[6], 2L, 2L, + new PerLoanAmount(Metadata.RecordB.Fields[14], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class FemaleAccountsPerTransactionsAmount : AnalyticBaseFactory + { + public FemaleAccountsPerTransactionsAmount(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "FemaleAccountsPerTransactionsAmount", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[6], 2L, 2L, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, + new PerTransactionsExponentialCondition(account => account.Transactions.Sum(transaction => transaction.Amount), 10.0, 6), CleanThreshold)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new WhereRange(Metadata.RecordB.Fields[6], 2L, 2L, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, + new PerTransactionsExponentialCondition(account => account.Transactions.Sum(transaction => transaction.Amount), 10.0, 6), CleanThreshold))); + } + } +} diff --git a/source/C#/UniTraX/Datasets/Financial/Queries/LocalInvestigation.cs b/source/C#/UniTraX/Datasets/Financial/Queries/LocalInvestigation.cs new file mode 100644 index 0000000000000000000000000000000000000000..bbbf8617d11dabc1a40db1bbec66fe898b0b43c0 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/Queries/LocalInvestigation.cs @@ -0,0 +1,59 @@ +namespace Qbb.Datasets.Financial.Queries +{ + using Qbb.Datasets.Financial.DataAccess; + using Qbb.Queries.Aggregation; + using Qbb.Queries.Core; + using System.Collections.Generic; + + public class LocalInvestigation : AnalyticBaseFactory + { + public LocalInvestigation(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long[] epsilons) : + base(id, "LocalInvestigation1", useEvenly, budgetOverRuntime, cleanThreshold, epsilons) { } + + public override IQuery GenerateQueryDirect() + { + var list = new List>(5) + { + new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[5], 2208992400.0, CleanThreshold), + new Median(Epsilons[2], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[5], 2208992400.0, 3155673600.0, CleanThreshold), + new Average(Epsilons[3], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[4], 150000.0, CleanThreshold), + new Median(Epsilons[4], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[4], 30000.0, 180000.0, CleanThreshold), + new Sum(Epsilons[5], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[4], 150000.0, CleanThreshold) + }; + var multi = new Multi(list); + + IQuery forOthers = null; + forOthers = new Dummy(Epsilons[1] + Epsilons[2] + Epsilons[3] + Epsilons[4] + Epsilons[5], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 19.5, new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), multi, forOthers); + forOthers = null; + return new WhereRange(Metadata.Record.Fields[6], 2L, 2L, + new PerDistrict(Metadata.Record.Fields[1], conditional, forOthers, BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + var list = new List>(5) + { + new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[5], 2208992400.0, CleanThreshold), + new Median(Epsilons[2], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[5], 2208992400.0, 3155673600.0, CleanThreshold), + new Average(Epsilons[3], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[4], 150000.0, CleanThreshold), + new Median(Epsilons[4], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[4], 30000.0, 180000.0, CleanThreshold), + new Sum(Epsilons[5], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[4], 150000.0, CleanThreshold) + }; + var multi = new Multi(list); + + IQuery forOthers = null; + forOthers = new Dummy(Epsilons[1] + Epsilons[2] + Epsilons[3] + Epsilons[4] + Epsilons[5], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 19.5, new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), multi, forOthers); + forOthers = null; + var budget = new WhereBudget(Epsilons[0] + Epsilons[1] + Epsilons[2] + Epsilons[3] + Epsilons[4] + Epsilons[5], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], conditional); + return new WhereRange(Metadata.RecordB.Fields[6], 2L, 2L, + new PerDistrict(Metadata.RecordB.Fields[1], budget, forOthers, BudgetOverRuntime)); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/Queries/PerAmount.cs b/source/C#/UniTraX/Datasets/Financial/Queries/PerAmount.cs new file mode 100644 index 0000000000000000000000000000000000000000..67f2cf54819bb76dd066eb3289a07c40d977c7e3 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/Queries/PerAmount.cs @@ -0,0 +1,62 @@ +namespace Qbb.Datasets.Financial.Queries +{ + using Qbb.Core.Specification; + using Qbb.Datasets.Financial.DataAccess; + using Qbb.Infrastructure.DataConversion; + using Qbb.Queries.Core; + using System.Collections.Generic; + + public class PerAccountBalance : PerNumbers + { + public static List GenerateParts() + { + return new List(18) + { + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-30000.0), DoubleTypeConverter.TwoDecimal.ToSystem(-15000.0 - 0.01), 0), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-15000.0), DoubleTypeConverter.TwoDecimal.ToSystem(-6000.0 - 0.01), 1), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-6000.0), DoubleTypeConverter.TwoDecimal.ToSystem(-3000.0 - 0.01), 2), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-3000.0), DoubleTypeConverter.TwoDecimal.ToSystem(-1500.0 - 0.01), 3), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-1500.0), DoubleTypeConverter.TwoDecimal.ToSystem(-600.0 - 0.01), 4), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-600.0), DoubleTypeConverter.TwoDecimal.ToSystem(-300.0 - 0.01), 5), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-300.0), DoubleTypeConverter.TwoDecimal.ToSystem(-150.0 - 0.01), 6), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(-150.0), DoubleTypeConverter.TwoDecimal.ToSystem(0.0 - 0.01), 7), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(0.0), DoubleTypeConverter.TwoDecimal.ToSystem(150.0 - 0.01), 8), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(150.0), DoubleTypeConverter.TwoDecimal.ToSystem(300.0 - 0.01), 9), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(300.0), DoubleTypeConverter.TwoDecimal.ToSystem(600.0 - 0.01), 10), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(600.0), DoubleTypeConverter.TwoDecimal.ToSystem(1500.0 - 0.01), 11), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(1500.0), DoubleTypeConverter.TwoDecimal.ToSystem(3000.0 - 0.01), 12), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(3000.0), DoubleTypeConverter.TwoDecimal.ToSystem(6000.0 - 0.01), 13), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(6000.0), DoubleTypeConverter.TwoDecimal.ToSystem(15000.0 - 0.01), 14), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(15000.0), DoubleTypeConverter.TwoDecimal.ToSystem(30000.0 - 0.01), 15), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(30000.0), DoubleTypeConverter.TwoDecimal.ToSystem(60000.0 - 0.01), 16), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(60000.0), DoubleTypeConverter.TwoDecimal.ToSystem(150000.0 - 0.01), 17) + }; + } + + public PerAccountBalance(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerLoanAmount : PerNumbers + { + public static List GenerateParts() + { + return new List(12) + { + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(0.0), DoubleTypeConverter.TwoDecimal.ToSystem(150.0 - 0.01), 0), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(150.0), DoubleTypeConverter.TwoDecimal.ToSystem(300.0 - 0.01), 1), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(300.0), DoubleTypeConverter.TwoDecimal.ToSystem(600.0 - 0.01), 2), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(600.0), DoubleTypeConverter.TwoDecimal.ToSystem(1500.0 - 0.01), 3), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(1500.0), DoubleTypeConverter.TwoDecimal.ToSystem(3000.0 - 0.01), 4), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(3000.0), DoubleTypeConverter.TwoDecimal.ToSystem(6000.0 - 0.01), 5), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(6000.0), DoubleTypeConverter.TwoDecimal.ToSystem(15000.0 - 0.01), 6), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(15000.0), DoubleTypeConverter.TwoDecimal.ToSystem(30000.0 - 0.01), 7), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(30000.0), DoubleTypeConverter.TwoDecimal.ToSystem(60000.0 - 0.01), 8), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(60000.0), DoubleTypeConverter.TwoDecimal.ToSystem(150000.0 - 0.01), 9), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(150000.0), DoubleTypeConverter.TwoDecimal.ToSystem(300000.0 - 0.01), 10), + new Part(DoubleTypeConverter.TwoDecimal.ToSystem(300000.0), DoubleTypeConverter.TwoDecimal.ToSystem(600000.0 - 0.01), 11) + }; + } + + public PerLoanAmount(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/Queries/PerDatetime.cs b/source/C#/UniTraX/Datasets/Financial/Queries/PerDatetime.cs new file mode 100644 index 0000000000000000000000000000000000000000..54798f2fa33d3313608677c99e9c9befff232ece --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/Queries/PerDatetime.cs @@ -0,0 +1,51 @@ +namespace Qbb.Datasets.Financial.Queries +{ + using Qbb.Core.Specification; + using Qbb.Datasets.Financial.DataAccess; + using Qbb.Queries.Core; + using System.Collections.Generic; + + public class PerYear1990to2000 : PerNumbers + { + public static List GenerateParts() + { + return new List(10) + { + new Part(631148400L, 662684400L - 1L, 1990), + new Part(662684400L, 694220400L - 1L, 1991), + new Part(694220400L, 725842800L - 1L, 1992), + new Part(725842800L, 757378800L - 1L, 1993), + new Part(757378800L, 788914800L - 1L, 1994), + new Part(788914800L, 820450800L - 1L, 1995), + new Part(820450800L, 852073200L - 1L, 1996), + new Part(852073200L, 883609200L - 1L, 1997), + new Part(883609200L, 915145200L - 1L, 1998), + new Part(915145200L, 946681200L - 1L, 1999) + }; + } + + public PerYear1990to2000(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerDecade1900to2000 : PerNumbers + { + public static List GenerateParts() + { + return new List(10) + { + new Part(-2208992400L, -1893459600L - 1L, 1900), + new Part(-1893459600L, -1577926800L - 1L, 1910), + new Part(-1577926800L, -1262307600L - 1L, 1920), + new Part(-1262307600L, -946774800L - 1L, 1930), + new Part(-946774800L, -631155600L - 1L, 1940), + new Part(-631155600L, -315622800L - 1L, 1950), + new Part(-315622800L, -3600L - 1L, 1960), + new Part(-3600L, 315529200L - 1L, 1970), + new Part( 315529200L, 631148400L - 1L, 1980), + new Part( 631148400L, 946681200L - 1L, 1990) + }; + } + + public PerDecade1900to2000(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/Queries/PerEnumerate.cs b/source/C#/UniTraX/Datasets/Financial/Queries/PerEnumerate.cs new file mode 100644 index 0000000000000000000000000000000000000000..c1db67e4e1996af219155b713dc854f88e6d8ddb --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/Queries/PerEnumerate.cs @@ -0,0 +1,30 @@ +namespace Qbb.Datasets.Financial.Queries +{ + using Qbb.Core.Specification; + using Qbb.Datasets.Financial.DataAccess; + using Qbb.Queries.Core; + + public class PerDistrict : PerNumbers + { + public PerDistrict(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 1L, 1L - 1L, 77, 1, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerCardType : PerNumbers + { + public PerCardType(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 0L, 1L - 1L, 4, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerSex : PerNumbers + { + public PerSex(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 1L, 1L - 1L, 2, 1, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerLoanStatus : PerNumbers + { + public PerLoanStatus(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 0L, 1L - 1L, 5, 0, forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Financial/Queries/PerTransactions.cs b/source/C#/UniTraX/Datasets/Financial/Queries/PerTransactions.cs new file mode 100644 index 0000000000000000000000000000000000000000..debf04d235d5c1c4d7fcedf1c33e0962dcecc94f --- /dev/null +++ b/source/C#/UniTraX/Datasets/Financial/Queries/PerTransactions.cs @@ -0,0 +1,86 @@ +namespace Qbb.Datasets.Financial.Queries +{ + using PINQ; + using Qbb.Queries.Core; + using System.Linq; + using System.Linq.Expressions; + using System; + + public class PerTransactionsExponentialCondition : IPINQCondition + { + private class KeyExpression + { + public static Expression> Generate(Expression> valueExpression, double baseValue, int maxPower) + { +#if DEBUG + if (valueExpression == null) throw new ArgumentNullException(nameof(valueExpression)); + if (baseValue <= 0.0) throw new ArgumentException("baseValue is not valid"); + if (maxPower < 1) throw new ArgumentException("maxPower is not valid"); +#endif + int power = maxPower; + double value = Math.Pow(baseValue, power); + Expression expression = Expression.Condition(Expression.LessThanOrEqual(valueExpression.Body, Expression.Constant(value)), Expression.Constant(power), Expression.Constant(-1)); + for (power = power - 1; power > 0; power--) + { + value = Math.Pow(baseValue, power); + expression = Expression.Condition(Expression.LessThanOrEqual(valueExpression.Body, Expression.Constant(value)), Expression.Constant(power), expression); + } + return Expression.Lambda>(expression, valueExpression.Parameters); + } + } + + private static int[] GenerateKeys(int maxPower) + { +#if DEBUG + if (maxPower < 1) throw new ArgumentException("maxPower is not valid"); +#endif + var keys = new int[maxPower + 1]; + keys[0] = -1; + for (int i = 1; i <= maxPower; i++) keys[i] = i; + return keys; + } + + private readonly Expression> ValueExpression; + private readonly double BaseValue; + private readonly int MaxPower; + + public PerTransactionsExponentialCondition(Expression> valueExpression, double baseValue, int maxPower) + { +#if DEBUG + if (valueExpression == null) throw new ArgumentNullException(nameof(valueExpression)); + if (baseValue <= 0.0) throw new ArgumentException("baseValue is not valid"); + if (maxPower < 1) throw new ArgumentException("maxPower is not valid"); +#endif + ValueExpression = valueExpression; + BaseValue = baseValue; + MaxPower = maxPower; + } + + public IQueryable[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var queryables = new IQueryable[MaxPower]; + foreach (var group in queryable.GroupBy(KeyExpression.Generate(ValueExpression, BaseValue, MaxPower))) if(group.Key > 0) queryables[group.Key - 1] = group.AsQueryable(); + for (int i = 0; i < queryables.Length; i++) if (queryables[i] == null) queryables[i] = Enumerable.Empty().AsQueryable(); + return queryables; + } + + public PINQueryable[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var queryableMap = queryable.Partition(GenerateKeys(MaxPower), KeyExpression.Generate(ValueExpression, BaseValue, MaxPower)); + var queryables = new PINQueryable[MaxPower]; + for (int i = 1; i <= MaxPower; i++) queryables[i - 1] = queryableMap[i]; + return queryables; + } + + public IPINQCondition Clone() + { + return new PerTransactionsExponentialCondition(ValueExpression, BaseValue, MaxPower); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataAccess/Laboratory.cs b/source/C#/UniTraX/Datasets/Medical/DataAccess/Laboratory.cs new file mode 100644 index 0000000000000000000000000000000000000000..4013f0f1a76b11ce2d0620a2d9d312145ac68ab4 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataAccess/Laboratory.cs @@ -0,0 +1,100 @@ +namespace Qbb.Datasets.Medical.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "laboratory")] + public class Laboratory + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "patient_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double PatientId; + [Column(Name = "date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Date; + [Column(Name = "got", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Got; + [Column(Name = "gpt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Gpt; + [Column(Name = "ldh", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ldh; + [Column(Name = "alp", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Alp; + [Column(Name = "alb", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Alb; + [Column(Name = "ua", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ua; + [Column(Name = "un", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Un; + [Column(Name = "cre", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Cre; + [Column(Name = "t-bil", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double TBil; + [Column(Name = "t-cho", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double TCho; + [Column(Name = "tg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Tg; + [Column(Name = "cpk", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Cpk; + [Column(Name = "glu", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Glu; + [Column(Name = "wbc", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Wbc; + [Column(Name = "rbc", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Rbc; + [Column(Name = "hgb", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Hgb; + [Column(Name = "hct", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Hct; + [Column(Name = "plt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Plt; + [Column(Name = "pt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Pt; + [Column(Name = "pt_note", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double PtNote; + [Column(Name = "aptt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Aptt; + [Column(Name = "fg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Fg; + [Column(Name = "at3", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double At3; + [Column(Name = "a2pi", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double A2pi; + [Column(Name = "u-pro", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double UPro; + [Column(Name = "igg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Igg; + [Column(Name = "iga", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Iga; + [Column(Name = "igm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Igm; + [Column(Name = "crp", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Crp; + [Column(Name = "ra", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ra; + [Column(Name = "rf", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Rf; + [Column(Name = "c3", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double C3; + [Column(Name = "c4", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double C4; + [Column(Name = "rnp", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Rnp; + [Column(Name = "sm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Sm; + [Column(Name = "sc170", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Sc170; + [Column(Name = "ssa", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ssa; + [Column(Name = "ssb", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ssb; + [Column(Name = "centromea", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Centromea; + [Column(Name = "dna", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Dna; + private EntityRef _patient; + [Association(Name = "FK_laboratory_patient_id", IsForeignKey = true, Storage = "_patient", OtherKey = "Id", ThisKey = "PatientId")] + public Patient Patient { get { return _patient.Entity; } set { _patient.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataAccess/LaboratoryB.cs b/source/C#/UniTraX/Datasets/Medical/DataAccess/LaboratoryB.cs new file mode 100644 index 0000000000000000000000000000000000000000..47ad9903281d86c92004d2cf40c01942de86eca6 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataAccess/LaboratoryB.cs @@ -0,0 +1,100 @@ +namespace Qbb.Datasets.Medical.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "laboratory")] + public class LaboratoryB + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "patient_id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double PatientId; + [Column(Name = "date", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Date; + [Column(Name = "got", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Got; + [Column(Name = "gpt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Gpt; + [Column(Name = "ldh", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ldh; + [Column(Name = "alp", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Alp; + [Column(Name = "alb", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Alb; + [Column(Name = "ua", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ua; + [Column(Name = "un", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Un; + [Column(Name = "cre", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Cre; + [Column(Name = "t-bil", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double TBil; + [Column(Name = "t-cho", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double TCho; + [Column(Name = "tg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Tg; + [Column(Name = "cpk", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Cpk; + [Column(Name = "glu", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Glu; + [Column(Name = "wbc", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Wbc; + [Column(Name = "rbc", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Rbc; + [Column(Name = "hgb", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Hgb; + [Column(Name = "hct", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Hct; + [Column(Name = "plt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Plt; + [Column(Name = "pt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Pt; + [Column(Name = "pt_note", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double PtNote; + [Column(Name = "aptt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Aptt; + [Column(Name = "fg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Fg; + [Column(Name = "at3", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double At3; + [Column(Name = "a2pi", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double A2pi; + [Column(Name = "u-pro", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double UPro; + [Column(Name = "igg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Igg; + [Column(Name = "iga", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Iga; + [Column(Name = "igm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Igm; + [Column(Name = "crp", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Crp; + [Column(Name = "ra", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ra; + [Column(Name = "rf", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Rf; + [Column(Name = "c3", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double C3; + [Column(Name = "c4", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double C4; + [Column(Name = "rnp", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Rnp; + [Column(Name = "sm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Sm; + [Column(Name = "sc170", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Sc170; + [Column(Name = "ssa", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ssa; + [Column(Name = "ssb", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Ssb; + [Column(Name = "centromea", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Centromea; + [Column(Name = "dna", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Dna; + private EntityRef _patient; + [Association(Name = "FK_laboratory_patient_id", IsForeignKey = true, Storage = "_patient", OtherKey = "Id", ThisKey = "PatientId")] + public PatientB Patient { get { return _patient.Entity; } set { _patient.Entity = value; } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataAccess/Medical.cs b/source/C#/UniTraX/Datasets/Medical/DataAccess/Medical.cs new file mode 100644 index 0000000000000000000000000000000000000000..245c531e422ef012cd1124959e26507340c60cf1 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataAccess/Medical.cs @@ -0,0 +1,26 @@ +namespace Qbb.Datasets.Medical.DataAccess +{ + using System.Data.Linq.Mapping; + + [Database(Name = "medical")] + public class Medical : System.Data.Linq.DataContext + { + private static readonly MappingSource MappingSource = new AttributeMappingSource(); + + public Medical(string connectionString) : base(connectionString, MappingSource) + { + ObjectTrackingEnabled = false; + CommandTimeout = 300; + } + + public System.Data.Linq.Table Patients + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Laboratories + { + get { return GetTable(); } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataAccess/MedicalB.cs b/source/C#/UniTraX/Datasets/Medical/DataAccess/MedicalB.cs new file mode 100644 index 0000000000000000000000000000000000000000..21f12a9b0a7b119dd7b04217809cab776a7ef6bf --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataAccess/MedicalB.cs @@ -0,0 +1,26 @@ +namespace Qbb.Datasets.Medical.DataAccess +{ + using System.Data.Linq.Mapping; + + [Database(Name = "medical")] + public class MedicalB : System.Data.Linq.DataContext + { + private static readonly MappingSource MappingSource = new AttributeMappingSource(); + + public MedicalB(string connectionString) : base(connectionString, MappingSource) + { + ObjectTrackingEnabled = false; + CommandTimeout = 300; + } + + public System.Data.Linq.Table Patients + { + get { return GetTable(); } + } + + public System.Data.Linq.Table Laboratories + { + get { return GetTable(); } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataAccess/Metadata.cs b/source/C#/UniTraX/Datasets/Medical/DataAccess/Metadata.cs new file mode 100644 index 0000000000000000000000000000000000000000..093c22eca44755032cdd4da4f706407a3282134f --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataAccess/Metadata.cs @@ -0,0 +1,136 @@ +namespace Qbb.Datasets.Medical.DataAccess +{ + using Qbb.Core.BoundedData; + using Qbb.Core.Specification; + using Qbb.Infrastructure.DataConversion; + using System; + using System.Linq; + + public class Metadata : IQueryableFactory + { + private static DoubleTypeConverter TypeMedicalId = new DoubleTypeConverter(new DoubleType(2110.0, 9334041.0, 1.0)); + private static DoubleTypeConverter TypeMedicalSex = new DoubleTypeConverter(new DoubleType(0.0, 2.0, 1.0)); + private static DoubleTypeConverter TypeMedicalDateOfBirth = new DoubleTypeConverter(new DoubleType(-2208988800.0, 631151999.0, 1.0)); + private static DoubleTypeConverter TypeMedicalDateOfInitialDataCollection = new DoubleTypeConverter(new DoubleType(631152000.0, 946684799.0, 1.0)); + private static DoubleTypeConverter TypeMedicalDate = new DoubleTypeConverter(new DoubleType(0.0, 946684799.0, 1.0)); + private static DoubleTypeConverter TypeMedicalQuadrupel = new DoubleTypeConverter(new DoubleType(0.0, 3.0, 1.0)); + private static DoubleTypeConverter TypeMedicalBinary = new DoubleTypeConverter(new DoubleType(0.0, 1.0, 1.0)); + private static DoubleTypeConverter TypeMedicalAclIga = new DoubleTypeConverter(new DoubleType(-1.0, 49999.9, 0.1)); + private static DoubleTypeConverter TypeMedicalAclIgg = new DoubleTypeConverter(new DoubleType(-1.0, 999.9, 0.1)); + private static DoubleTypeConverter TypeMedicalAclIgm = new DoubleTypeConverter(new DoubleType(-1.0, 199999.9, 0.1)); + private static DoubleTypeConverter TypeMedicalAna = new DoubleTypeConverter(new DoubleType(-1.0, 4097.0, 1.0)); + private static DoubleTypeConverter TypeMedicalThrombosis = new DoubleTypeConverter(new DoubleType(-1.0, 3.0, 1.0)); + private static DoubleTypeConverter TypeMedicalBudget = new DoubleTypeConverter(new DoubleType(0.0, 10000000000.0, 0.01)); + + public static Record GenerateRecord() + { + var fields = new[] + { + new Field(0, "Id", TypeMedicalId, accessRecord => accessRecord.Id), + new Field(1, "Sex", TypeMedicalSex, accessRecord => accessRecord.Sex), + new Field(2, "DateOfBirth", TypeMedicalDateOfBirth, accessRecord => accessRecord.DateOfBirth), + new Field(3, "DateOfInitialDataCollection", TypeMedicalDateOfInitialDataCollection, accessRecord => accessRecord.DateOfInitialDataCollection), + new Field(4, "DateOfFirstVisit", TypeMedicalDate, accessRecord => accessRecord.DateOfFirstVisit), + new Field(5, "DateOfExamination", TypeMedicalDate, accessRecord => accessRecord.DateOfExamination), + new Field(6, "Admission", TypeMedicalQuadrupel, accessRecord => accessRecord.Admission), + new Field(7, "DiagnosisSjs", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisSjs), + new Field(8, "DiagnosisSle", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisSle), + new Field(9, "DiagnosisBehcet", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisBehcet), + new Field(10, "DiagnosisPss", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisPss), + new Field(11, "DiagnosisRa", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisRa), + new Field(12, "DiagnosisMctd", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisMctd), + new Field(13, "DiagnosisAps", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisAps), + new Field(14, "DiagnosisPm", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisPm), + new Field(15, "DiagnosisDm", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisDm), + new Field(16, "DiagnosisPn", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisPn), + new Field(17, "DiagnosisFuo", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisFuo), + new Field(18, "DiagnosisAortitis", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisAortitis), + new Field(19, "DiagnosisIp", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisIp), + new Field(20, "DiagnosisMra", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisMra), + new Field(21, "DiagnosisCollagen", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisCollagen), + new Field(22, "DiagnosisRaynaud", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisRaynaud), + new Field(23, "DiagnosisOther", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisOther), + new Field(24, "AclIga", TypeMedicalAclIga, accessRecord => accessRecord.AclIga), + new Field(25, "AclIgg", TypeMedicalAclIgg, accessRecord => accessRecord.AclIgg), + new Field(26, "AclIgm", TypeMedicalAclIgm, accessRecord => accessRecord.AclIgm), + new Field(27, "Ana", TypeMedicalAna, accessRecord => accessRecord.Ana), + new Field(28, "AnaPatternD", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternD), + new Field(29, "AnaPatternN", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternN), + new Field(30, "AnaPatternP", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternP), + new Field(31, "AnaPatternS", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternS), + new Field(32, "Kct", TypeMedicalQuadrupel, accessRecord => accessRecord.Kct), + new Field(33, "Rvvt", TypeMedicalQuadrupel, accessRecord => accessRecord.Rvvt), + new Field(34, "Lac", TypeMedicalQuadrupel, accessRecord => accessRecord.Lac), + new Field(35, "Thrombosis", TypeMedicalThrombosis, accessRecord => accessRecord.Thrombosis) + }; + return new Record(fields); + } + + public static Record GenerateRecordB() + { + var fields = new[] + { + new Field(0, "Id", TypeMedicalId, accessRecord => accessRecord.Id), + new Field(1, "Sex", TypeMedicalSex, accessRecord => accessRecord.Sex), + new Field(2, "DateOfBirth", TypeMedicalDateOfBirth, accessRecord => accessRecord.DateOfBirth), + new Field(3, "DateOfInitialDataCollection", TypeMedicalDateOfInitialDataCollection, accessRecord => accessRecord.DateOfInitialDataCollection), + new Field(4, "DateOfFirstVisit", TypeMedicalDate, accessRecord => accessRecord.DateOfFirstVisit), + new Field(5, "DateOfExamination", TypeMedicalDate, accessRecord => accessRecord.DateOfExamination), + new Field(6, "Admission", TypeMedicalQuadrupel, accessRecord => accessRecord.Admission), + new Field(7, "DiagnosisSjs", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisSjs), + new Field(8, "DiagnosisSle", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisSle), + new Field(9, "DiagnosisBehcet", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisBehcet), + new Field(10, "DiagnosisPss", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisPss), + new Field(11, "DiagnosisRa", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisRa), + new Field(12, "DiagnosisMctd", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisMctd), + new Field(13, "DiagnosisAps", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisAps), + new Field(14, "DiagnosisPm", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisPm), + new Field(15, "DiagnosisDm", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisDm), + new Field(16, "DiagnosisPn", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisPn), + new Field(17, "DiagnosisFuo", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisFuo), + new Field(18, "DiagnosisAortitis", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisAortitis), + new Field(19, "DiagnosisIp", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisIp), + new Field(20, "DiagnosisMra", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisMra), + new Field(21, "DiagnosisCollagen", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisCollagen), + new Field(22, "DiagnosisRaynaud", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisRaynaud), + new Field(23, "DiagnosisOther", TypeMedicalBinary, accessRecord => accessRecord.DiagnosisOther), + new Field(24, "AclIga", TypeMedicalAclIga, accessRecord => accessRecord.AclIga), + new Field(25, "AclIgg", TypeMedicalAclIgg, accessRecord => accessRecord.AclIgg), + new Field(26, "AclIgm", TypeMedicalAclIgm, accessRecord => accessRecord.AclIgm), + new Field(27, "Ana", TypeMedicalAna, accessRecord => accessRecord.Ana), + new Field(28, "AnaPatternD", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternD), + new Field(29, "AnaPatternN", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternN), + new Field(30, "AnaPatternP", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternP), + new Field(31, "AnaPatternS", TypeMedicalBinary, accessRecord => accessRecord.AnaPatternS), + new Field(32, "Kct", TypeMedicalQuadrupel, accessRecord => accessRecord.Kct), + new Field(33, "Rvvt", TypeMedicalQuadrupel, accessRecord => accessRecord.Rvvt), + new Field(34, "Lac", TypeMedicalQuadrupel, accessRecord => accessRecord.Lac), + new Field(35, "Thrombosis", TypeMedicalThrombosis, accessRecord => accessRecord.Thrombosis), + new Field(36, "Budget", TypeMedicalBudget, accessRecord => accessRecord.Budget) + }; + return new Record(36, fields); + } + + public static readonly Record Record = GenerateRecord(); + public static readonly Record RecordB = GenerateRecordB(); + + public string ConnectionString { get; private set; } + public BoundedDataAccess BoundedDataAccess { get; private set; } + public IQueryable Patients; + + public Metadata(string connectionString) + { +#if DEBUG + if (connectionString == null) throw new ArgumentNullException(nameof(connectionString)); +#endif + ConnectionString = connectionString; + BoundedDataAccess = new BoundedDataAccess(RecordB, this, new ExpressionProvider()); + Patients = new MedicalB(ConnectionString).Patients; + } + + public IQueryable Create() + { + return Patients; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataAccess/Patient.cs b/source/C#/UniTraX/Datasets/Medical/DataAccess/Patient.cs new file mode 100644 index 0000000000000000000000000000000000000000..2f741fbeaad574ba19d84498a3d0d820d97826ff --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataAccess/Patient.cs @@ -0,0 +1,92 @@ +namespace Qbb.Datasets.Medical.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "patient")] + public class Patient + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "sex", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Sex; + [Column(Name = "date_of_birth", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfBirth; + [Column(Name = "date_of_initial_data_collection", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfInitialDataCollection; + [Column(Name = "date_of_first_visit", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfFirstVisit; + [Column(Name = "date_of_examination", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfExamination; + [Column(Name = "admission", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Admission; + [Column(Name = "diagnosis_sjs", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisSjs; + [Column(Name = "diagnosis_sle", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisSle; + [Column(Name = "diagnosis_behcet", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisBehcet; + [Column(Name = "diagnosis_pss", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisPss; + [Column(Name = "diagnosis_ra", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisRa; + [Column(Name = "diagnosis_mctd", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisMctd; + [Column(Name = "diagnosis_aps", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisAps; + [Column(Name = "diagnosis_pm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisPm; + [Column(Name = "diagnosis_dm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisDm; + [Column(Name = "diagnosis_pn", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisPn; + [Column(Name = "diagnosis_fuo", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisFuo; + [Column(Name = "diagnosis_aortitis", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisAortitis; + [Column(Name = "diagnosis_ip", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisIp; + [Column(Name = "diagnosis_mra", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisMra; + [Column(Name = "diagnosis_collagen", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisCollagen; + [Column(Name = "diagnosis_raynaud", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisRaynaud; + [Column(Name = "diagnosis_other", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisOther; + [Column(Name = "acl_iga", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AclIga; + [Column(Name = "acl_igg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AclIgg; + [Column(Name = "acl_igm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AclIgm; + [Column(Name = "ana", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Ana; + [Column(Name = "ana_pattern_d", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternD; + [Column(Name = "ana_pattern_n", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternN; + [Column(Name = "ana_pattern_p", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternP; + [Column(Name = "ana_pattern_s", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternS; + [Column(Name = "kct", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Kct; + [Column(Name = "rvvt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Rvvt; + [Column(Name = "lac", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Lac; + [Column(Name = "thrombosis", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Thrombosis; + [Column(Name = "diagnosis_general", DbType = "NChar(41) NOT NULL", UpdateCheck = UpdateCheck.Never)] + public string DiagnosisGeneral; + [Column(Name = "diagnosis_special", DbType = "NChar(25) NOT NULL", UpdateCheck = UpdateCheck.Never)] + public string DiagnosisSpecial; + [Column(Name = "symptoms", DbType = "NChar(59) NOT NULL", UpdateCheck = UpdateCheck.Never)] + public string Symptoms; + private EntitySet _laboratories = new EntitySet(); + [Association(Name = "FK_laboratory_patient_id", Storage = "_laboratories", OtherKey = "PatientId", ThisKey = "Id")] + public ICollection Laboratories { get { return _laboratories; } set { _laboratories.Assign(value); } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataAccess/PatientB.cs b/source/C#/UniTraX/Datasets/Medical/DataAccess/PatientB.cs new file mode 100644 index 0000000000000000000000000000000000000000..aa601f1d4b097b63359482d91527d50f9416fb69 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataAccess/PatientB.cs @@ -0,0 +1,94 @@ +namespace Qbb.Datasets.Medical.DataAccess +{ + using System.Collections.Generic; + using System.Data.Linq; + using System.Data.Linq.Mapping; + + [Table(Name = "patient")] + public class PatientB + { + [Column(Name = "id", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Id; + [Column(Name = "sex", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Sex; + [Column(Name = "date_of_birth", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfBirth; + [Column(Name = "date_of_initial_data_collection", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfInitialDataCollection; + [Column(Name = "date_of_first_visit", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfFirstVisit; + [Column(Name = "date_of_examination", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DateOfExamination; + [Column(Name = "admission", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Admission; + [Column(Name = "diagnosis_sjs", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisSjs; + [Column(Name = "diagnosis_sle", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisSle; + [Column(Name = "diagnosis_behcet", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisBehcet; + [Column(Name = "diagnosis_pss", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisPss; + [Column(Name = "diagnosis_ra", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisRa; + [Column(Name = "diagnosis_mctd", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisMctd; + [Column(Name = "diagnosis_aps", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisAps; + [Column(Name = "diagnosis_pm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisPm; + [Column(Name = "diagnosis_dm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisDm; + [Column(Name = "diagnosis_pn", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisPn; + [Column(Name = "diagnosis_fuo", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisFuo; + [Column(Name = "diagnosis_aortitis", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisAortitis; + [Column(Name = "diagnosis_ip", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisIp; + [Column(Name = "diagnosis_mra", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisMra; + [Column(Name = "diagnosis_collagen", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisCollagen; + [Column(Name = "diagnosis_raynaud", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisRaynaud; + [Column(Name = "diagnosis_other", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DiagnosisOther; + [Column(Name = "acl_iga", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AclIga; + [Column(Name = "acl_igg", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AclIgg; + [Column(Name = "acl_igm", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AclIgm; + [Column(Name = "ana", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Ana; + [Column(Name = "ana_pattern_d", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternD; + [Column(Name = "ana_pattern_n", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternN; + [Column(Name = "ana_pattern_p", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternP; + [Column(Name = "ana_pattern_s", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double AnaPatternS; + [Column(Name = "kct", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Kct; + [Column(Name = "rvvt", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Rvvt; + [Column(Name = "lac", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Lac; + [Column(Name = "thrombosis", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Thrombosis; + [Column(Name = "budget", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Budget; + [Column(Name = "diagnosis_general", DbType = "NChar(41) NOT NULL", UpdateCheck = UpdateCheck.Never)] + public string DiagnosisGeneral; + [Column(Name = "diagnosis_special", DbType = "NChar(25) NOT NULL", UpdateCheck = UpdateCheck.Never)] + public string DiagnosisSpecial; + [Column(Name = "symptoms", DbType = "NChar(59) NOT NULL", UpdateCheck = UpdateCheck.Never)] + public string Symptoms; + private EntitySet _laboratories = new EntitySet(); + [Association(Name = "FK_laboratory_patient_id", Storage = "_laboratories", OtherKey = "PatientId", ThisKey = "Id")] + public ICollection Laboratories { get { return _laboratories; } set { _laboratories.Assign(value); } } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataLoading/MedicalDisk.cs b/source/C#/UniTraX/Datasets/Medical/DataLoading/MedicalDisk.cs new file mode 100644 index 0000000000000000000000000000000000000000..734c337963b401d8fd1d7a3049f092233edb2123 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataLoading/MedicalDisk.cs @@ -0,0 +1,264 @@ +namespace Qbb.Datasets.Medical.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using Qbb.Infrastructure.DataLoading; + using System; + using System.Collections.Generic; + + public class MedicalDisk + { + public static string[] PatientColumnNames = new string[] { + "id", + "sex", + "date_of_birth", + "date_of_initial_data_collection", + "date_of_first_visit", + "date_of_examination", + "admission", + "diagnosis_sjs", + "diagnosis_sle", + "diagnosis_behcet", + "diagnosis_pss", + "diagnosis_ra", + "diagnosis_mctd", + "diagnosis_aps", + "diagnosis_pm", + "diagnosis_dm", + "diagnosis_pn", + "diagnosis_fuo", + "diagnosis_aortitis", + "diagnosis_ip", + "diagnosis_mra", + "diagnosis_collagen", + "diagnosis_raynaud", + "diagnosis_other", + "acl_iga", + "acl_igg", + "acl_igm", + "ana", + "ana_pattern_d", + "ana_pattern_n", + "ana_pattern_p", + "ana_pattern_s", + "kct", + "rvvt", + "lac", + "thrombosis", + "budget", + "diagnosis_general", + "diagnosis_special", + "symptoms" + }; + public static string[] LaboratoryColumnNames = new string[] { + "id", + "patient_id", + "date", + "got", + "gpt", + "ldh", + "alp", + "tp", + "alb", + "ua", + "un", + "cre", + "t-bil", + "t-cho", + "tg", + "cpk", + "glu", + "wbc", + "rbc", + "hgb", + "hct", + "plt", + "pt", + "pt_note", + "aptt", + "fg", + "at3", + "a2pi", + "u-pro", + "igg", + "iga", + "igm", + "crp", + "ra", + "rf", + "c3", + "c4", + "rnp", + "sm", + "sc170", + "ssa", + "ssb", + "centromea", + "dna" + }; + public static Dictionary SpecialColumns() + { + return new Dictionary + { + { "diagnosis_general", DataType.NChar(41) }, + { "diagnosis_special", DataType.NChar(25) }, + { "symptoms", DataType.NChar(59) } + }; + } + + public static void CreateTable(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Table patientTable = TableDisk.CreateTable(database, withBudget, "medical-disk", "patient", PatientColumnNames, "id", "budget", SpecialColumns()); + Table laboratoryTable = TableDisk.CreateTable(database, false, "medical-disk", "laboratory", LaboratoryColumnNames, "id", "budget", null); + ForeignKey laboratoryPatientFK = new ForeignKey(laboratoryTable, "FK_laboratory_patient_id"); + ForeignKeyColumn laboratoryPatientFKC = new ForeignKeyColumn(laboratoryPatientFK, "patient_id", "id"); + laboratoryPatientFK.Columns.Add(laboratoryPatientFKC); + laboratoryPatientFK.ReferencedTable = "patient"; + laboratoryPatientFK.Create(); + laboratoryTable.Alter(); + } + + public static void CreateIndexes(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Console.WriteLine("MedicalDisk: Creating indexes on tables patient and laboratory " + (withBudget ? "with" : "without") + " budget."); + var patientTable = database.Tables["patient"]; + var patientIndexT = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis"); + patientIndexT.IndexedColumns.Add(new IndexedColumn(patientIndexT, "thrombosis", false)); + if (withBudget) patientIndexT.IndexedColumns.Add(new IndexedColumn(patientIndexT, "budget", false)); + patientTable.Indexes.Add(patientIndexT); + patientIndexT.Create(); + patientIndexT.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexT.Alter(); + var patientIndexTS = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_sex"); + patientIndexTS.IndexedColumns.Add(new IndexedColumn(patientIndexTS, "thrombosis", false)); + patientIndexTS.IndexedColumns.Add(new IndexedColumn(patientIndexTS, "sex", false)); + if (withBudget) patientIndexTS.IndexedColumns.Add(new IndexedColumn(patientIndexTS, "budget", false)); + patientTable.Indexes.Add(patientIndexTS); + patientIndexTS.Create(); + patientIndexTS.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTS.Alter(); + var patientIndexTDOB = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_date_of_birth"); + patientIndexTDOB.IndexedColumns.Add(new IndexedColumn(patientIndexTDOB, "thrombosis", false)); + patientIndexTDOB.IndexedColumns.Add(new IndexedColumn(patientIndexTDOB, "date_of_birth", false)); + if (withBudget) patientIndexTDOB.IndexedColumns.Add(new IndexedColumn(patientIndexTDOB, "budget", false)); + patientTable.Indexes.Add(patientIndexTDOB); + patientIndexTDOB.Create(); + patientIndexTDOB.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTDOB.Alter(); + var patientIndexTDOIDC = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_date_of_initial_data_collection"); + patientIndexTDOIDC.IndexedColumns.Add(new IndexedColumn(patientIndexTDOIDC, "thrombosis", false)); + patientIndexTDOIDC.IndexedColumns.Add(new IndexedColumn(patientIndexTDOIDC, "date_of_initial_data_collection", false)); + if (withBudget) patientIndexTDOIDC.IndexedColumns.Add(new IndexedColumn(patientIndexTDOIDC, "budget", false)); + patientTable.Indexes.Add(patientIndexTDOIDC); + patientIndexTDOIDC.Create(); + patientIndexTDOIDC.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTDOIDC.Alter(); + var patientIndexTDOFV = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_date_of_first_visit"); + patientIndexTDOFV.IndexedColumns.Add(new IndexedColumn(patientIndexTDOFV, "thrombosis", false)); + patientIndexTDOFV.IndexedColumns.Add(new IndexedColumn(patientIndexTDOFV, "date_of_first_visit", false)); + if (withBudget) patientIndexTDOFV.IndexedColumns.Add(new IndexedColumn(patientIndexTDOFV, "budget", false)); + patientTable.Indexes.Add(patientIndexTDOFV); + patientIndexTDOFV.Create(); + patientIndexTDOFV.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTDOFV.Alter(); + var patientIndexTDOE = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_date_of_examination"); + patientIndexTDOE.IndexedColumns.Add(new IndexedColumn(patientIndexTDOE, "thrombosis", false)); + patientIndexTDOE.IndexedColumns.Add(new IndexedColumn(patientIndexTDOE, "date_of_examination", false)); + if (withBudget) patientIndexTDOE.IndexedColumns.Add(new IndexedColumn(patientIndexTDOE, "budget", false)); + patientTable.Indexes.Add(patientIndexTDOE); + patientIndexTDOE.Create(); + patientIndexTDOE.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTDOE.Alter(); + var patientIndexTA = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_admission"); + patientIndexTA.IndexedColumns.Add(new IndexedColumn(patientIndexTA, "thrombosis", false)); + patientIndexTA.IndexedColumns.Add(new IndexedColumn(patientIndexTA, "admission", false)); + if (withBudget) patientIndexTA.IndexedColumns.Add(new IndexedColumn(patientIndexTA, "budget", false)); + patientTable.Indexes.Add(patientIndexTA); + patientIndexTA.Create(); + patientIndexTA.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTA.Alter(); + var patientIndexTAIGA = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_acl_iga"); + patientIndexTAIGA.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGA, "thrombosis", false)); + patientIndexTAIGA.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGA, "acl_iga", false)); + if (withBudget) patientIndexTAIGA.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGA, "budget", false)); + patientTable.Indexes.Add(patientIndexTAIGA); + patientIndexTAIGA.Create(); + patientIndexTAIGA.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTAIGA.Alter(); + var patientIndexTAIGG = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_acl_igg"); + patientIndexTAIGG.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGG, "thrombosis", false)); + patientIndexTAIGG.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGG, "acl_igg", false)); + if (withBudget) patientIndexTAIGG.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGG, "budget", false)); + patientTable.Indexes.Add(patientIndexTAIGG); + patientIndexTAIGG.Create(); + patientIndexTAIGG.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTAIGG.Alter(); + var patientIndexTAIGM = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_acl_igm"); + patientIndexTAIGM.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGM, "thrombosis", false)); + patientIndexTAIGM.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGM, "acl_igm", false)); + if (withBudget) patientIndexTAIGM.IndexedColumns.Add(new IndexedColumn(patientIndexTAIGM, "budget", false)); + patientTable.Indexes.Add(patientIndexTAIGM); + patientIndexTAIGM.Create(); + patientIndexTAIGM.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTAIGM.Alter(); + var patientIndexTANA = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_ana"); + patientIndexTANA.IndexedColumns.Add(new IndexedColumn(patientIndexTANA, "thrombosis", false)); + patientIndexTANA.IndexedColumns.Add(new IndexedColumn(patientIndexTANA, "ana", false)); + if (withBudget) patientIndexTANA.IndexedColumns.Add(new IndexedColumn(patientIndexTANA, "budget", false)); + patientTable.Indexes.Add(patientIndexTANA); + patientIndexTANA.Create(); + patientIndexTANA.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTANA.Alter(); + var patientIndexTK = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_kct"); + patientIndexTK.IndexedColumns.Add(new IndexedColumn(patientIndexTK, "thrombosis", false)); + patientIndexTK.IndexedColumns.Add(new IndexedColumn(patientIndexTK, "kct", false)); + if (withBudget) patientIndexTK.IndexedColumns.Add(new IndexedColumn(patientIndexTK, "budget", false)); + patientTable.Indexes.Add(patientIndexTK); + patientIndexTK.Create(); + patientIndexTK.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTK.Alter(); + var patientIndexTR = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_rvvt"); + patientIndexTR.IndexedColumns.Add(new IndexedColumn(patientIndexTR, "thrombosis", false)); + patientIndexTR.IndexedColumns.Add(new IndexedColumn(patientIndexTR, "rvvt", false)); + if (withBudget) patientIndexTR.IndexedColumns.Add(new IndexedColumn(patientIndexTR, "budget", false)); + patientTable.Indexes.Add(patientIndexTR); + patientIndexTR.Create(); + patientIndexTR.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTR.Alter(); + var patientIndexTL = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_thrombosis_lac"); + patientIndexTL.IndexedColumns.Add(new IndexedColumn(patientIndexTL, "thrombosis", false)); + patientIndexTL.IndexedColumns.Add(new IndexedColumn(patientIndexTL, "lac", false)); + if (withBudget) patientIndexTL.IndexedColumns.Add(new IndexedColumn(patientIndexTL, "budget", false)); + patientTable.Indexes.Add(patientIndexTL); + patientIndexTL.Create(); + patientIndexTL.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexTL.Alter(); + if (withBudget) + { + var patientIndexB = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_budget"); + patientIndexB.IndexedColumns.Add(new IndexedColumn(patientIndexB, "budget", false)); + patientIndexB.IndexedColumns.Add(new IndexedColumn(patientIndexB, "diagnosis_special") { IsIncluded = true }); + patientTable.Indexes.Add(patientIndexB); + patientIndexB.Create(); + patientIndexB.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + patientIndexB.Alter(); + } + patientTable.Alter(); + var laboratoryTable = database.Tables["laboratory"]; + var laboratoryIndexPID = TableDisk.CreateDefaultIndex(laboratoryTable, "index_nc_medical-disk_laboratory_patient_id_date"); + laboratoryIndexPID.IndexedColumns.Add(new IndexedColumn(laboratoryIndexPID, "patient_id", false)); + laboratoryIndexPID.IndexedColumns.Add(new IndexedColumn(laboratoryIndexPID, "date", false)); + laboratoryTable.Indexes.Add(laboratoryIndexPID); + laboratoryIndexPID.Create(); + laboratoryIndexPID.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + laboratoryIndexPID.Alter(); + laboratoryTable.Alter(); + Console.WriteLine("MedicalDisk: Created indexes on tables patient and laboratory."); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/DataLoading/MedicalMem.cs b/source/C#/UniTraX/Datasets/Medical/DataLoading/MedicalMem.cs new file mode 100644 index 0000000000000000000000000000000000000000..f2651155c71d9a0ccb52ec8e41456426c7b0744c --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/DataLoading/MedicalMem.cs @@ -0,0 +1,64 @@ +namespace Qbb.Datasets.Medical.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using Qbb.Infrastructure.DataLoading; + using System; + + public class MedicalMem + { + public static void CreateTable(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Table patientTable = TableMem.CreateTableInit(database, withBudget, "medical-mem", "patient", MedicalDisk.PatientColumnNames, "id", "budget", MedicalDisk.SpecialColumns()); + var patientIndexDOB = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_date_of_birth"); + patientIndexDOB.IndexedColumns.Add(new IndexedColumn(patientIndexDOB, "date_of_birth", false)); + if (withBudget) patientIndexDOB.IndexedColumns.Add(new IndexedColumn(patientIndexDOB, "budget", false)); + patientTable.Indexes.Add(patientIndexDOB); + var patientIndexDOIDC = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_date_of_initial_data_collection"); + patientIndexDOIDC.IndexedColumns.Add(new IndexedColumn(patientIndexDOIDC, "date_of_initial_data_collection", false)); + if (withBudget) patientIndexDOIDC.IndexedColumns.Add(new IndexedColumn(patientIndexDOIDC, "budget", false)); + patientTable.Indexes.Add(patientIndexDOIDC); + var patientIndexDOFV = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_date_of_first_visit"); + patientIndexDOFV.IndexedColumns.Add(new IndexedColumn(patientIndexDOFV, "date_of_first_visit", false)); + if (withBudget) patientIndexDOFV.IndexedColumns.Add(new IndexedColumn(patientIndexDOFV, "budget", false)); + patientTable.Indexes.Add(patientIndexDOFV); + var patientIndexDOE = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_date_of_examination"); + patientIndexDOE.IndexedColumns.Add(new IndexedColumn(patientIndexDOE, "date_of_examination", false)); + if (withBudget) patientIndexDOE.IndexedColumns.Add(new IndexedColumn(patientIndexDOE, "budget", false)); + patientTable.Indexes.Add(patientIndexDOE); + var patientIndexAIGA = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_acl_iga"); + patientIndexAIGA.IndexedColumns.Add(new IndexedColumn(patientIndexAIGA, "acl_iga", false)); + if (withBudget) patientIndexAIGA.IndexedColumns.Add(new IndexedColumn(patientIndexAIGA, "budget", false)); + patientTable.Indexes.Add(patientIndexAIGA); + var patientIndexAIGG = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_acl_igg"); + patientIndexAIGG.IndexedColumns.Add(new IndexedColumn(patientIndexAIGG, "acl_igg", false)); + if (withBudget) patientIndexAIGG.IndexedColumns.Add(new IndexedColumn(patientIndexAIGG, "budget", false)); + patientTable.Indexes.Add(patientIndexAIGG); + var patientIndexAIGM = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_acl_igm"); + patientIndexAIGM.IndexedColumns.Add(new IndexedColumn(patientIndexAIGM, "acl_igm", false)); + if (withBudget) patientIndexAIGM.IndexedColumns.Add(new IndexedColumn(patientIndexAIGM, "budget", false)); + patientTable.Indexes.Add(patientIndexAIGM); + var patientIndexA = TableDisk.CreateDefaultIndex(patientTable, "index_nc_medical-disk_patient_ana"); + patientIndexA.IndexedColumns.Add(new IndexedColumn(patientIndexA, "ana", false)); + if (withBudget) patientIndexA.IndexedColumns.Add(new IndexedColumn(patientIndexA, "budget", false)); + patientTable.Indexes.Add(patientIndexA); + TableMem.CreateTableFinish(patientTable, "patient"); + Table laboratoryTable = TableMem.CreateTableInit(database, withBudget, "medical-mem", "laboratory", MedicalDisk.LaboratoryColumnNames, "id", "budget", null); + var laboratoryIndexPI = TableDisk.CreateDefaultIndex(laboratoryTable, "index_nc_medical-disk_laboratory_patient_id"); + laboratoryIndexPI.IndexedColumns.Add(new IndexedColumn(laboratoryIndexPI, "patient_id", false)); + laboratoryTable.Indexes.Add(laboratoryIndexPI); + var laboratoryIndexD = TableDisk.CreateDefaultIndex(laboratoryTable, "index_nc_medical-disk_laboratory_date"); + laboratoryIndexD.IndexedColumns.Add(new IndexedColumn(laboratoryIndexD, "date", false)); + laboratoryTable.Indexes.Add(laboratoryIndexD); + TableMem.CreateTableFinish(laboratoryTable, "laboratory"); + ForeignKey laboratoryPatientFK = new ForeignKey(laboratoryTable, "FK_laboratory_patient_id"); + ForeignKeyColumn laboratoryPatientFKC = new ForeignKeyColumn(laboratoryPatientFK, "patient_id", "id"); + laboratoryPatientFK.Columns.Add(laboratoryPatientFKC); + laboratoryPatientFK.ReferencedTable = "patient"; + laboratoryPatientFK.Create(); + laboratoryTable.Alter(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/Experiment/MedicalExperiment.cs b/source/C#/UniTraX/Datasets/Medical/Experiment/MedicalExperiment.cs new file mode 100644 index 0000000000000000000000000000000000000000..790fe13923e1dc69d8b2625943c1b02728fedadc --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/Experiment/MedicalExperiment.cs @@ -0,0 +1,414 @@ +namespace Qbb.Datasets.Medical.Experiment +{ + using Qbb.Datasets.Medical.DataAccess; + using Qbb.Datasets.Medical.DataLoading; + using Qbb.Datasets.Medical.Queries; + using Qbb.Infrastructure.BudgetProvider; + using Qbb.Infrastructure.DataConversion; + using Qbb.Infrastructure.DataLoading; + using Qbb.Infrastructure.ExperimentUtils; + using Qbb.Infrastructure.Provisioning; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + + public class MedicalQueries + { + public AnalyticBaseProvider Provider { get; private set; } + + public MedicalQueries(long baseId, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long defaultEpsilon) + { +#if DEBUG + if (defaultEpsilon < 0L) throw new ArgumentException("defaultEpsilon is below zero"); +#endif + var factories = new List>(16) + { + + new PatientsPerThrombosis(baseId + 1L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerSex(baseId + 2L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerDateOfBirth(baseId + 3L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerDateOfInitialDataCollection(baseId + 4L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerDateOfFirstVisit(baseId + 5L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerDateOfExamination(baseId + 6L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerAdmission(baseId + 7L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerAclIga(baseId + 8L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerAclIgg(baseId + 9L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerAclIgm(baseId + 10L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerAna(baseId + 11L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerKct(baseId + 12L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerRvvt(baseId + 13L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerLac(baseId + 14L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerSusp(baseId + 15L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new SevereThrombosisPatientsPerLaboratories(baseId + 16L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon) + }; + Provider = new AnalyticBaseProvider(factories); + } + } + + public class MedicalExperiment + { + public static bool DiskOnly = false; + public static bool RuntimeOnly = false; + public static int Repeats = 8; + public static int CleanThreshold = 500; + public static long BaseId = 3300L; + public static long DefaultEpsilon = DoubleTypeConverter.TwoDecimal.ToSystem(10000.0); + public static string BaseDirName = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "experiments", "medical", "experiment", ); + public static string InstanceName = BaseId + "_" + DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss-fff", CultureInfo.InvariantCulture); + public static string InstanceDirName = Path.Combine(BaseDirName, InstanceName); + public static string DataFileNamePatient = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "medical", "transformed", "patient.csv"); + public static string DataFileNameLaboratory = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "medical", "transformed", "laboratory.csv"); + public static double DefaultTotalBudget = 100000000.0; + public static IBudgetProvider BudgetProvider = new ConstantBudgetProvider(DefaultTotalBudget); + + public static void SetupDatabaseMem(IExperimentServer server, IBudgetProvider budgetProvider) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); +#endif + server.Setup(); + server.CreateDatabaseInMem("medical-mem"); + MedicalMem.CreateTable(server.GetDatabase("medical-mem"), budgetProvider != null); + var dataTablePatient = new DataTableProvider(DataFileNamePatient, budgetProvider, MedicalDisk.PatientColumnNames, "budget", new string[] { "diagnosis_general", "diagnosis_special", "symptoms" }).GetDataTable(); + var dataTableLaboratory = new DataTableProvider(DataFileNameLaboratory, budgetProvider, MedicalDisk.LaboratoryColumnNames, "budget", null).GetDataTable(); + var connection = server.GetSqlConnection("medical-mem"); + connection.Open(); + TableMem.FillTable(dataTablePatient, connection, "patient"); + TableMem.FillTable(dataTableLaboratory, connection, "laboratory"); + connection.Close(); + dataTablePatient.Dispose(); + dataTableLaboratory.Dispose(); + } + + public static void SetupDatabaseDisk(IExperimentServer server, IBudgetProvider budgetProvider) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); +#endif + server.Setup(); + server.CreateDatabaseOnDisk("medical-disk"); + MedicalDisk.CreateTable(server.GetDatabase("medical-disk"), budgetProvider != null); + var dataTablePatient = new DataTableProvider(DataFileNamePatient, budgetProvider, MedicalDisk.PatientColumnNames, "budget", new string[] { "diagnosis_general", "diagnosis_special", "symptoms" }).GetDataTable(); + var dataTableLaboratory = new DataTableProvider(DataFileNameLaboratory, budgetProvider, MedicalDisk.LaboratoryColumnNames, "budget", null).GetDataTable(); + var connection = server.GetSqlConnection("medical-disk"); + connection.Open(); + TableDisk.FillTable(dataTablePatient, connection, "patient"); + TableDisk.FillTable(dataTableLaboratory, connection, "laboratory"); + connection.Close(); + dataTablePatient.Dispose(); + dataTableLaboratory.Dispose(); + MedicalDisk.CreateIndexes(server.GetDatabase("medical-disk"), budgetProvider != null); + server.MakeDatabaseReadonly("medical-disk"); + } + + public static void Main(string[] args) + { +#if RECORD + var timerT = Recorder.StartTimer(); +#endif + Console.WriteLine("Experiment Medical: STARTED"); + var server = new ExampleDBServer(); + + var queriesAlwaysClean = new MedicalQueries(BaseId, false, false, -1, DefaultEpsilon); + var queriesAlwaysCleanEven = new MedicalQueries(BaseId, true, false, -1, DefaultEpsilon); + var queriesThresholdClean = new MedicalQueries(BaseId, false, false, CleanThreshold, DefaultEpsilon); + var queriesThresholdCleanEven = new MedicalQueries(BaseId, true, false, CleanThreshold, DefaultEpsilon); + + var queriesAlwaysCleanBudget = new MedicalQueries(BaseId, false, true, -1, DefaultEpsilon); + var queriesAlwaysCleanEvenBudget = new MedicalQueries(BaseId, true, true, -1, DefaultEpsilon); + var queriesThresholdCleanBudget = new MedicalQueries(BaseId, false, true, CleanThreshold, DefaultEpsilon); + var queriesThresholdCleanEvenBudget = new MedicalQueries(BaseId, true, true, CleanThreshold, DefaultEpsilon); + + Metadata metadataMem = null; + if (!DiskOnly) metadataMem = new Metadata(server.GetDatabaseConnectionString("medical-mem")); + var metadataDisk = new Metadata(server.GetDatabaseConnectionString("medical-disk")); + + Directory.CreateDirectory(InstanceDirName); + + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysClean.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanEven.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Even-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Direct-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Disk-Direct-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Disk-Direct-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Direct-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Disk-Direct-Runtime", server, + new Medical(server.GetDatabaseConnectionString("medical-disk")).Patients, + queriesAlwaysClean.Provider.GetQueriesDirect(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Direct-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-PINQ-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Disk-PINQ-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Disk-PINQ-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-PINQ-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Disk-PINQ-Runtime", server, + new PINQueryableFactory(new Medical(server.GetDatabaseConnectionString("medical-disk")).Patients, DefaultTotalBudget), + queriesAlwaysClean.Provider.GetQueriesPINQ(), DefaultTotalBudget, "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-PINQ-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdClean.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanEven.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Even-Runtime: FINISHED"); + + if (!RuntimeOnly) + { + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Budget"); + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanBudget.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Medical Disk-Always-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanEvenBudget.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Always-BK-Even-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Direct-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Disk-Direct-Budget"); + Console.WriteLine("Setup SubExperiment Medical Disk-Direct-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Direct-Budget: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Disk-Direct-Budget", server, + new Medical(server.GetDatabaseConnectionString("medical-disk")).Patients, + queriesAlwaysCleanBudget.Provider.GetQueriesDirect(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Direct-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-PINQ-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Disk-PINQ-Budget"); + Console.WriteLine("Setup SubExperiment Medical Disk-PINQ-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-PINQ-Budget: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Disk-PINQ-Budget", server, + new PINQueryableFactory(new Medical(server.GetDatabaseConnectionString("medical-disk")).Patients, DefaultTotalBudget), + queriesAlwaysCleanBudget.Provider.GetQueriesPINQ(), DefaultTotalBudget, "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-PINQ-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Budget"); + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanBudget.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Medical Disk-Threshold-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanEvenBudget.Provider.GetQueriesBK(), "medical-disk", Repeats); + Console.WriteLine("SubExperiment Medical Disk-Threshold-BK-Even-Budget: FINISHED"); + } + + if (!DiskOnly) + { + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysClean.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanEven.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Even-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Direct-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Mem-Direct-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Mem-Direct-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Direct-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Mem-Direct-Runtime", server, + new Medical(server.GetDatabaseConnectionString("medical-mem")).Patients, + queriesAlwaysClean.Provider.GetQueriesDirect(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Direct-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-PINQ-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Mem-PINQ-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Mem-PINQ-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-PINQ-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Mem-PINQ-Runtime", server, + new PINQueryableFactory(new Medical(server.GetDatabaseConnectionString("medical-mem")).Patients, DefaultTotalBudget), + queriesAlwaysClean.Provider.GetQueriesPINQ(), DefaultTotalBudget, "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-PINQ-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdClean.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanEven.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Even-Runtime: FINISHED"); + + if (!RuntimeOnly) + { + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Budget"); + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanBudget.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Medical Mem-Always-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanEvenBudget.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Always-BK-Even-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Direct-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Mem-Direct-Budget"); + Console.WriteLine("Setup SubExperiment Medical Mem-Direct-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Direct-Budget: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Mem-Direct-Budget", server, + new Medical(server.GetDatabaseConnectionString("medical-mem")).Patients, + queriesAlwaysCleanBudget.Provider.GetQueriesDirect(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Direct-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-PINQ-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Mem-PINQ-Budget"); + Console.WriteLine("Setup SubExperiment Medical Mem-PINQ-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-PINQ-Budget: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Mem-PINQ-Budget", server, + new PINQueryableFactory(new Medical(server.GetDatabaseConnectionString("medical-mem")).Patients, DefaultTotalBudget), + queriesAlwaysCleanBudget.Provider.GetQueriesPINQ(), DefaultTotalBudget, "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-PINQ-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Budget"); + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanBudget.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Medical Mem-Threshold-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanEvenBudget.Provider.GetQueriesBK(), "medical-mem", Repeats); + Console.WriteLine("SubExperiment Medical Mem-Threshold-BK-Even-Budget: FINISHED"); + } + } + +#if RECORD + var timeT = Recorder.StopTimer(timerT); + Console.WriteLine("Took time ms: " + timeT); +#endif + Console.WriteLine("Experiment Medical: FINISHED"); + Console.Read(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/Queries/PatientsPer.cs b/source/C#/UniTraX/Datasets/Medical/Queries/PatientsPer.cs new file mode 100644 index 0000000000000000000000000000000000000000..e015a345ebba83d9cb6a62eba0ceeb9bee1991fc --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/Queries/PatientsPer.cs @@ -0,0 +1,449 @@ +namespace Qbb.Datasets.Medical.Queries +{ + using Qbb.Datasets.Medical.DataAccess; + using Qbb.Queries.Aggregation; + using Qbb.Queries.Core; + using System.Linq; + + public class PatientsPerThrombosis : AnalyticBaseFactory + { + public PatientsPerThrombosis(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerThrombosis", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new PerThrombosis(Metadata.Record.Fields[35], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerThrombosis(Metadata.RecordB.Fields[35], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } + + public class SevereThrombosisPatientsPerSex : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerSex(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerSex", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerSex(Metadata.Record.Fields[1], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerSex(Metadata.RecordB.Fields[1], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerDateOfBirth : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerDateOfBirth(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerDateOfBirth", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerDecade1900to1990(Metadata.Record.Fields[2], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerDecade1900to1990(Metadata.RecordB.Fields[2], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerDateOfInitialDataCollection : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerDateOfInitialDataCollection(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerDateOfInitialDataCollection", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerYear1990to2000(Metadata.Record.Fields[3], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerYear1990to2000(Metadata.RecordB.Fields[3], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerDateOfFirstVisit : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerDateOfFirstVisit(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerDateOfFirstVisit", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerDecade1970to2000(Metadata.Record.Fields[4], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerDecade1970to2000(Metadata.RecordB.Fields[4], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerDateOfExamination : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerDateOfExamination(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerDateOfExamination", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerYear1970to2000(Metadata.Record.Fields[5], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerYear1970to2000(Metadata.RecordB.Fields[5], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerAdmission : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerAdmission(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerAdmission", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerAdmission(Metadata.Record.Fields[6], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerAdmission(Metadata.RecordB.Fields[6], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerAclIga : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerAclIga(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerAclIga", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerAclIga(Metadata.Record.Fields[24], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerAclIga(Metadata.RecordB.Fields[24], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerAclIgg : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerAclIgg(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerAclIgg", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerAclIgg(Metadata.Record.Fields[25], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerAclIgg(Metadata.RecordB.Fields[25], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerAclIgm : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerAclIgm(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerAclIgm", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerAclIgm(Metadata.Record.Fields[26], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerAclIgm(Metadata.RecordB.Fields[26], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerAna : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerAna(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerAna", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerAna(Metadata.Record.Fields[27], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerAna(Metadata.RecordB.Fields[27], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerKct : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerKct(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerKct", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerKct(Metadata.Record.Fields[32], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerKct(Metadata.RecordB.Fields[32], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerRvvt : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerRvvt(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerRvvt", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerRvvt(Metadata.Record.Fields[33], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerRvvt(Metadata.RecordB.Fields[33], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerLac : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerLac(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerLac", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new PerLac(Metadata.Record.Fields[34], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerLac(Metadata.RecordB.Fields[34], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime))); + } + } + + public class SevereThrombosisPatientsPerSusp : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerSusp(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerSusp", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, + new WhereSusp(patient => patient.DiagnosisSpecial.Contains("susp")), CleanThreshold)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, + new WhereSusp(patient => patient.DiagnosisSpecial.Contains("susp")), CleanThreshold))); + } + } + + public class SevereThrombosisPatientsPerLaboratories : AnalyticBaseFactory + { + public SevereThrombosisPatientsPerLaboratories(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "PatientsPerLaboratories", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new WhereRange(Metadata.Record.Fields[35], 2L, 3L, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, + new PerLaboratoriesCount(patient => patient.Laboratories.Count()), CleanThreshold)); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereRange(Metadata.RecordB.Fields[35], 2L, 3L, + new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, + new PerLaboratoriesCount(patient => patient.Laboratories.Count()), CleanThreshold))); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/Queries/PerAmount.cs b/source/C#/UniTraX/Datasets/Medical/Queries/PerAmount.cs new file mode 100644 index 0000000000000000000000000000000000000000..8b5cc47ef498c12ea51b1470cb3c1e077be1be97 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/Queries/PerAmount.cs @@ -0,0 +1,129 @@ +namespace Qbb.Datasets.Medical.Queries +{ + using Qbb.Core.Specification; + using Qbb.Datasets.Medical.DataAccess; + using Qbb.Infrastructure.DataConversion; + using Qbb.Queries.Core; + using System.Collections.Generic; + + public class PerAclIga : PerNumbers + { + private static DoubleTypeConverter TypeMedicalAclIga = new DoubleTypeConverter(new DoubleType(-1.0, 49999.9, 0.1)); + + public static List GenerateParts() + { + return new List(16) + { + new Part(TypeMedicalAclIga.ToSystem(-1.0), TypeMedicalAclIga.ToSystem(-0.1), 99999), + new Part(TypeMedicalAclIga.ToSystem(0.0), TypeMedicalAclIga.ToSystem(0.0), 0), + new Part(TypeMedicalAclIga.ToSystem(0.1), TypeMedicalAclIga.ToSystem(1.0), 1), + new Part(TypeMedicalAclIga.ToSystem(1.1), TypeMedicalAclIga.ToSystem(2.0), 2), + new Part(TypeMedicalAclIga.ToSystem(2.1), TypeMedicalAclIga.ToSystem(3.0), 3), + new Part(TypeMedicalAclIga.ToSystem(3.1), TypeMedicalAclIga.ToSystem(4.0), 4), + new Part(TypeMedicalAclIga.ToSystem(4.1), TypeMedicalAclIga.ToSystem(5.0), 5), + new Part(TypeMedicalAclIga.ToSystem(5.1), TypeMedicalAclIga.ToSystem(6.0), 6), + new Part(TypeMedicalAclIga.ToSystem(6.1), TypeMedicalAclIga.ToSystem(7.0), 7), + new Part(TypeMedicalAclIga.ToSystem(7.1), TypeMedicalAclIga.ToSystem(8.0), 8), + new Part(TypeMedicalAclIga.ToSystem(8.1), TypeMedicalAclIga.ToSystem(9.0), 9), + new Part(TypeMedicalAclIga.ToSystem(9.1), TypeMedicalAclIga.ToSystem(10.0), 10), + new Part(TypeMedicalAclIga.ToSystem(10.1), TypeMedicalAclIga.ToSystem(15.0), 15), + new Part(TypeMedicalAclIga.ToSystem(15.1), TypeMedicalAclIga.ToSystem(20.0), 20), + new Part(TypeMedicalAclIga.ToSystem(20.1), TypeMedicalAclIga.ToSystem(30.0), 30), + new Part(TypeMedicalAclIga.ToSystem(30.1), TypeMedicalAclIga.ToSystem(50000.0 - 0.1), 50000) + }; + } + + public PerAclIga(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerAclIgg : PerNumbers + { + private static DoubleTypeConverter TypeMedicalAclIgg = new DoubleTypeConverter(new DoubleType(-1.0, 999.9, 0.1)); + + public static List GenerateParts() + { + return new List(23) + { + new Part(TypeMedicalAclIgg.ToSystem(-1.0), TypeMedicalAclIgg.ToSystem(-0.1), 99999), + new Part(TypeMedicalAclIgg.ToSystem(0.0), TypeMedicalAclIgg.ToSystem(0.0), 0), + new Part(TypeMedicalAclIgg.ToSystem(0.1), TypeMedicalAclIgg.ToSystem(0.1), 1), + new Part(TypeMedicalAclIgg.ToSystem(0.2), TypeMedicalAclIgg.ToSystem(0.2), 2), + new Part(TypeMedicalAclIgg.ToSystem(0.3), TypeMedicalAclIgg.ToSystem(0.3), 3), + new Part(TypeMedicalAclIgg.ToSystem(0.4), TypeMedicalAclIgg.ToSystem(0.4), 4), + new Part(TypeMedicalAclIgg.ToSystem(0.5), TypeMedicalAclIgg.ToSystem(0.5), 5), + new Part(TypeMedicalAclIgg.ToSystem(0.6), TypeMedicalAclIgg.ToSystem(0.6), 6), + new Part(TypeMedicalAclIgg.ToSystem(0.7), TypeMedicalAclIgg.ToSystem(0.7), 7), + new Part(TypeMedicalAclIgg.ToSystem(0.8), TypeMedicalAclIgg.ToSystem(0.8), 8), + new Part(TypeMedicalAclIgg.ToSystem(0.9), TypeMedicalAclIgg.ToSystem(0.9), 9), + new Part(TypeMedicalAclIgg.ToSystem(1.0), TypeMedicalAclIgg.ToSystem(1.0), 10), + new Part(TypeMedicalAclIgg.ToSystem(1.1), TypeMedicalAclIgg.ToSystem(1.1), 11), + new Part(TypeMedicalAclIgg.ToSystem(1.2), TypeMedicalAclIgg.ToSystem(1.2), 12), + new Part(TypeMedicalAclIgg.ToSystem(1.3), TypeMedicalAclIgg.ToSystem(1.3), 13), + new Part(TypeMedicalAclIgg.ToSystem(1.4), TypeMedicalAclIgg.ToSystem(1.4), 14), + new Part(TypeMedicalAclIgg.ToSystem(1.5), TypeMedicalAclIgg.ToSystem(1.5), 15), + new Part(TypeMedicalAclIgg.ToSystem(1.6), TypeMedicalAclIgg.ToSystem(2.0), 20), + new Part(TypeMedicalAclIgg.ToSystem(2.1), TypeMedicalAclIgg.ToSystem(3.0), 30), + new Part(TypeMedicalAclIgg.ToSystem(3.1), TypeMedicalAclIgg.ToSystem(4.0), 40), + new Part(TypeMedicalAclIgg.ToSystem(4.1), TypeMedicalAclIgg.ToSystem(5.0), 50), + new Part(TypeMedicalAclIgg.ToSystem(5.1), TypeMedicalAclIgg.ToSystem(10.0), 100), + new Part(TypeMedicalAclIgg.ToSystem(10.1), TypeMedicalAclIgg.ToSystem(1000.0 - 0.1), 10000) + }; + } + + public PerAclIgg(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerAclIgm : PerNumbers + { + private static DoubleTypeConverter TypeMedicalAclIgm = new DoubleTypeConverter(new DoubleType(-1.0, 199999.9, 0.1)); + + public static List GenerateParts() + { + return new List(17) + { + new Part(TypeMedicalAclIgm.ToSystem(-1.0), TypeMedicalAclIgm.ToSystem(-0.1), 9999999), + new Part(TypeMedicalAclIgm.ToSystem(0.0), TypeMedicalAclIgm.ToSystem(0.0), 0), + new Part(TypeMedicalAclIgm.ToSystem(0.1), TypeMedicalAclIgm.ToSystem(1.0), 10), + new Part(TypeMedicalAclIgm.ToSystem(1.1), TypeMedicalAclIgm.ToSystem(1.2), 12), + new Part(TypeMedicalAclIgm.ToSystem(1.3), TypeMedicalAclIgm.ToSystem(1.4), 14), + new Part(TypeMedicalAclIgm.ToSystem(1.5), TypeMedicalAclIgm.ToSystem(1.6), 16), + new Part(TypeMedicalAclIgm.ToSystem(1.7), TypeMedicalAclIgm.ToSystem(1.8), 18), + new Part(TypeMedicalAclIgm.ToSystem(1.9), TypeMedicalAclIgm.ToSystem(2.0), 20), + new Part(TypeMedicalAclIgm.ToSystem(2.1), TypeMedicalAclIgm.ToSystem(2.2), 22), + new Part(TypeMedicalAclIgm.ToSystem(2.3), TypeMedicalAclIgm.ToSystem(2.4), 24), + new Part(TypeMedicalAclIgm.ToSystem(2.5), TypeMedicalAclIgm.ToSystem(2.6), 26), + new Part(TypeMedicalAclIgm.ToSystem(2.7), TypeMedicalAclIgm.ToSystem(2.8), 28), + new Part(TypeMedicalAclIgm.ToSystem(2.9), TypeMedicalAclIgm.ToSystem(3.0), 30), + new Part(TypeMedicalAclIgm.ToSystem(3.1), TypeMedicalAclIgm.ToSystem(4.0), 40), + new Part(TypeMedicalAclIgm.ToSystem(4.1), TypeMedicalAclIgm.ToSystem(5.0), 50), + new Part(TypeMedicalAclIgm.ToSystem(5.1), TypeMedicalAclIgm.ToSystem(10.0), 100), + new Part(TypeMedicalAclIgm.ToSystem(10.1), TypeMedicalAclIgm.ToSystem(200000.0 - 0.1), 2000000) + }; + } + + public PerAclIgm(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerAna : PerNumbers + { + private static DoubleTypeConverter TypeMedicalAna = new DoubleTypeConverter(new DoubleType(-1.0, 4097.0, 1.0)); + + public static List GenerateParts() + { + return new List(9) + { + new Part(TypeMedicalAna.ToSystem(-1.0), TypeMedicalAna.ToSystem(-1.0), 9999), + new Part(TypeMedicalAna.ToSystem(0.0), TypeMedicalAna.ToSystem(0.0), 0), + new Part(TypeMedicalAna.ToSystem(1.0), TypeMedicalAna.ToSystem(4.0), 4), + new Part(TypeMedicalAna.ToSystem(5.0), TypeMedicalAna.ToSystem(16.0), 16), + new Part(TypeMedicalAna.ToSystem(17.0), TypeMedicalAna.ToSystem(64.0), 64), + new Part(TypeMedicalAna.ToSystem(65.0), TypeMedicalAna.ToSystem(256.0), 256), + new Part(TypeMedicalAna.ToSystem(257.0), TypeMedicalAna.ToSystem(1024.0), 1024), + new Part(TypeMedicalAna.ToSystem(1025.0), TypeMedicalAna.ToSystem(4096.0), 4096), + new Part(TypeMedicalAna.ToSystem(4097.0), TypeMedicalAna.ToSystem(4097.0), 4097), + }; + } + + public PerAna(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/Queries/PerDatetime.cs b/source/C#/UniTraX/Datasets/Medical/Queries/PerDatetime.cs new file mode 100644 index 0000000000000000000000000000000000000000..38519cda3c8b03f49fe0730e966d8fe8a3d88cd3 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/Queries/PerDatetime.cs @@ -0,0 +1,120 @@ +namespace Qbb.Datasets.Medical.Queries +{ + using Qbb.Core.Specification; + using Qbb.Datasets.Medical.DataAccess; + using Qbb.Infrastructure.DataConversion; + using Qbb.Queries.Core; + using System.Collections.Generic; + + public class PerYear1970to2000 : PerNumbers + { + private static DoubleTypeConverter TypeDate1970to2000 = new DoubleTypeConverter(new DoubleType(0.0, 946684799.0, 1.0)); + + public static List GenerateParts() + { + return new List(31) + { + new Part(TypeDate1970to2000.ToSystem(0.0), TypeDate1970to2000.ToSystem(0.0), 99999), + new Part(TypeDate1970to2000.ToSystem(1.0), TypeDate1970to2000.ToSystem(31536000.0 - 1.0), 1970), + new Part(TypeDate1970to2000.ToSystem(31536000.0), TypeDate1970to2000.ToSystem(63072000.0 - 1.0), 1971), + new Part(TypeDate1970to2000.ToSystem(63072000.0), TypeDate1970to2000.ToSystem(94694400.0 - 1.0), 1972), + new Part(TypeDate1970to2000.ToSystem(94694400.0), TypeDate1970to2000.ToSystem(126230400.0 - 1.0), 1973), + new Part(TypeDate1970to2000.ToSystem(126230400.0), TypeDate1970to2000.ToSystem(157766400.0 - 1.0), 1974), + new Part(TypeDate1970to2000.ToSystem(157766400.0), TypeDate1970to2000.ToSystem(189302400.0 - 1.0), 1975), + new Part(TypeDate1970to2000.ToSystem(189302400.0), TypeDate1970to2000.ToSystem(220924800.0 - 1.0), 1976), + new Part(TypeDate1970to2000.ToSystem(220924800.0), TypeDate1970to2000.ToSystem(252460800.0 - 1.0), 1977), + new Part(TypeDate1970to2000.ToSystem(252460800.0), TypeDate1970to2000.ToSystem(283996800.0 - 1.0), 1978), + new Part(TypeDate1970to2000.ToSystem(283996800.0), TypeDate1970to2000.ToSystem(315532800.0 - 1.0), 1979), + new Part(TypeDate1970to2000.ToSystem(315532800.0), TypeDate1970to2000.ToSystem(347155200.0 - 1.0), 1980), + new Part(TypeDate1970to2000.ToSystem(347155200.0), TypeDate1970to2000.ToSystem(378691200.0 - 1.0), 1981), + new Part(TypeDate1970to2000.ToSystem(378691200.0), TypeDate1970to2000.ToSystem(410227200.0 - 1.0), 1982), + new Part(TypeDate1970to2000.ToSystem(410227200.0), TypeDate1970to2000.ToSystem(441763200.0 - 1.0), 1983), + new Part(TypeDate1970to2000.ToSystem(441763200.0), TypeDate1970to2000.ToSystem(473385600.0 - 1.0), 1984), + new Part(TypeDate1970to2000.ToSystem(473385600.0), TypeDate1970to2000.ToSystem(504921600.0 - 1.0), 1985), + new Part(TypeDate1970to2000.ToSystem(504921600.0), TypeDate1970to2000.ToSystem(536457600.0 - 1.0), 1986), + new Part(TypeDate1970to2000.ToSystem(536457600.0), TypeDate1970to2000.ToSystem(567993600.0 - 1.0), 1987), + new Part(TypeDate1970to2000.ToSystem(567993600.0), TypeDate1970to2000.ToSystem(599616000.0 - 1.0), 1988), + new Part(TypeDate1970to2000.ToSystem(599616000.0), TypeDate1970to2000.ToSystem(631152000.0 - 1.0), 1989), + new Part(TypeDate1970to2000.ToSystem(631152000.0), TypeDate1970to2000.ToSystem(662688000.0 - 1.0), 1990), + new Part(TypeDate1970to2000.ToSystem(662688000.0), TypeDate1970to2000.ToSystem(694224000.0 - 1.0), 1991), + new Part(TypeDate1970to2000.ToSystem(694224000.0), TypeDate1970to2000.ToSystem(725846400.0 - 1.0), 1992), + new Part(TypeDate1970to2000.ToSystem(725846400.0), TypeDate1970to2000.ToSystem(757382400.0 - 1.0), 1993), + new Part(TypeDate1970to2000.ToSystem(757382400.0), TypeDate1970to2000.ToSystem(788918400.0 - 1.0), 1994), + new Part(TypeDate1970to2000.ToSystem(788918400.0), TypeDate1970to2000.ToSystem(820454400.0 - 1.0), 1995), + new Part(TypeDate1970to2000.ToSystem(820454400.0), TypeDate1970to2000.ToSystem(852076800.0 - 1.0), 1996), + new Part(TypeDate1970to2000.ToSystem(852076800.0), TypeDate1970to2000.ToSystem(883612800.0 - 1.0), 1997), + new Part(TypeDate1970to2000.ToSystem(883612800.0), TypeDate1970to2000.ToSystem(915148800.0 - 1.0), 1998), + new Part(TypeDate1970to2000.ToSystem(915148800.0), TypeDate1970to2000.ToSystem(946684800.0 - 1.0), 1999) + }; + } + + public PerYear1970to2000(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerYear1990to2000 : PerNumbers + { + private static DoubleTypeConverter TypeDate1990to2000 = new DoubleTypeConverter(new DoubleType(631152000.0, 946684799.0, 1.0)); + + public static List GenerateParts() + { + return new List(11) + { + new Part(TypeDate1990to2000.ToSystem(631152000.0), TypeDate1990to2000.ToSystem(631152000.0), 99999), + new Part(TypeDate1990to2000.ToSystem(631152001.0), TypeDate1990to2000.ToSystem(662688000.0 - 1.0), 1990), + new Part(TypeDate1990to2000.ToSystem(662688000.0), TypeDate1990to2000.ToSystem(694224000.0 - 1.0), 1991), + new Part(TypeDate1990to2000.ToSystem(694224000.0), TypeDate1990to2000.ToSystem(725846400.0 - 1.0), 1992), + new Part(TypeDate1990to2000.ToSystem(725846400.0), TypeDate1990to2000.ToSystem(757382400.0 - 1.0), 1993), + new Part(TypeDate1990to2000.ToSystem(757382400.0), TypeDate1990to2000.ToSystem(788918400.0 - 1.0), 1994), + new Part(TypeDate1990to2000.ToSystem(788918400.0), TypeDate1990to2000.ToSystem(820454400.0 - 1.0), 1995), + new Part(TypeDate1990to2000.ToSystem(820454400.0), TypeDate1990to2000.ToSystem(852076800.0 - 1.0), 1996), + new Part(TypeDate1990to2000.ToSystem(852076800.0), TypeDate1990to2000.ToSystem(883612800.0 - 1.0), 1997), + new Part(TypeDate1990to2000.ToSystem(883612800.0), TypeDate1990to2000.ToSystem(915148800.0 - 1.0), 1998), + new Part(TypeDate1990to2000.ToSystem(915148800.0), TypeDate1990to2000.ToSystem(946684800.0 - 1.0), 1999) + }; + } + + public PerYear1990to2000(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerDecade1900to1990 : PerNumbers + { + private static DoubleTypeConverter TypeDate1900to1990 = new DoubleTypeConverter(new DoubleType(-2208988800.0, 631151999.0, 1.0)); + + public static List GenerateParts() + { + return new List(10) + { + new Part(TypeDate1900to1990.ToSystem(-2208988800.0), TypeDate1900to1990.ToSystem(-2208988800.0), 99999), + new Part(TypeDate1900to1990.ToSystem(-2208988799.0), TypeDate1900to1990.ToSystem(-1893456000.0 - 1.0), 1900), + new Part(TypeDate1900to1990.ToSystem(-1893456000.0), TypeDate1900to1990.ToSystem(-1577923200.0 - 1.0), 1910), + new Part(TypeDate1900to1990.ToSystem(-1577923200.0), TypeDate1900to1990.ToSystem(-1262304000.0 - 1.0), 1920), + new Part(TypeDate1900to1990.ToSystem(-1262304000.0), TypeDate1900to1990.ToSystem(-946771200.0 - 1.0), 1930), + new Part(TypeDate1900to1990.ToSystem(-946771200.0), TypeDate1900to1990.ToSystem(-631152000.0 - 1.0), 1940), + new Part(TypeDate1900to1990.ToSystem(-631152000.0), TypeDate1900to1990.ToSystem(-315619200.0 - 1.0), 1950), + new Part(TypeDate1900to1990.ToSystem(-315619200.0), TypeDate1900to1990.ToSystem(0.0 - 1.0), 1960), + new Part(TypeDate1900to1990.ToSystem(0.0), TypeDate1900to1990.ToSystem(315532800.0 - 1.0), 1970), + new Part(TypeDate1900to1990.ToSystem(315532800.0), TypeDate1900to1990.ToSystem(631152000.0 - 1.0), 1980) + }; + } + + public PerDecade1900to1990(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } + + public class PerDecade1970to2000 : PerNumbers + { + private static DoubleTypeConverter TypeDate1970to2000 = new DoubleTypeConverter(new DoubleType(0.0, 946684799.0, 1.0)); + + public static List GenerateParts() + { + return new List(4) + { + new Part(TypeDate1970to2000.ToSystem(0.0), TypeDate1970to2000.ToSystem(0.0), 99999), + new Part(TypeDate1970to2000.ToSystem(1.0), TypeDate1970to2000.ToSystem(315532800.0 - 1.0), 1970), + new Part(TypeDate1970to2000.ToSystem(315532800.0), TypeDate1970to2000.ToSystem(631152000.0 - 1.0), 1980), + new Part(TypeDate1970to2000.ToSystem(631152000.0), TypeDate1970to2000.ToSystem(946684800.0 - 1.0), 1990) + }; + } + + public PerDecade1970to2000(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, GenerateParts(), forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/Queries/PerEnumerate.cs b/source/C#/UniTraX/Datasets/Medical/Queries/PerEnumerate.cs new file mode 100644 index 0000000000000000000000000000000000000000..14ece0975f5dd68d6269e62726022e95c80eaa3d --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/Queries/PerEnumerate.cs @@ -0,0 +1,35 @@ +namespace Qbb.Datasets.Medical.Queries +{ + using Qbb.Core.Specification; + using Qbb.Queries.Core; + + public class PerSex : PerNumbers + { + public PerSex(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, 0L, 0L, 3, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerAdmission : PerNumbers + { + public PerAdmission(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, 0L, 0L, 4, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerKct : PerNumbers + { + public PerKct(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, 0L, 0L, 4, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerRvvt : PerNumbers + { + public PerRvvt(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, 0L, 0L, 4, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerLac : PerNumbers + { + public PerLac(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, 0L, 0L, 4, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerThrombosis : PerNumbers + { + public PerThrombosis(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : base(field, -1L, 0L, 5, 0, forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/Queries/PerLaboratories.cs b/source/C#/UniTraX/Datasets/Medical/Queries/PerLaboratories.cs new file mode 100644 index 0000000000000000000000000000000000000000..3ba1e047051afec415af52a28ed4ffbbe54b88c7 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/Queries/PerLaboratories.cs @@ -0,0 +1,71 @@ +namespace Qbb.Datasets.Medical.Queries +{ + using PINQ; + using Qbb.Queries.Core; + using System.Linq; + using System.Linq.Expressions; + using System; + + public class PerLaboratoriesCount : IPINQCondition + { + private class KeyExpression + { + public static Expression> Generate(Expression> countExpression) + { +#if DEBUG + if (countExpression == null) throw new ArgumentNullException(nameof(countExpression)); +#endif + Expression expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(100.0)), Expression.Constant(100), Expression.Constant(101)); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(50.0)), Expression.Constant(50), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(20.0)), Expression.Constant(20), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(10.0)), Expression.Constant(10), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(5.0)), Expression.Constant(5), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(4.0)), Expression.Constant(4), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(3.0)), Expression.Constant(3), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(2.0)), Expression.Constant(2), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(1.0)), Expression.Constant(1), expression); + expression = Expression.Condition(Expression.LessThanOrEqual(countExpression.Body, Expression.Constant(0.0)), Expression.Constant(0), expression); + return Expression.Lambda>(expression, countExpression.Parameters); + } + } + + private static readonly int[] Keys = new int[] { 0, 1, 2, 3, 4, 5, 10, 20, 50, 100, 101 }; + + private readonly Expression> CountExpression; + + public PerLaboratoriesCount(Expression> countExpression) + { +#if DEBUG + if (countExpression == null) throw new ArgumentNullException(nameof(countExpression)); +#endif + CountExpression = countExpression; + } + + public IQueryable[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + IQueryable[] queryables = new IQueryable[Keys.Length]; + foreach (var group in queryable.GroupBy(KeyExpression.Generate(CountExpression))) queryables[Array.IndexOf(Keys, group.Key)] = group.AsQueryable(); + for (int i = 0; i < queryables.Length; i++) if (queryables[i] == null) queryables[i] = Enumerable.Empty().AsQueryable(); + return queryables; + } + + public PINQueryable[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var queryableMap = queryable.Partition(Keys, KeyExpression.Generate(CountExpression)); + var queryables = new PINQueryable[Keys.Length]; + for (int i = 0; i < Keys.Length; i++) queryables[i] = queryableMap[Keys[i]]; + return queryables; + } + + public IPINQCondition Clone() + { + return new PerLaboratoriesCount(CountExpression); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Medical/Queries/WhereSusp.cs b/source/C#/UniTraX/Datasets/Medical/Queries/WhereSusp.cs new file mode 100644 index 0000000000000000000000000000000000000000..716e38e8fd9f1a015b80f68908557a2ee95983a1 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Medical/Queries/WhereSusp.cs @@ -0,0 +1,48 @@ +namespace Qbb.Datasets.Medical.Queries +{ + using PINQ; + using Qbb.Queries.Core; + using System; + using System.Linq; + using System.Linq.Expressions; + + public class WhereSusp : IPINQCondition + { + private readonly Expression> SuspExpression; + + public WhereSusp(Expression> suspExpression) + { +#if DEBUG + if (suspExpression == null) throw new ArgumentNullException(nameof(suspExpression)); +#endif + SuspExpression = suspExpression; + } + + public IQueryable[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return new IQueryable[] + { + queryable.Where(SuspExpression) + }; + } + + public PINQueryable[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return new PINQueryable[] + { + queryable.Where(SuspExpression) + }; + } + + public IPINQCondition Clone() + { + return new WhereSusp(SuspExpression); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/DataAccess/Metadata.cs b/source/C#/UniTraX/Datasets/Mobility/DataAccess/Metadata.cs new file mode 100644 index 0000000000000000000000000000000000000000..5b8611de85019ff7f266fdb54fa9804aedc00688 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/DataAccess/Metadata.cs @@ -0,0 +1,92 @@ +namespace Qbb.Datasets.Mobility.DataAccess +{ + using Qbb.Core.BoundedData; + using Qbb.Core.Specification; + using Qbb.Infrastructure.DataConversion; + using System; + using System.Linq; + + public class Metadata : IQueryableFactory + { + private static DoubleTypeConverter TypeMobilityPk = new DoubleTypeConverter(new DoubleType(1.0, 15000000.0, 1.0)); + private static DoubleTypeConverter TypeMobilityPickup = new DoubleTypeConverter(new DoubleType(1356994800.0, 1359673199.0, 1.0)); + private static DoubleTypeConverter TypeMobilityDropoff = new DoubleTypeConverter(new DoubleType(1356994800.0, 1359716399.0, 1.0)); + private static DoubleTypeConverter TypeMobilityCount = new DoubleTypeConverter(new DoubleType(0.0, 300.0, 1.0)); + private static DoubleTypeConverter TypeMobilityDuration = new DoubleTypeConverter(new DoubleType(0.0, 10800.0, 1.0)); + private static DoubleTypeConverter TypeMobilityDistance = new DoubleTypeConverter(new DoubleType(0.0, 100.0, 0.01)); + private static DoubleTypeConverter TypeMobilityGps = new DoubleTypeConverter(new DoubleType(-5000.0, 5000.0, 0.000001)); + private static DoubleTypeConverter TypeMobilityDollars = new DoubleTypeConverter(new DoubleType(0.0, 1000.0, 0.01)); + private static DoubleTypeConverter TypeMobilityBudget = new DoubleTypeConverter(new DoubleType(0.0, 10000000000.0, 0.01)); + + private static Record GenerateRecord() + { + var fields = new[] + { + new Field(0, "Pk", TypeMobilityPk, accessRecord => accessRecord.Pk), + new Field(1, "PickupDatetime", TypeMobilityPickup, accessRecord => accessRecord.PickupDatetime), + new Field(2, "DropoffDatetime", TypeMobilityDropoff, accessRecord => accessRecord.DropoffDatetime), + new Field(3, "PassengerCount", TypeMobilityCount, accessRecord => accessRecord.PassengerCount), + new Field(4, "TripTimeInSecs", TypeMobilityDuration, accessRecord => accessRecord.TripTimeInSecs), + new Field(5, "TripDistance", TypeMobilityDistance, accessRecord => accessRecord.TripDistance), + new Field(6, "PickupLongitude", TypeMobilityGps, accessRecord => accessRecord.PickupLongitude), + new Field(7, "PickupLatitude", TypeMobilityGps, accessRecord => accessRecord.PickupLatitude), + new Field(8, "DropoffLongitude", TypeMobilityGps, accessRecord => accessRecord.DropoffLongitude), + new Field(9, "DropoffLatitude", TypeMobilityGps, accessRecord => accessRecord.DropoffLatitude), + new Field(10, "FareAmount", TypeMobilityDollars, accessRecord => accessRecord.FareAmount), + new Field(11, "Surcharge", TypeMobilityDollars, accessRecord => accessRecord.Surcharge), + new Field(12, "MtaTax", TypeMobilityDollars, accessRecord => accessRecord.MtaTax), + new Field(13, "TipAmount", TypeMobilityDollars, accessRecord => accessRecord.TipAmount), + new Field(14, "TollsAmount", TypeMobilityDollars, accessRecord => accessRecord.TollsAmount), + new Field(15, "TotalAmount", TypeMobilityDollars, accessRecord => accessRecord.TotalAmount) + }; + return new Record(fields); + } + + private static Record GenerateRecordB() + { + var fields = new[] + { + new Field(0, "Pk", TypeMobilityPk, accessRecord => accessRecord.Pk), + new Field(1, "PickupDatetime", TypeMobilityPickup, accessRecord => accessRecord.PickupDatetime), + new Field(2, "DropoffDatetime", TypeMobilityDropoff, accessRecord => accessRecord.DropoffDatetime), + new Field(3, "PassengerCount", TypeMobilityCount, accessRecord => accessRecord.PassengerCount), + new Field(4, "TripTimeInSecs", TypeMobilityDuration, accessRecord => accessRecord.TripTimeInSecs), + new Field(5, "TripDistance", TypeMobilityDistance, accessRecord => accessRecord.TripDistance), + new Field(6, "PickupLongitude", TypeMobilityGps, accessRecord => accessRecord.PickupLongitude), + new Field(7, "PickupLatitude", TypeMobilityGps, accessRecord => accessRecord.PickupLatitude), + new Field(8, "DropoffLongitude", TypeMobilityGps, accessRecord => accessRecord.DropoffLongitude), + new Field(9, "DropoffLatitude", TypeMobilityGps, accessRecord => accessRecord.DropoffLatitude), + new Field(10, "FareAmount", TypeMobilityDollars, accessRecord => accessRecord.FareAmount), + new Field(11, "Surcharge", TypeMobilityDollars, accessRecord => accessRecord.Surcharge), + new Field(12, "MtaTax", TypeMobilityDollars, accessRecord => accessRecord.MtaTax), + new Field(13, "TipAmount", TypeMobilityDollars, accessRecord => accessRecord.TipAmount), + new Field(14, "TollsAmount", TypeMobilityDollars, accessRecord => accessRecord.TollsAmount), + new Field(15, "TotalAmount", TypeMobilityDollars, accessRecord => accessRecord.TotalAmount), + new Field(16, "Budget", TypeMobilityBudget, accessRecord => accessRecord.Budget) + }; + return new Record(16, fields); + } + + public static readonly Record Record = GenerateRecord(); + public static readonly Record RecordB = GenerateRecordB(); + + public string ConnectionString { get; private set; } + public BoundedDataAccess BoundedDataAccess { get; private set; } + public IQueryable Rides; + + public Metadata(string connectionString) + { +#if DEBUG + if (connectionString == null) throw new ArgumentNullException(nameof(connectionString)); +#endif + ConnectionString = connectionString; + BoundedDataAccess = new BoundedDataAccess(RecordB, this, new ExpressionProvider()); + Rides = new MobilityB(ConnectionString).Rides; + } + + public IQueryable Create() + { + return Rides; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/DataAccess/Mobility.cs b/source/C#/UniTraX/Datasets/Mobility/DataAccess/Mobility.cs new file mode 100644 index 0000000000000000000000000000000000000000..a06b3150ad760ab3dd19a5b86cf2f7eaba11bf10 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/DataAccess/Mobility.cs @@ -0,0 +1,21 @@ +namespace Qbb.Datasets.Mobility.DataAccess +{ + using System.Data.Linq.Mapping; + + [Database(Name = "mobility")] + public class Mobility : System.Data.Linq.DataContext + { + private static readonly MappingSource MappingSource = new AttributeMappingSource(); + + public Mobility(string connectionString) : base(connectionString, MappingSource) + { + ObjectTrackingEnabled = false; + CommandTimeout = 300; + } + + public System.Data.Linq.Table Rides + { + get { return GetTable(); } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/DataAccess/MobilityB.cs b/source/C#/UniTraX/Datasets/Mobility/DataAccess/MobilityB.cs new file mode 100644 index 0000000000000000000000000000000000000000..9d1dbe6208ac7e5a956087c9037b43f6ad5576ce --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/DataAccess/MobilityB.cs @@ -0,0 +1,21 @@ +namespace Qbb.Datasets.Mobility.DataAccess +{ + using System.Data.Linq.Mapping; + + [Database(Name = "mobility")] + public class MobilityB : System.Data.Linq.DataContext + { + private static readonly MappingSource MappingSource = new AttributeMappingSource(); + + public MobilityB(string connectionString) : base(connectionString, MappingSource) + { + ObjectTrackingEnabled = false; + CommandTimeout = 300; + } + + public System.Data.Linq.Table Rides + { + get { return GetTable(); } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/DataAccess/Ride.cs b/source/C#/UniTraX/Datasets/Mobility/DataAccess/Ride.cs new file mode 100644 index 0000000000000000000000000000000000000000..87b4208e8859bf830cd95c6511f72d847957915f --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/DataAccess/Ride.cs @@ -0,0 +1,41 @@ +namespace Qbb.Datasets.Mobility.DataAccess +{ + using System.Data.Linq.Mapping; + + [Table(Name = "rides")] + public class Ride + { + [Column(Name = "pk", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Pk; + [Column(Name = "pickup_datetime", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PickupDatetime; + [Column(Name = "dropoff_datetime", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DropoffDatetime; + [Column(Name = "passenger_count", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PassengerCount; + [Column(Name = "trip_time_in_secs", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TripTimeInSecs; + [Column(Name = "trip_distance", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TripDistance; + [Column(Name = "pickup_longitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PickupLongitude; + [Column(Name = "pickup_latitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PickupLatitude; + [Column(Name = "dropoff_longitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DropoffLongitude; + [Column(Name = "dropoff_latitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DropoffLatitude; + [Column(Name = "fare_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double FareAmount; + [Column(Name = "surcharge", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Surcharge; + [Column(Name = "mta_tax", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double MtaTax; + [Column(Name = "tip_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TipAmount; + [Column(Name = "tolls_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TollsAmount; + [Column(Name = "total_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TotalAmount; + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/DataAccess/RideB.cs b/source/C#/UniTraX/Datasets/Mobility/DataAccess/RideB.cs new file mode 100644 index 0000000000000000000000000000000000000000..49c6b5cdcc4ac76c7bf6af39b1699cf34a0f55ed --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/DataAccess/RideB.cs @@ -0,0 +1,43 @@ +namespace Qbb.Datasets.Mobility.DataAccess +{ + using System.Data.Linq.Mapping; + + [Table(Name = "rides")] + public class RideB + { + [Column(Name = "pk", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never, IsPrimaryKey = true)] + public double Pk; + [Column(Name = "pickup_datetime", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PickupDatetime; + [Column(Name = "dropoff_datetime", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DropoffDatetime; + [Column(Name = "passenger_count", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PassengerCount; + [Column(Name = "trip_time_in_secs", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TripTimeInSecs; + [Column(Name = "trip_distance", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TripDistance; + [Column(Name = "pickup_longitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PickupLongitude; + [Column(Name = "pickup_latitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double PickupLatitude; + [Column(Name = "dropoff_longitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DropoffLongitude; + [Column(Name = "dropoff_latitude", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double DropoffLatitude; + [Column(Name = "fare_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double FareAmount; + [Column(Name = "surcharge", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Surcharge; + [Column(Name = "mta_tax", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double MtaTax; + [Column(Name = "tip_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TipAmount; + [Column(Name = "tolls_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TollsAmount; + [Column(Name = "total_amount", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double TotalAmount; + [Column(Name = "budget", DbType = "Float NOT NULL", UpdateCheck = UpdateCheck.Never)] + public double Budget; + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/DataLoading/MobilityDisk.cs b/source/C#/UniTraX/Datasets/Mobility/DataLoading/MobilityDisk.cs new file mode 100644 index 0000000000000000000000000000000000000000..b7105d9bbbfe2f18f08976725fae68b6b8d8a363 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/DataLoading/MobilityDisk.cs @@ -0,0 +1,122 @@ +namespace Qbb.Datasets.Mobility.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using Qbb.Infrastructure.DataLoading; + using System; + + public class MobilityDisk + { + public static string[] ColumnNames = new string[] { + "pk", + "pickup_datetime", + "dropoff_datetime", + "passenger_count", + "trip_time_in_secs", + "trip_distance", + "pickup_longitude", + "pickup_latitude", + "dropoff_longitude", + "dropoff_latitude", + "fare_amount", + "surcharge", + "mta_tax", + "tip_amount", + "tolls_amount", + "total_amount", + "budget" + }; + + public static void CreateTable(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + TableDisk.CreateTable(database, withBudget, "mobility-disk", "rides", ColumnNames, "pk", "budget", null); + } + + public static void CreateIndexes(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Console.WriteLine("MobilityDisk: Creating indexes on table rides " + (withBudget ? "with" : "without") + " budget."); + var table = database.Tables["rides"]; + var indexPD = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_pickup_datetime"); + indexPD.IndexedColumns.Add(new IndexedColumn(indexPD, "pickup_datetime", false)); + if (withBudget) indexPD.IndexedColumns.Add(new IndexedColumn(indexPD, "budget", false)); + table.Indexes.Add(indexPD); + indexPD.Create(); + indexPD.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexPD.Alter(); + + var indexToA = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_total_amount"); + indexToA.IndexedColumns.Add(new IndexedColumn(indexToA, "total_amount", false)); + if (withBudget) indexToA.IndexedColumns.Add(new IndexedColumn(indexToA, "budget", false)); + table.Indexes.Add(indexToA); + indexToA.Create(); + indexToA.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexToA.Alter(); + + var indexTiA = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_tip_amount"); + indexTiA.IndexedColumns.Add(new IndexedColumn(indexTiA, "tip_amount", false)); + if (withBudget) indexTiA.IndexedColumns.Add(new IndexedColumn(indexTiA, "budget", false)); + table.Indexes.Add(indexTiA); + indexTiA.Create(); + indexTiA.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexTiA.Alter(); + + var indexPC = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_passenger_count"); + indexPC.IndexedColumns.Add(new IndexedColumn(indexPC, "passenger_count", false)); + if (withBudget) indexPC.IndexedColumns.Add(new IndexedColumn(indexPC, "budget", false)); + table.Indexes.Add(indexPC); + indexPC.Create(); + indexPC.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexPC.Alter(); + + var indexTTIS = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_trip_time_in_secs"); + indexTTIS.IndexedColumns.Add(new IndexedColumn(indexTTIS, "trip_time_in_secs", false)); + if (withBudget) indexTTIS.IndexedColumns.Add(new IndexedColumn(indexTTIS, "budget", false)); + table.Indexes.Add(indexTTIS); + indexTTIS.Create(); + indexTTIS.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexTTIS.Alter(); + + var indexTD = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_trip_distance"); + indexTD.IndexedColumns.Add(new IndexedColumn(indexTD, "trip_distance", false)); + if (withBudget) indexTD.IndexedColumns.Add(new IndexedColumn(indexTD, "budget", false)); + table.Indexes.Add(indexTD); + indexTD.Create(); + indexTD.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexTD.Alter(); + + var indexPL = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_pickup_longitude_pickup_latitude"); + indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "pickup_longitude", false)); + indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "pickup_latitude", false)); + if (withBudget) indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "budget", false)); + indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "trip_distance") { IsIncluded = true }); + indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "tip_amount") { IsIncluded = true }); + indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "total_amount") { IsIncluded = true }); + table.Indexes.Add(indexPL); + indexPL.Create(); + indexPL.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexPL.Alter(); + + var indexStream = TableDisk.CreateDefaultIndex(table, "index_nc_mobility-disk_rides_stream"); + indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "dropoff_datetime", false)); + indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "pickup_longitude", false)); + indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "pickup_latitude", false)); + indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "dropoff_longitude", false)); + indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "dropoff_latitude", false)); + if (withBudget) indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "budget", false)); + indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "trip_time_in_secs") { IsIncluded = true }); + indexStream.IndexedColumns.Add(new IndexedColumn(indexStream, "trip_distance") { IsIncluded = true }); + table.Indexes.Add(indexStream); + indexStream.Create(); + indexStream.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexStream.Alter(); + + table.Alter(); + Console.WriteLine("MobilityDisk: Created indexes on table rides."); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/DataLoading/MobilityMem.cs b/source/C#/UniTraX/Datasets/Mobility/DataLoading/MobilityMem.cs new file mode 100644 index 0000000000000000000000000000000000000000..1d12201e379ac6b50eed782f68e34723cd7e8ac2 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/DataLoading/MobilityMem.cs @@ -0,0 +1,53 @@ +namespace Qbb.Datasets.Mobility.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using Qbb.Infrastructure.DataLoading; + using System; + + public class MobilityMem + { + public static void CreateTable(Database database, bool withBudget) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); +#endif + Table table = TableMem.CreateTableInit(database, withBudget, "mobility-mem", "rides", MobilityDisk.ColumnNames, "pk", "budget", null); + var indexPD = TableMem.CreateDefaultIndex(table, "index_nc_mobility-mem_rides_pickup_datetime"); + indexPD.IndexedColumns.Add(new IndexedColumn(indexPD, "pickup_datetime", false)); + if (withBudget) indexPD.IndexedColumns.Add(new IndexedColumn(indexPD, "budget", false)); + table.Indexes.Add(indexPD); + + var indexToA = TableMem.CreateDefaultIndex(table, "index_nc_mobility-mem_rides_total_amount"); + indexToA.IndexedColumns.Add(new IndexedColumn(indexToA, "total_amount", false)); + if (withBudget) indexToA.IndexedColumns.Add(new IndexedColumn(indexToA, "budget", false)); + table.Indexes.Add(indexToA); + + var indexTiA = TableMem.CreateDefaultIndex(table, "index_nc_mobility-mem_rides_tip_amount"); + indexTiA.IndexedColumns.Add(new IndexedColumn(indexTiA, "tip_amount", false)); + if (withBudget) indexTiA.IndexedColumns.Add(new IndexedColumn(indexTiA, "budget", false)); + table.Indexes.Add(indexTiA); + + var indexPC = TableMem.CreateDefaultIndex(table, "index_nc_mobility-mem_rides_passenger_count"); + indexPC.IndexedColumns.Add(new IndexedColumn(indexPC, "passenger_count", false)); + if (withBudget) indexPC.IndexedColumns.Add(new IndexedColumn(indexPC, "budget", false)); + table.Indexes.Add(indexPC); + + var indexTTIS = TableMem.CreateDefaultIndex(table, "index_nc_mobility-mem_rides_trip_time_in_secs"); + indexTTIS.IndexedColumns.Add(new IndexedColumn(indexTTIS, "trip_time_in_secs", false)); + if (withBudget) indexTTIS.IndexedColumns.Add(new IndexedColumn(indexTTIS, "budget", false)); + table.Indexes.Add(indexTTIS); + + var indexTD = TableMem.CreateDefaultIndex(table, "index_nc_mobility-mem_rides_trip_distance"); + indexTD.IndexedColumns.Add(new IndexedColumn(indexTD, "trip_distance", false)); + if (withBudget) indexTD.IndexedColumns.Add(new IndexedColumn(indexTD, "budget", false)); + table.Indexes.Add(indexTD); + + var indexPL = TableMem.CreateDefaultIndex(table, "index_nc_mobility-mem_rides_pickup_longitude_pickup_latitude"); + indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "pickup_longitude", false)); + indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "pickup_latitude", false)); + if (withBudget) indexPL.IndexedColumns.Add(new IndexedColumn(indexPL, "budget", false)); + table.Indexes.Add(indexPL); + TableMem.CreateTableFinish(table, "rides"); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/Experiment/MobilityExperiment.cs b/source/C#/UniTraX/Datasets/Mobility/Experiment/MobilityExperiment.cs new file mode 100644 index 0000000000000000000000000000000000000000..68e7d0fc2cab83c89df9a8ca5f726829dd3a69a2 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/Experiment/MobilityExperiment.cs @@ -0,0 +1,399 @@ +namespace Qbb.Datasets.Mobility.Experiment +{ + using Qbb.Datasets.Mobility.DataAccess; + using Qbb.Datasets.Mobility.DataLoading; + using Qbb.Datasets.Mobility.Queries; + using Qbb.Infrastructure.BudgetProvider; + using Qbb.Infrastructure.DataConversion; + using Qbb.Infrastructure.DataLoading; + using Qbb.Infrastructure.ExperimentUtils; + using Qbb.Infrastructure.Provisioning; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + + public class MobilityQueriesOriginal + { + public AnalyticBaseProvider Provider { get; private set; } + + public MobilityQueriesOriginal(long baseId, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long defaultEpsilon) + { +#if DEBUG + if (defaultEpsilon < 0L) throw new ArgumentException("defaultEpsilon is below zero"); +#endif + var factories = new List>(8) + { + new RidesPerPickupDateTime(baseId + 1L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new RidesPerTotalAmount(baseId + 2L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new RidesPerTipAmount(baseId + 3L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new RidesPerPassenegerCount(baseId + 4L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new RidesPerTripTimeInSecs(baseId + 5L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new RidesPerTripDistance(baseId + 6L, useEvenly, budgetOverRuntime, cleanThreshold, defaultEpsilon), + new LocalInvestigation1(baseId + 7L, useEvenly, budgetOverRuntime, cleanThreshold, new long[3] { defaultEpsilon, defaultEpsilon, defaultEpsilon }), + new LocalInvestigation2(baseId + 8L, useEvenly, budgetOverRuntime, cleanThreshold, new long[2] { defaultEpsilon, defaultEpsilon }) + }; + Provider = new AnalyticBaseProvider(factories); + } + } + + public class MobilityExperimentOriginal + { + public static bool DiskOnly = false; + public static bool RuntimeOnly = false; + public static int Repeats = 8; + public static int CleanThreshold = 500; + public static long BaseId = 1100L; + public static long DefaultEpsilon = DoubleTypeConverter.TwoDecimal.ToSystem(10000.0); + public static string BaseDirName = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "experiments", "mobility", "experiment"); + public static string InstanceName = BaseId + "_" + DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss-fff", CultureInfo.InvariantCulture); + public static string InstanceDirName = Path.Combine(BaseDirName, InstanceName); + public static string DataFileName = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "mobility", "transformed", "rides.csv"); + public static double DefaultTotalBudget = 100000000.0; + public static IBudgetProvider BudgetProvider = new ConstantBudgetProvider(DefaultTotalBudget); + + public static void SetupDatabaseMem(IExperimentServer server, IBudgetProvider budgetProvider) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); +#endif + server.Setup(); + server.CreateDatabaseInMem("mobility-mem"); + MobilityMem.CreateTable(server.GetDatabase("mobility-mem"), budgetProvider != null); + var dataTable = new DataTableProvider(DataFileName, budgetProvider, MobilityDisk.ColumnNames, "budget", null).GetDataTable(); + var connection = server.GetSqlConnection("mobility-mem"); + connection.Open(); + TableMem.FillTable(dataTable, connection, "rides"); + connection.Close(); + dataTable.Dispose(); + } + + public static void SetupDatabaseDisk(IExperimentServer server, IBudgetProvider budgetProvider) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); +#endif + server.Setup(); + server.CreateDatabaseOnDisk("mobility-disk"); + MobilityDisk.CreateTable(server.GetDatabase("mobility-disk"), budgetProvider != null); + var dataTable = new DataTableProvider(DataFileName, budgetProvider, MobilityDisk.ColumnNames, "budget", null).GetDataTable(); + var connection = server.GetSqlConnection("mobility-disk"); + connection.Open(); + TableDisk.FillTable(dataTable, connection, "rides"); + connection.Close(); + dataTable.Dispose(); + MobilityDisk.CreateIndexes(server.GetDatabase("mobility-disk"), budgetProvider != null); + server.MakeDatabaseReadonly("mobility-disk"); + } + + public static void Main(string[] args) + { +#if RECORD + var timerT = Recorder.StartTimer(); +#endif + Console.WriteLine("Experiment Mobility Original: STARTED"); + var server = new ExampleDBServer(); + + var queriesAlwaysClean = new MobilityQueriesOriginal(BaseId, false, false, -1, DefaultEpsilon); + var queriesAlwaysCleanEven = new MobilityQueriesOriginal(BaseId, true, false, -1, DefaultEpsilon); + var queriesThresholdClean = new MobilityQueriesOriginal(BaseId, false, false, CleanThreshold, DefaultEpsilon); + var queriesThresholdCleanEven = new MobilityQueriesOriginal(BaseId, true, false, CleanThreshold, DefaultEpsilon); + + var queriesAlwaysCleanBudget = new MobilityQueriesOriginal(BaseId, false, true, -1, DefaultEpsilon); + var queriesAlwaysCleanEvenBudget = new MobilityQueriesOriginal(BaseId, true, true, -1, DefaultEpsilon); + var queriesThresholdCleanBudget = new MobilityQueriesOriginal(BaseId, false, true, CleanThreshold, DefaultEpsilon); + var queriesThresholdCleanEvenBudget = new MobilityQueriesOriginal(BaseId, true, true, CleanThreshold, DefaultEpsilon); + + Metadata metadataMem = null; + if (!DiskOnly) metadataMem = new Metadata(server.GetDatabaseConnectionString("mobility-mem")); + var metadataDisk = new Metadata(server.GetDatabaseConnectionString("mobility-disk")); + + Directory.CreateDirectory(InstanceDirName); + + if (!RuntimeOnly) + { + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanBudget.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanEvenBudget.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Even-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Direct-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Disk-Direct-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Direct-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Direc-Budget: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Disk-Direct-Budget", server, + new Mobility(server.GetDatabaseConnectionString("mobility-disk")).Rides, + queriesAlwaysCleanBudget.Provider.GetQueriesDirect(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Direct-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-PINQ-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Disk-PINQ-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Disk-PINQ-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-PINQ-Budget: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Disk-PINQ-Budget", server, + new PINQueryableFactory(new Mobility(server.GetDatabaseConnectionString("mobility-disk")).Rides, DefaultTotalBudget), + queriesAlwaysCleanBudget.Provider.GetQueriesPINQ(), DefaultTotalBudget, "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-PINQ-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanBudget.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Budget", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanEvenBudget.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Even-Budget: FINISHED"); + } + + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysClean.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Always-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Always-BK-Even-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesAlwaysCleanEven.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Always-BK-Even-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Direct-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Disk-Direct-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Direct-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Direc-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Disk-Direct-Runtime", server, + new Mobility(server.GetDatabaseConnectionString("mobility-disk")).Rides, + queriesAlwaysClean.Provider.GetQueriesDirect(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Direct-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-PINQ-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Disk-PINQ-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Disk-PINQ-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-PINQ-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Disk-PINQ-Runtime", server, + new PINQueryableFactory(new Mobility(server.GetDatabaseConnectionString("mobility-disk")).Rides, DefaultTotalBudget), + queriesAlwaysClean.Provider.GetQueriesPINQ(), DefaultTotalBudget, "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-PINQ-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdClean.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseDisk(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Disk-Threshold-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Disk-Threshold-BK-Even-Runtime", server, + new BKQueryableFactory(metadataDisk.BoundedDataAccess), + queriesThresholdCleanEven.Provider.GetQueriesBK(), "mobility-disk", Repeats); + Console.WriteLine("SubExperiment Mobility Disk-Threshold-BK-Even-Runtime: FINISHED"); + + + if (!DiskOnly) + { + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysClean.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanEven.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Even-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Direct-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Mem-Direct-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Direct-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Direct-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Mem-Direct-Runtime", server, + new Mobility(server.GetDatabaseConnectionString("mobility-mem")).Rides, + queriesAlwaysClean.Provider.GetQueriesDirect(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Direct-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-PINQ-Runtime-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Mem-PINQ-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Mem-PINQ-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-PINQ-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Mem-PINQ-Runtime", server, + new PINQueryableFactory(new Mobility(server.GetDatabaseConnectionString("mobility-mem")).Rides, DefaultTotalBudget), + queriesAlwaysClean.Provider.GetQueriesPINQ(), DefaultTotalBudget, "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-PINQ-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdClean.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Runtime: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Even-Runtime: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Runtime"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Even-Runtime: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Even-Runtime: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Runtime", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanEven.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Even-Runtime: FINISHED"); + + if (!RuntimeOnly) + { + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanBudget.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Always-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Always-BK-Even-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesAlwaysCleanEvenBudget.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Always-BK-Even-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Direct-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, "Mem-Direct-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Direct-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Direct-Budget: STARTED"); + ExperimentUtils.RepeatExperimentDirect(InstanceDirName, "Mem-Direct-Budget", server, + new Mobility(server.GetDatabaseConnectionString("mobility-mem")).Rides, + queriesAlwaysCleanBudget.Provider.GetQueriesDirect(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Direct-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-PINQ-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, null); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, "Mem-PINQ-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Mem-PINQ-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-PINQ-Budget: STARTED"); + ExperimentUtils.RepeatExperimentPINQ(InstanceDirName, "Mem-PINQ-Budget", server, + new PINQueryableFactory(new Mobility(server.GetDatabaseConnectionString("mobility-mem")).Rides, DefaultTotalBudget), + queriesAlwaysCleanBudget.Provider.GetQueriesPINQ(), DefaultTotalBudget, "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-PINQ-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanBudget.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Budget: FINISHED"); + + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Even-Budget: STARTED"); + server.Wipe(); + SetupDatabaseMem(server, BudgetProvider); + ExperimentUtils.SetupExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Budget"); + Console.WriteLine("Setup SubExperiment Mobility Mem-Threshold-BK-Even-Budget: FINISHED"); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Even-Budget: STARTED"); + ExperimentUtils.RepeatExperimentBK(InstanceDirName, "Mem-Threshold-BK-Even-Budget", server, + new BKQueryableFactory(metadataMem.BoundedDataAccess), + queriesThresholdCleanEvenBudget.Provider.GetQueriesBK(), "mobility-mem", Repeats); + Console.WriteLine("SubExperiment Mobility Mem-Threshold-BK-Even-Budget: FINISHED"); + } + } + +#if RECORD + var timeT = Recorder.StopTimer(timerT); + Console.WriteLine("Took time ms: " + timeT); +#endif + Console.WriteLine("Experiment Mobility Original: FINISHED"); + Console.Read(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/ITimedAction.cs b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/ITimedAction.cs new file mode 100644 index 0000000000000000000000000000000000000000..5f91348a5cc875cef00c5c9425cb77e90db4ad4c --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/ITimedAction.cs @@ -0,0 +1,19 @@ +namespace Qbb.Datasets.Mobility.ExperimentStream +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using System.Data.SqlClient; + using System.Linq; + + public interface ITimedAction + { + int GetId(); + bool HasNextTime(); + long GetNextTime(); + void Execute(IQueryable queryable, long endOn); + void Execute(PINQueryable queryable, long endOn); + void Execute(BKQueryable queryable, long endOn); + void Reset(SqlConnection sqlConnection); + void Close(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/MobilityExperimentStream.cs b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/MobilityExperimentStream.cs new file mode 100644 index 0000000000000000000000000000000000000000..6b6276a766101484c6a274ccf0b349e84c5e6b84 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/MobilityExperimentStream.cs @@ -0,0 +1,302 @@ +namespace Qbb.Datasets.Mobility.ExperimentStream +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.History; + using Qbb.Core.Partition; + using Qbb.Core.Specification; + using Qbb.Core.Utils; + using Qbb.Datasets.Mobility.DataAccess; + using Qbb.Datasets.Mobility.DataLoading; + using Qbb.Datasets.Mobility.Queries; + using Qbb.Infrastructure.BudgetProvider; + using Qbb.Infrastructure.DataConversion; + using Qbb.Infrastructure.DataLoading; + using Qbb.Infrastructure.ExperimentUtils; + using Qbb.Infrastructure.Provisioning; + using Qbb.Queries.Recording; + using System; + using System.Collections.Generic; + using System.Data; + using System.Globalization; + using System.IO; + using System.Linq; + + public class MobilityQueriesStream + { + public List> List { get; private set; } + + public MobilityQueriesStream(long baseId, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long defaultEpsilon, Field field, Field fieldB, ContiguousPartition originalFocus, int timedColumnIndex) + { +#if DEBUG + if (defaultEpsilon < 0L) throw new ArgumentException("defaultEpsilon is below zero"); + if (field == null) throw new ArgumentNullException(nameof(field)); + if (fieldB == null) throw new ArgumentNullException(nameof(fieldB)); + if (originalFocus == null) throw new ArgumentNullException(nameof(originalFocus)); +#endif + List = new List>(2) + { + new TimedQueryExecution(0, field, fieldB, new StreamInvestigationHourly(baseId + 1L, useEvenly, budgetOverRuntime, cleanThreshold, + new long[8] { defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon, defaultEpsilon }), 1356994800L, 1359673200L, 3600L, originalFocus, timedColumnIndex), + new TimedQueryExecution(1, field, fieldB, new StreamInvestigationMonthly(baseId + 2L, useEvenly, budgetOverRuntime, cleanThreshold, + new long[2] { defaultEpsilon, defaultEpsilon }), 1356994800L, 1359673200L, 2678400L, originalFocus, timedColumnIndex) + }; + } + } + + public class MobilityExperimentStream + { + public static bool LoadingSimulated = true; + public static int Repeats = 8; + public static int CleanThreshold = 500; + public static long BaseId = 4400L; + public static long DefaultEpsilon = DoubleTypeConverter.TwoDecimal.ToSystem(10000.0); + public static string BaseDirName = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "experiments", "mobility", "experimentStream", ); + public static string InstanceName = BaseId + "_" + DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss-fff", CultureInfo.InvariantCulture); + public static string InstanceDirName = Path.Combine(BaseDirName, InstanceName); + public static string DataFileName = Path.Combine("C:" + Path.DirectorySeparatorChar, "Users", "Administrator", "Documents", "unitrax", "data", "mobility", "transformed", "rides.csv"); + public static double DefaultTotalBudget = 10000000000L; + public static IBudgetProvider BudgetProvider = new ConstantBudgetProvider(DefaultTotalBudget); + + public static TimedRecordLoader SetupDatabaseMem(IExperimentServer server, DoubleTypeConverter typeConverter, int loaderId, DataTable dataTable, bool withBudget, bool loadingSimulated, bool repeat) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); + if (typeConverter == null) throw new ArgumentNullException(nameof(typeConverter)); + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); +#endif + if ((!loadingSimulated) || (loadingSimulated && !repeat)) + { + server.Setup(); + server.CreateDatabaseInMem("mobility-mem"); + MobilityMem.CreateTable(server.GetDatabase("mobility-mem"), withBudget); + } + if (loadingSimulated && !repeat) + { + var connection = server.GetSqlConnection("mobility-mem"); + connection.Open(); + Console.WriteLine("Filling table rides"); + TableMem.FillTable(dataTable, connection, "rides"); + connection.Close(); + } + return new TimedRecordLoader(loaderId, server.GetSqlConnection("mobility-mem"), dataTable, true, "rides", MobilityDisk.ColumnNames[2], 2, typeConverter, loadingSimulated); + } + + public static TimedRecordLoader SetupDatabaseDisk(IExperimentServer server, DoubleTypeConverter typeConverter, int loaderId, DataTable dataTable, bool withBudget, bool loadingSimulated, bool repeat) + { +#if DEBUG + if (server == null) throw new ArgumentNullException(nameof(server)); + if (typeConverter == null) throw new ArgumentNullException(nameof(typeConverter)); + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); +#endif + if ((!loadingSimulated) || (loadingSimulated && !repeat)) + { + server.Setup(); + server.CreateDatabaseOnDisk("mobility-disk"); + MobilityDisk.CreateTable(server.GetDatabase("mobility-disk"), withBudget); + } + if (loadingSimulated && !repeat) + { + var connection = server.GetSqlConnection("mobility-disk"); + connection.Open(); + TableDisk.FillTable(dataTable, connection, "rides"); + connection.Close(); + } + if ((!loadingSimulated) || (loadingSimulated && !repeat)) MobilityDisk.CreateIndexes(server.GetDatabase("mobility-disk"), withBudget); + return new TimedRecordLoader(loaderId, server.GetSqlConnection("mobility-disk"), dataTable, false, "rides", MobilityDisk.ColumnNames[2], 2, typeConverter, loadingSimulated); + } + + public static TimedExecution PrepareTimedExecution(List> queries, bool inMem, IExperimentServer server, DoubleTypeConverter typeConverter, IFieldType fieldType, DataTable dataTable, bool withBudget, bool loadingSimulated, bool repeat) + { +#if DEBUG + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (queries.Count == 0) throw new ArgumentException("queries is empty"); + foreach (ITimedAction q in queries) if (q == null) throw new ArgumentNullException(nameof(q)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (typeConverter == null) throw new ArgumentNullException(nameof(typeConverter)); + if (fieldType == null) throw new ArgumentNullException(nameof(fieldType)); + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); +#endif + if ((!loadingSimulated) || (loadingSimulated && !repeat)) server.Wipe(); + foreach (ITimedAction q in queries) q.Reset(null); + TimedRecordLoader recordLoader; + if (inMem) recordLoader = SetupDatabaseMem(server, typeConverter, queries.Count, dataTable, withBudget, loadingSimulated, repeat); + else recordLoader = SetupDatabaseDisk(server, typeConverter, queries.Count, dataTable, withBudget, loadingSimulated, repeat); + var actions = new List>(queries.Count + 1); + actions.AddRange(queries); + actions.Add(recordLoader); + return new TimedExecution(fieldType, actions); + } + + public static void RepeatDirect(string experimentName, List> queries, bool inMem, IExperimentServer server, DoubleTypeConverter typeConverter, IFieldType fieldType, DataTable dataTable, bool withBudget, bool loadingSimulated) + { +#if DEBUG + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (queries.Count == 0) throw new ArgumentException("queries is empty"); + foreach (ITimedAction q in queries) if (q == null) throw new ArgumentNullException(nameof(q)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (typeConverter == null) throw new ArgumentNullException(nameof(typeConverter)); + if (fieldType == null) throw new ArgumentNullException(nameof(fieldType)); + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); +#endif + Console.WriteLine("Setup SubExperiment Mobility " + experimentName + ": STARTED"); + ExperimentUtils.SetupExperimentDirect(InstanceDirName, experimentName); + Console.WriteLine("Setup SubExperiment Mobility " + experimentName + ": FINISHED"); + Console.WriteLine("SubExperiment Mobility " + experimentName + ": STARTED"); + for (int i = 0; i < Repeats; i++) + { + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": STARTED"); + var experimentDirName = Path.Combine(InstanceDirName, experimentName); + TimedExecution timedExecution = PrepareTimedExecution(queries, inMem, server, typeConverter, fieldType, dataTable, withBudget, loadingSimulated, i != 0); + Console.WriteLine("Execution prepared"); + IQueryable queryable; + if (inMem) queryable = new Mobility(server.GetDatabaseConnectionString("mobility-mem")).Rides; + else queryable = new Mobility(server.GetDatabaseConnectionString("mobility-disk")).Rides; + Console.WriteLine("Queryable prepared"); + Recorder.Reset(); + Recorder.StartGlobalTimer(); + timedExecution.Execute(queryable); + Recorder.StopGlobalTimer(); + Recorder.ToFile(Path.Combine(experimentDirName, "run" + i + "_results.csv")); + FileUtils.WriteLineToFile("0," + queryable.Count(), Path.Combine(experimentDirName, "run" + i + "_budgetUsed.csv")); + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": FINISHED"); + } + Console.WriteLine("SubExperiment Mobility " + experimentName + ": FINISHED"); + } + + public static void RepeatPINQ(string experimentName, List> queries, bool inMem, IExperimentServer server, DoubleTypeConverter typeConverter, IFieldType fieldType, DataTable dataTable, bool withBudget, bool loadingSimulated) + { +#if DEBUG + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (queries.Count == 0) throw new ArgumentException("queries is empty"); + foreach (ITimedAction q in queries) if (q == null) throw new ArgumentNullException(nameof(q)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (typeConverter == null) throw new ArgumentNullException(nameof(typeConverter)); + if (fieldType == null) throw new ArgumentNullException(nameof(fieldType)); + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); +#endif + Console.WriteLine("Setup SubExperiment Mobility " + experimentName + ": STARTED"); + ExperimentUtils.SetupExperimentPINQ(InstanceDirName, experimentName); + Console.WriteLine("Setup SubExperiment Mobility " + experimentName + ": FINISHED"); + Console.WriteLine("SubExperiment Mobility " + experimentName + ": STARTED"); + for (int i = 0; i < Repeats; i++) + { + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": STARTED"); + var experimentDirName = Path.Combine(InstanceDirName, experimentName); + TimedExecution timedExecution = PrepareTimedExecution(queries, inMem, server, typeConverter, fieldType, dataTable, withBudget, loadingSimulated, i != 0); + IQueryable queryable; + if (inMem) queryable = new Mobility(server.GetDatabaseConnectionString("mobility-mem")).Rides; + else queryable = new Mobility(server.GetDatabaseConnectionString("mobility-disk")).Rides; + PINQueryable queryablePINQ; + if (inMem) queryablePINQ = new PINQueryable(new Mobility(server.GetDatabaseConnectionString("mobility-mem")).Rides, new PINQAgentBudget(DefaultTotalBudget)); + else queryablePINQ = new PINQueryable(new Mobility(server.GetDatabaseConnectionString("mobility-disk")).Rides, new PINQAgentBudget(DefaultTotalBudget)); + Recorder.Reset(); + Recorder.StartGlobalTimer(); + timedExecution.Execute(queryablePINQ); + Recorder.StopGlobalTimer(); + Recorder.ToFile(Path.Combine(experimentDirName, "run" + i + "_results.csv")); + FileUtils.WriteLineToFile((DefaultTotalBudget - queryablePINQ.remaining()) + "," + queryable.Count(), Path.Combine(experimentDirName, "run" + i + "_budgetUsed.csv")); + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": FINISHED"); + } + Console.WriteLine("SubExperiment Mobility " + experimentName + ": FINISHED"); + } + + public static void RepeatBK(string experimentName, List> queries, bool inMem, IExperimentServer server, DoubleTypeConverter typeConverter, IFieldType fieldType, DataTable dataTable, bool withBudget, bool loadingSimulated) + { +#if DEBUG + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (queries.Count == 0) throw new ArgumentException("queries is empty"); + foreach (ITimedAction q in queries) if (q == null) throw new ArgumentNullException(nameof(q)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (typeConverter == null) throw new ArgumentNullException(nameof(typeConverter)); + if (fieldType == null) throw new ArgumentNullException(nameof(fieldType)); + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); +#endif + Console.WriteLine("Setup SubExperiment Mobility " + experimentName + ": STARTED"); + ExperimentUtils.SetupExperimentBK(InstanceDirName, experimentName); + Console.WriteLine("Setup SubExperiment Mobility " + experimentName + ": FINISHED"); + Console.WriteLine("SubExperiment Mobility " + experimentName + ": STARTED"); + for (int i = 0; i < Repeats; i++) + { + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": STARTED"); + var experimentDirName = Path.Combine(InstanceDirName, experimentName); + Metadata metadata; + if (inMem) metadata = new Metadata(server.GetDatabaseConnectionString("mobility-mem")); + else metadata = new Metadata(server.GetDatabaseConnectionString("mobility-disk")); + var originalFocus = metadata.BoundedDataAccess.Record.Boundary; + var startTime = originalFocus.Intervals[2].Low; + originalFocus = ResetFocus(originalFocus, 2, startTime); + TimedExecution timedExecution = PrepareTimedExecution(queries, inMem, server, typeConverter, fieldType, dataTable, withBudget, loadingSimulated, i != 0); + BKQueryable queryableBK = new BKQueryable(metadata.BoundedDataAccess.GetQueryable(), new Bookkeeper(originalFocus, metadata.BoundedDataAccess, new InMemoryHistory(metadata.BoundedDataAccess.Record.GetBudgetIndex())), metadata.BoundedDataAccess.Record.Boundary); + Recorder.Reset(); + Recorder.StartGlobalTimer(); + timedExecution.Execute(queryableBK); + Recorder.StopGlobalTimer(); + Recorder.ToFile(Path.Combine(experimentDirName, "run" + i + "_results.csv")); + queryableBK.CleanHistory(); + queryableBK.WriteDetailedBudgetUseToFile(Path.Combine(experimentDirName, "run" + i + "_budgetUsed.csv")); + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": FINISHED"); + } + Console.WriteLine("SubExperiment Mobility " + experimentName + ": FINISHED"); + } + + public static ContiguousPartition ResetFocus(ContiguousPartition focus, int index, long high) + { +#if DEBUG + if (focus == null) throw new ArgumentNullException(nameof(focus)); + if (index < 0) throw new ArgumentException("index below zero"); + if (index > focus.Intervals.Length - 1) throw new ArgumentException("index above number of columns"); + if (!focus.Intervals[index].Encloses(high)) throw new ArgumentException("high not valid in column"); +#endif + var newIntervals = new Interval[focus.Intervals.Length]; + for (int i = 0; i < focus.Intervals.Length; i++) + { + if (i == index) newIntervals[i] = new Interval(focus.Intervals[i].Low, high); + else newIntervals[i] = focus.Intervals[i]; + } + return new ContiguousPartition(newIntervals); + } + + public static void Main(string[] args) + { +#if RECORD + var timerT = Recorder.StartTimer(); +#endif + Console.WriteLine("Experiment Mobility Stream: STARTED"); + var server = new ExampleDBServer(); + + ContiguousPartition originalFocus = Metadata.Record.Boundary; + var startTime = originalFocus.Intervals[2].Low; + originalFocus = ResetFocus(originalFocus, 2, startTime); + + var queriesAlwaysCleanDirectPINQ = new MobilityQueriesStream(BaseId, false, false, -1, DefaultEpsilon, Metadata.Record.Fields[2], Metadata.RecordB.Fields[2], originalFocus, 2); + + var TypeMobilityDropoff = new DoubleTypeConverter(new DoubleType(1356994800.0, 1359716399.0, 1.0)); + + Directory.CreateDirectory(InstanceDirName); + + var dataTable = new DataTableProvider(DataFileName, null, MobilityDisk.ColumnNames, "budget", null).GetDataTable(); + + RepeatDirect("Disk-Direct-Runtime", queriesAlwaysCleanDirectPINQ.List, false, server, TypeMobilityDropoff, Metadata.Record.Fields[2].FieldType, dataTable, false, LoadingSimulated); + RepeatPINQ("Disk-PINQ-Runtime", queriesAlwaysCleanDirectPINQ.List, false, server, TypeMobilityDropoff, Metadata.Record.Fields[2].FieldType, dataTable, false, LoadingSimulated); + + ContiguousPartition originalFocusB = Metadata.RecordB.Boundary; + var startTimeB = originalFocusB.Intervals[2].Low; + originalFocusB = ResetFocus(originalFocusB, 2, startTimeB); + + var queriesAlwaysCleanB = new MobilityQueriesStream(BaseId, false, false, -1, DefaultEpsilon, Metadata.Record.Fields[2], Metadata.RecordB.Fields[2], originalFocusB, 2); + var dataTableB = new DataTableProvider(DataFileName, BudgetProvider, MobilityDisk.ColumnNames, "budget", null).GetDataTable(); + + RepeatBK("Disk-Always-BK-Runtime", queriesAlwaysCleanB.List, false, server, TypeMobilityDropoff, Metadata.Record.Fields[2].FieldType, dataTableB, true, LoadingSimulated); +#if RECORD + var timeT = Recorder.StopTimer(timerT); + Console.WriteLine("Took time ms: " + timeT); +#endif + Console.WriteLine("Experiment Mobility Stream: FINISHED"); + Console.Read(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedExecution.cs b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedExecution.cs new file mode 100644 index 0000000000000000000000000000000000000000..ba23ef1a50dcf76f7b5d7a9da931d5ea0358483e --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedExecution.cs @@ -0,0 +1,105 @@ +namespace Qbb.Datasets.Mobility.ExperimentStream +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using System; + using System.Collections.Generic; + using System.Linq; + + public class TimedExecution + { + private class TimedActionComparer : Comparer> + { + public override int Compare(ITimedAction x, ITimedAction y) + { +#if DEBUG + if (x == null) throw new ArgumentNullException(nameof(x)); + if (y == null) throw new ArgumentNullException(nameof(y)); +#endif + if (!x.HasNextTime() && !y.HasNextTime()) return x.GetId().CompareTo(y.GetId()); + if (!x.HasNextTime()) return 1; + if (!y.HasNextTime()) return -1; + var c = x.GetNextTime().CompareTo(y.GetNextTime()); + if (c == 0) return x.GetId().CompareTo(y.GetId()); + return c; + } + } + + private readonly IFieldType FieldType; + private readonly List> Actions; + private readonly TimedActionComparer Comparer; + + public TimedExecution(IFieldType fieldType, List> actions) + { +#if DEBUG + if (fieldType == null) throw new ArgumentNullException(nameof(fieldType)); + if (actions == null) throw new ArgumentNullException(nameof(actions)); + if (!actions.Any()) throw new ArgumentException("actions is empty"); + foreach (ITimedAction a in actions) + { + if (a == null) throw new ArgumentNullException(nameof(a)); + if (!a.HasNextTime()) throw new ArgumentException("action can never be executed"); + } +#endif + FieldType = fieldType; + Actions = actions; + Comparer = new TimedActionComparer(); + Actions.Sort(Comparer); + } + + public void Execute(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + while (Actions.Count > 0) + { + if (Actions.Count > 1) Actions.First().Execute(queryable, Actions[1].GetNextTime()); + else Actions.First().Execute(queryable, FieldType.Bounds.High); + Actions.Sort(Comparer); + while (Actions.Count > 0 && !Actions.Last().HasNextTime()) + { + Actions.Last().Close(); + Actions.RemoveAt(Actions.Count - 1); + } + } + } + + public void Execute(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + while (Actions.Count > 0) + { + if (Actions.Count > 1) Actions.First().Execute(queryable, Actions[1].GetNextTime()); + else Actions.First().Execute(queryable, FieldType.Bounds.High); + Actions.Sort(Comparer); + while (Actions.Count > 0 && !Actions.Last().HasNextTime()) + { + Actions.Last().Close(); + Actions.RemoveAt(Actions.Count - 1); + } + } + } + + public void Execute(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + while (Actions.Count > 0) + { + if (Actions.Count > 1) Actions.First().Execute(queryable, Actions[1].GetNextTime()); + else Actions.First().Execute(queryable, FieldType.Bounds.High); + Actions.Sort(Comparer); + while (Actions.Count > 0 && !Actions.Last().HasNextTime()) + { + Actions.Last().Close(); + Actions.RemoveAt(Actions.Count - 1); + } + } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedQueryExecution.cs b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedQueryExecution.cs new file mode 100644 index 0000000000000000000000000000000000000000..56a92d054052bde05b0c21f537886c17c9e5e0cf --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedQueryExecution.cs @@ -0,0 +1,112 @@ +namespace Qbb.Datasets.Mobility.ExperimentStream +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Partition; + using Qbb.Core.Specification; + using Qbb.Datasets.Mobility.Queries; + using System; + using System.Collections.Generic; + using System.Data.SqlClient; + using System.Linq; + + public class TimedQueryExecution : ITimedAction + { + private readonly int Id; + private readonly Field Field; + private readonly Field FieldB; + private readonly ITimedAnalyticBaseFactory AnalyticBaseFactory; + private readonly long StartTime; + private readonly long EndTime; + private long NextTime; + private readonly long Interval; + private readonly ContiguousPartition OriginalFocus; + private readonly int TimedColumnIndex; + + public TimedQueryExecution(int id, Field field, Field fieldB, ITimedAnalyticBaseFactory analyticBaseFactory, long startTime, long endTime, long interval, ContiguousPartition originalFocus, int timedColumnIndex) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (fieldB == null) throw new ArgumentNullException(nameof(fieldB)); + if (analyticBaseFactory == null) throw new ArgumentNullException(nameof(analyticBaseFactory)); + if (startTime < endTime && interval < 0) throw new ArgumentException("interval has wrong direction"); + if (startTime > endTime && interval > 0) throw new ArgumentException("interval has wrong direction"); + if (startTime != endTime && interval == 0) throw new ArgumentException("interval is zero"); + if (originalFocus == null) throw new ArgumentNullException(nameof(originalFocus)); + if (startTime < originalFocus.Intervals[timedColumnIndex].Low) throw new ArgumentException("startTime cannot be before timedcolumn low"); +#endif + Id = id; + Field = field; + FieldB = fieldB; + AnalyticBaseFactory = analyticBaseFactory; + StartTime = startTime; + EndTime = endTime; + Interval = interval; + OriginalFocus = originalFocus; + TimedColumnIndex = timedColumnIndex; + Reset(null); + } + + public int GetId() + { + return Id; + } + + public void Execute(IQueryable queryable, long endOn) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (!HasNextTime()) throw new Exception("there is no NextTime"); + AnalyticBaseFactory.GenerateQuery(NextTime, Field, FieldB).GetAnalyticBaseDirect().ApplyTo(queryable); + NextTime += Interval; + } + + public void Execute(PINQueryable queryable, long endOn) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (!HasNextTime()) throw new Exception("there is no NextTime"); + AnalyticBaseFactory.GenerateQuery(NextTime, Field, FieldB).GetAnalyticBasePINQ().ApplyTo(queryable); + NextTime += Interval; + } + + public void Execute(BKQueryable queryable, long endOn) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (!HasNextTime()) throw new Exception("there is no NextTime"); + + var newIntervals = new Interval[OriginalFocus.Intervals.Length]; + for (int i = 0; i < OriginalFocus.Intervals.Length; i++) + { + if (i == TimedColumnIndex) newIntervals[i] = new Interval(OriginalFocus.Intervals[i].Low, NextTime); + else newIntervals[i] = OriginalFocus.Intervals[i]; + } + queryable.Bookkeeper.ExtendFocus(new Partition(new HashSet() { new ContiguousPartition(newIntervals) })); + + AnalyticBaseFactory.GenerateQuery(NextTime, Field, FieldB).GetAnalyticBaseBK().ApplyTo(queryable); + NextTime += Interval; + } + + public bool HasNextTime() + { + return NextTime <= EndTime; + } + + public long GetNextTime() + { + if (!HasNextTime()) throw new Exception("there is no NextTime"); + return NextTime; + } + + public void Close() { } + + public void Reset(SqlConnection sqlConnection) + { + NextTime = StartTime + Interval - 1L; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedRecordLoader.cs b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedRecordLoader.cs new file mode 100644 index 0000000000000000000000000000000000000000..0969a393791358f4562b01f9a490e48074d35e0f --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/ExperimentStream/TimedRecordLoader.cs @@ -0,0 +1,157 @@ +namespace Qbb.Datasets.Mobility.ExperimentStream +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Infrastructure.DataConversion; + using Qbb.Infrastructure.DataLoading; + using System; + using System.Collections.Generic; + using System.Data; + using System.Data.SqlClient; + using System.Linq; + + public class TimedRecordLoader : ITimedAction + { + private readonly int Id; + private readonly DataTable DataTable; + private readonly bool InMem; + private readonly string TableName; + private readonly string TimedColumnName; + private readonly DoubleTypeConverter TimedFieldConverter; + private SqlConnection SqlConnection; + private IEnumerator RowEnumerator; + private bool HasCurrent; + private bool Closed; + private int ConsoleCursorTop; + private int ConsoleMessageLength; + private int CurrentCount; + private readonly bool LoadingSimulated; + + public TimedRecordLoader(int id, SqlConnection sqlConnection, DataTable dataTable, bool inMem, string tableName, string timedColumnName, int timedColumnIndex, DoubleTypeConverter timedFieldConverter, bool loadingSimulated) + { +#if DEBUG + if (sqlConnection == null) throw new ArgumentNullException(nameof(sqlConnection)); + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); + if (tableName == null) throw new ArgumentNullException(nameof(tableName)); + if (timedColumnName == null) throw new ArgumentNullException(nameof(timedColumnName)); + if (timedColumnIndex < 0 || timedColumnIndex > dataTable.Columns.Count - 1) throw new IndexOutOfRangeException("timedColumnIndex is out of range"); + if (timedFieldConverter == null) throw new ArgumentNullException(nameof(timedFieldConverter)); + if (!dataTable.Columns[timedColumnIndex].ColumnName.Equals(timedColumnName)) throw new ArgumentException("name and index do not match"); +#endif + Id = id; + DataTable = dataTable; + InMem = inMem; + TableName = tableName; + TimedColumnName = timedColumnName; + TimedFieldConverter = timedFieldConverter; + LoadingSimulated = loadingSimulated; + Closed = true; + Reset(sqlConnection); + } + + public int GetId() + { + return Id; + } + + private bool Execute(long endOn) + { + if (Closed) + { + SqlConnection.Open(); + Closed = false; + } + var currentTime = GetNextTime(); + DataTable dataTable = null; + if (!LoadingSimulated) dataTable = DataTable.Clone(); + int count = 0; + while(HasCurrent && TimedFieldConverter.ToSystem(RowEnumerator.Current.Field(TimedColumnName)) <= endOn) + { + if (!LoadingSimulated) dataTable.Rows.Add(RowEnumerator.Current.ItemArray); + count++; + HasCurrent = RowEnumerator.MoveNext(); + } + if (count == 0) return false; + if (!LoadingSimulated) + { + if (InMem) TableMem.FillTable(dataTable, SqlConnection, TableName); + else TableDisk.FillTable(dataTable, SqlConnection, TableName); + CurrentCount += dataTable.Rows.Count; + var message = "Loaded " + dataTable.Rows.Count + " records now counting " + CurrentCount + "/" + DataTable.Rows.Count + " at time " + currentTime; + var padding = ""; + if (Console.CursorTop == ConsoleCursorTop) + { + Console.SetCursorPosition(0, ConsoleCursorTop - 1); + if (message.Length < ConsoleMessageLength) padding = new string(' ', ConsoleMessageLength - message.Length); + } + Console.WriteLine(message + padding); + ConsoleCursorTop = Console.CursorTop; + ConsoleMessageLength = message.Length; + dataTable.Dispose(); + } + return true; + } + + public void Execute(IQueryable queryable, long endOn) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (!HasNextTime()) throw new Exception("there is no NextTime"); + Execute(endOn); + } + + public void Execute(PINQueryable queryable, long endOn) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (!HasNextTime()) throw new Exception("there is no NextTime"); + Execute(endOn); + } + + public void Execute(BKQueryable queryable, long endOn) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (!HasNextTime()) throw new Exception("there is no NextTime"); + Execute(endOn); + } + + public bool HasNextTime() + { + return HasCurrent; + } + + public long GetNextTime() + { + if (!HasNextTime()) throw new Exception("there is no NextTime"); + return TimedFieldConverter.ToSystem(RowEnumerator.Current.Field(TimedColumnName)); + } + + public void Close() + { + if (!Closed) + { + SqlConnection.Close(); + Closed = true; + } + } + + public void Reset(SqlConnection sqlConnection) + { +#if DEBUG + if (sqlConnection == null) throw new ArgumentNullException(nameof(sqlConnection)); +#endif + Close(); + SqlConnection = sqlConnection; + RowEnumerator = DataTable.AsEnumerable().OrderBy(x => x.Field(TimedColumnName)).GetEnumerator(); + HasCurrent = RowEnumerator.MoveNext(); + Closed = true; + ConsoleCursorTop = -1; + ConsoleMessageLength = -1; + CurrentCount = 0; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/Queries/ITimedAnalyticBaseFactory.cs b/source/C#/UniTraX/Datasets/Mobility/Queries/ITimedAnalyticBaseFactory.cs new file mode 100644 index 0000000000000000000000000000000000000000..3dd93501a8c298eaa309b2618702dc3186769bb2 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/Queries/ITimedAnalyticBaseFactory.cs @@ -0,0 +1,10 @@ +namespace Qbb.Datasets.Mobility.Queries +{ + using Qbb.Core.Specification; + using Qbb.Queries.Core; + + public interface ITimedAnalyticBaseFactory + { + AnalyticBaseFactory GenerateQuery(long time, Field field, Field fieldB); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/Queries/LocalInvestigation.cs b/source/C#/UniTraX/Datasets/Mobility/Queries/LocalInvestigation.cs new file mode 100644 index 0000000000000000000000000000000000000000..e9c16e046bfd36a9209edb47007de8a948d3f4b4 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/Queries/LocalInvestigation.cs @@ -0,0 +1,91 @@ +namespace Qbb.Datasets.Mobility.Queries +{ + using Qbb.Datasets.Mobility.DataAccess; + using Qbb.Queries.Aggregation; + using Qbb.Queries.Core; + using System.Collections.Generic; + + public class LocalInvestigation1 : AnalyticBaseFactory + { + public LocalInvestigation1(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long[] epsilons) : + base(id, "LocalInvestigation1", useEvenly, budgetOverRuntime, cleanThreshold, epsilons) { } + + public override IQuery GenerateQueryDirect() + { + var list = new List>(2) + { + new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[13], 1000.0, CleanThreshold), + new Average(Epsilons[2], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[15], 1000.0, CleanThreshold) + }; + var multi = new Multi(list); + IQuery< Ride> forOthers = new Dummy(Epsilons[1] + Epsilons[2], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 5000.0, new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), multi, forOthers); + forOthers = null; + return new Partition2DGeo( + Metadata.Record.Fields[6], -74010000L, 2500L - 1L, 16, + Metadata.Record.Fields[7], 40710000L, 2500L - 1L, 16, + conditional, forOthers, BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + var list = new List>(2) + { + new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[13], 1000.0, CleanThreshold), + new Average(Epsilons[2], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[15], 1000.0, CleanThreshold) + }; + var multi = new Multi(list); + IQuery forOthers = new Dummy(Epsilons[1] + Epsilons[2], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 5000.0, new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), multi, forOthers); + forOthers = null; + var budget = new WhereBudget(Epsilons[0] + Epsilons[1] + Epsilons[2], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], conditional); + return new Partition2DGeo( + Metadata.RecordB.Fields[6], -74010000L, 2500L - 1L, 16, + Metadata.RecordB.Fields[7], 40710000L, 2500L - 1L, 16, + budget, forOthers, BudgetOverRuntime); + } + } + + public class LocalInvestigation2 : AnalyticBaseFactory + { + public LocalInvestigation2(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long[] epsilons) : + base(id, "LocalInvestigation2", useEvenly, budgetOverRuntime, cleanThreshold, epsilons) { } + + public override IQuery GenerateQueryDirect() + { + var median = new Median(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[5], 0.0, 100.0, CleanThreshold); + IQuery forOthers = new Dummy(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 1000.0, new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), median, forOthers); + forOthers = null; + return new Partition2DGeo( + Metadata.Record.Fields[6], -74010000L, 2500L - 1L, 16, + Metadata.Record.Fields[7], 40710000L, 2500L - 1L, 16, + conditional, forOthers, BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + var median = new Median(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), + Metadata.RecordB.Fields[5], 0.0, 100.0, CleanThreshold); + IQuery forOthers = new Dummy(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 1000.0, new Count(Epsilons[0], + Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), median, forOthers); + forOthers = null; + var budget = new WhereBudget(Epsilons[0] + Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], conditional); + return new Partition2DGeo( + Metadata.RecordB.Fields[6], -74010000L, 2500L - 1L, 16, + Metadata.RecordB.Fields[7], 40710000L, 2500L - 1L, 16, + budget, forOthers, BudgetOverRuntime); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/Queries/PerAmount.cs b/source/C#/UniTraX/Datasets/Mobility/Queries/PerAmount.cs new file mode 100644 index 0000000000000000000000000000000000000000..1816e168233376beb05e1f53b74b6ee2184aec10 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/Queries/PerAmount.cs @@ -0,0 +1,42 @@ +namespace Qbb.Datasets.Mobility.Queries +{ + using Qbb.Core.Specification; + using Qbb.Infrastructure.DataConversion; + using Qbb.Queries.Core; + + public class Per50Cents : PerNumbers + { + public Per50Cents(Field field, long start, int number50Cents, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, start, DoubleTypeConverter.TwoDecimal.ToSystem(0.5 - 0.01), number50Cents, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerDollar : PerNumbers + { + public PerDollar(Field field, long start, int numberDollars, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, start, DoubleTypeConverter.TwoDecimal.ToSystem(1.0 - 0.01), numberDollars, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class Per2Dollars50 : PerNumbers + { + public Per2Dollars50(Field field, long start, int number2Dollars50, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, start, DoubleTypeConverter.TwoDecimal.ToSystem(2.5 - 0.01), number2Dollars50, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class Per50Cents0To10Dollar : Per50Cents + { + public Per50Cents0To10Dollar(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 0L, 20, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerDollar0To10 : PerDollar + { + public PerDollar0To10(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 0L, 10, forEach, forOthers, budgetOverRuntime) { } + } + + public class Per2Dollars50From0To50 : Per2Dollars50 + { + public Per2Dollars50From0To50(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 0L, 20, forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/Queries/PerDatetime.cs b/source/C#/UniTraX/Datasets/Mobility/Queries/PerDatetime.cs new file mode 100644 index 0000000000000000000000000000000000000000..d39038a060f3fb9f072128f66861e7097075ba5c --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/Queries/PerDatetime.cs @@ -0,0 +1,29 @@ +namespace Qbb.Datasets.Mobility.Queries +{ + using Qbb.Core.Specification; + using Qbb.Queries.Core; + + public class PerDay : PerNumbers + { + public PerDay(Field field, long startSecond, int numberDays, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, startSecond, 86400L - 1L, numberDays, 1, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerHour : PerNumbers + { + public PerHour(Field field, long startSecond, int numberHours, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, startSecond, 3600L - 1L, numberHours, 0, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerDayJanuary2013 : PerDay + { + public PerDayJanuary2013(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 1356994800L, 31, forEach, forOthers, budgetOverRuntime) { } + } + + public class PerHourJanuary1st2013 : PerHour + { + public PerHourJanuary1st2013(Field field, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + base(field, 1356994800L, 24, forEach, forOthers, budgetOverRuntime) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/Queries/RidesPer.cs b/source/C#/UniTraX/Datasets/Mobility/Queries/RidesPer.cs new file mode 100644 index 0000000000000000000000000000000000000000..ee0c249849ba1654638b93f7af406f2caffc9703 --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/Queries/RidesPer.cs @@ -0,0 +1,163 @@ +namespace Qbb.Datasets.Mobility.Queries +{ + using Qbb.Datasets.Mobility.DataAccess; + using Qbb.Infrastructure.DataConversion; + using Qbb.Queries.Aggregation; + using Qbb.Queries.Core; + + public class RidesPerPickupDateTime : AnalyticBaseFactory + { + public RidesPerPickupDateTime(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "RidesPerPickupDateTime", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new PerDayJanuary2013(Metadata.Record.Fields[1], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerDayJanuary2013(Metadata.RecordB.Fields[1], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } + + public class RidesPerTotalAmount : AnalyticBaseFactory + { + public RidesPerTotalAmount(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "RidesPerTotalAmount", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new Per2Dollars50From0To50(Metadata.Record.Fields[15], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new Per2Dollars50From0To50(Metadata.RecordB.Fields[15], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } + + public class RidesPerTipAmount : AnalyticBaseFactory + { + public RidesPerTipAmount(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "RidesPerTipAmount", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new Per50Cents0To10Dollar(Metadata.Record.Fields[13], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new Per50Cents0To10Dollar(Metadata.RecordB.Fields[13], + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } + + public class RidesPerPassenegerCount : AnalyticBaseFactory + { + public RidesPerPassenegerCount(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "RidesPerPassenegerCount", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new PerNumbers(Metadata.Record.Fields[3], 0L, DoubleTypeConverter.WholeNumber.ToSystem(1.0 - 1.0), 11, 0, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerNumbers(Metadata.RecordB.Fields[3], 0L, DoubleTypeConverter.WholeNumber.ToSystem(1.0 - 1.0), 11, 0, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } + + public class RidesPerTripTimeInSecs : AnalyticBaseFactory + { + public RidesPerTripTimeInSecs(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "RidesPerTripTimeInSecs", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new PerNumbers(Metadata.Record.Fields[4], 0L, DoubleTypeConverter.WholeNumber.ToSystem(120.0 - 1.0), 15, 0, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerNumbers(Metadata.RecordB.Fields[4], 0L, DoubleTypeConverter.WholeNumber.ToSystem(120.0 - 1.0), 15, 0, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } + + public class RidesPerTripDistance : AnalyticBaseFactory + { + public RidesPerTripDistance(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long epsilon) : + base(id, "RidesPerTripDistance", useEvenly, budgetOverRuntime, cleanThreshold, new long[] { epsilon }) { } + + public override IQuery GenerateQueryDirect() + { + return new PerNumbers(Metadata.Record.Fields[5], 0L, DoubleTypeConverter.TwoDecimal.ToSystem(0.5 - 0.01), 20, 0, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + return new WhereBudget(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new PerNumbers(Metadata.RecordB.Fields[5], 0L, DoubleTypeConverter.TwoDecimal.ToSystem(0.5 - 0.01), 20, 0, + new Count(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), + new Dummy(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), UseEvenly, CleanThreshold), BudgetOverRuntime)); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Datasets/Mobility/Queries/StreamInvestigation.cs b/source/C#/UniTraX/Datasets/Mobility/Queries/StreamInvestigation.cs new file mode 100644 index 0000000000000000000000000000000000000000..855fdfa4a1cc2484e17e3103535a41bb171aaa2c --- /dev/null +++ b/source/C#/UniTraX/Datasets/Mobility/Queries/StreamInvestigation.cs @@ -0,0 +1,157 @@ +namespace Qbb.Datasets.Mobility.Queries +{ + using Qbb.Core.Specification; + using Qbb.Datasets.Mobility.DataAccess; + using Qbb.Queries.Aggregation; + using Qbb.Queries.Core; + using System; + using System.Collections.Generic; + + + public class StreamInvestigationHourly : AnalyticBaseFactory, ITimedAnalyticBaseFactory + { + private Field Field; + private Field FieldB; + private long Time = long.MinValue; + + public StreamInvestigationHourly(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long[] epsilons) : + base(id, "StreamInvestigationHourly", useEvenly, budgetOverRuntime, cleanThreshold, epsilons) { } + + public override IQuery GenerateQueryDirect() + { + var analysisList = new List>(2) + { + new Average(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[4], 10800.0, CleanThreshold), + new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.Record.Fields[5], 100.0, CleanThreshold) + }; + var analysis = new Multi(analysisList); + + var numHours = 6L; + while (Time - numHours * 3600L + 1L < Field.FieldType.Bounds.Low) numHours--; + IQuery conditional = null; + for (long i = 0; i < numHours; i++) + { + conditional = new Conditional(x => x[0] > 24.5, + new WhereRange(Field, Time - (numHours - i) * 3600L + 1L, Time, + new Count(Epsilons[2 + i], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold)), + new WhereRange(Field, Time - (numHours - i) * 3600L + 1L, Time, analysis), + conditional); + } + IQuery query = new WhereRange(Metadata.Record.Fields[6], -74010000L, -73969999L, conditional); + query = new WhereRange(Metadata.Record.Fields[7], 40710000L, 40749999L, query); + query = new WhereRange(Metadata.Record.Fields[8], -73987500L, -73982499L, query); + return new WhereRange(Metadata.Record.Fields[9], 40757500L, 40759999L, query); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + var analysisList = new List>(2) + { + new Average(Epsilons[0], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[4], 10800.0, CleanThreshold), + new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), Metadata.RecordB.Fields[5], 100.0, CleanThreshold) + }; + var analysis = new Multi(analysisList); + + var numHours = 6L; + while (Time - numHours * 3600L + 1L < FieldB.FieldType.Bounds.Low) numHours--; + IQuery conditional = null; + for (long i = 0; i < numHours; i++) + { + conditional = new Conditional(x => x[0] > 24.5, + new WhereRange(FieldB, Time - (numHours - i) * 3600L + 1L, Time, + new WhereBudget(Epsilons[0] + Epsilons[1] + Epsilons[2 + i], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], + new Count(Epsilons[2 + i], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold))), + new WhereRange(FieldB, Time - (numHours - i) * 3600L + 1L, Time, + new WhereBudget(Epsilons[0] + Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], analysis)), + conditional); + } + IQuery query = new WhereRange(Metadata.RecordB.Fields[6], -74010000L, -73969999L, conditional); + query = new WhereRange(Metadata.RecordB.Fields[7], 40710000L, 40749999L, query); + query = new WhereRange(Metadata.RecordB.Fields[8], -73987500L, -73982499L, query); + return new WhereRange(Metadata.RecordB.Fields[9], 40757500L, 40759999L, query); + } + + public AnalyticBaseFactory GenerateQuery(long time, Field field, Field fieldB) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (fieldB == null) throw new ArgumentNullException(nameof(fieldB)); +#endif + Time = time; + Field = field; + FieldB = fieldB; + + return this; + } + } + + public class StreamInvestigationMonthly : AnalyticBaseFactory, ITimedAnalyticBaseFactory + { + private Field Field; + private Field FieldB; + private long Time = long.MinValue; + + public StreamInvestigationMonthly(long id, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long[] epsilons) : + base(id, "LocalInvestigationStreamMonthly", useEvenly, budgetOverRuntime, cleanThreshold, epsilons) { } + + public override IQuery GenerateQueryDirect() + { + var aggregate = new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), + Metadata.Record.Fields[4], 10800.0, CleanThreshold); + IQuery forOthers = new Dummy(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), + UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 24.5, new Count(Epsilons[0], + Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), aggregate, forOthers); + var time = new WhereRange(Field, Time - 2678400L + 1L, Time, conditional); + IQuery query = new WhereRange(Metadata.Record.Fields[6], -73815000L, -73780000L - 1L, time); + query = new WhereRange(Metadata.Record.Fields[7], 40640000L, 40650000L - 1L, query); + forOthers = null; + return new Partition2DGeo( + Metadata.Record.Fields[8], -74010000L, 2500L - 1L, 16, + Metadata.Record.Fields[9], 40710000L, 2500L - 1L, 16, + query, forOthers, BudgetOverRuntime); + } + + public override IQuery GenerateQueryPINQ() + { + return GenerateQueryDirect(); + } + + public override IQuery GenerateQueryBK() + { + var aggregate = new Average(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), + Metadata.RecordB.Fields[4], 10800.0, CleanThreshold); + IQuery forOthers = new Dummy(Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), + UseEvenly, CleanThreshold); + var conditional = new Conditional(x => x[0] > 24.5, new Count(Epsilons[0], + Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()].FieldType.Convert, new PINQDummy(), CleanThreshold), aggregate, forOthers); + var budget = new WhereBudget(Epsilons[0] + Epsilons[1], Metadata.RecordB.Fields[Metadata.RecordB.GetBudgetIndex()], conditional); + var time = new WhereRange(FieldB, Time - 2678400L + 1L, Time, budget); + IQuery query = new WhereRange(Metadata.RecordB.Fields[6], -73815000L, -73780000L - 1L, time); + query = new WhereRange(Metadata.RecordB.Fields[7], 40640000L, 40650000L - 1L, query); + forOthers = null; + return new Partition2DGeo( + Metadata.RecordB.Fields[8], -74010000L, 2500L - 1L, 16, + Metadata.RecordB.Fields[9], 40710000L, 2500L - 1L, 16, + query, forOthers, BudgetOverRuntime); + } + + public AnalyticBaseFactory GenerateQuery(long time, Field field, Field fieldB) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (fieldB == null) throw new ArgumentNullException(nameof(fieldB)); +#endif + Time = time; + Field = field; + FieldB = fieldB; + + return this; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/BudgetProvider/ConstantBudgetProvider.cs b/source/C#/UniTraX/Infrastructure/BudgetProvider/ConstantBudgetProvider.cs new file mode 100644 index 0000000000000000000000000000000000000000..d7aecf751ad87d823365e8a812bad914261407d7 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/BudgetProvider/ConstantBudgetProvider.cs @@ -0,0 +1,17 @@ +namespace Qbb.Infrastructure.BudgetProvider +{ + public class ConstantBudgetProvider :IBudgetProvider + { + private readonly double Budget; + + public ConstantBudgetProvider(double budget) + { + Budget = budget; + } + + public double NextBudget() + { + return Budget; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/BudgetProvider/IBudgetProvider.cs b/source/C#/UniTraX/Infrastructure/BudgetProvider/IBudgetProvider.cs new file mode 100644 index 0000000000000000000000000000000000000000..8c794cc36d84487654f21739ffb1b22f1bda9515 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/BudgetProvider/IBudgetProvider.cs @@ -0,0 +1,7 @@ +namespace Qbb.Infrastructure.BudgetProvider +{ + public interface IBudgetProvider + { + double NextBudget(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/DataConversion/DoubleType.cs b/source/C#/UniTraX/Infrastructure/DataConversion/DoubleType.cs new file mode 100644 index 0000000000000000000000000000000000000000..97170ff985b67946446b3f4f2df2ebc1e06993fa --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/DataConversion/DoubleType.cs @@ -0,0 +1,63 @@ +namespace Qbb.Infrastructure.DataConversion +{ + using System; + + public class DoubleType : IType + { + public static DoubleType WholeNumber = new DoubleType(0.0, 100000000000.0, 1.0); + public static DoubleType Integer = new DoubleType(-100000000000.0, 100000000000.0, 1.0); + public static DoubleType TwoDecimal = new DoubleType(-100000000000.0, 100000000000.0, 0.01); + public static DoubleType LongLat = new DoubleType(-5000.0, 5000.0, 0.000001); + + public static bool IsValid(double value, double origin, double step, double tolerance) + { +#if DEBUG + if (step <= 0.0) throw new ArgumentException("step must be greater zero"); + if (tolerance < 0.0) throw new ArgumentException("tolerance cannot be negative"); +#endif + var difference = Math.Abs(value - origin); + var remainder = difference % step; + if (0 - tolerance < remainder && remainder < tolerance) return true; + remainder = step - remainder; + if (0 - tolerance < remainder && remainder < tolerance) return true; + return false; + } + + public double Min { get; private set; } + public double Center { get; private set; } + public double Max { get; private set; } + public double Step { get; private set; } + public double Tolerance { get; private set; } + + public DoubleType(double min, double max, double step) + { + var tolerance = step / 1000.0; +#if DEBUG + if (step <= 0.0) throw new ArgumentException("step must be above zero"); + if (min - max > tolerance) throw new ArgumentException("min is greater than max"); + if (!IsValid(max, min, step, tolerance)) throw new ArgumentException("center 0.0 is not a multiple of step from min away"); + if (!IsValid(0.0, min, step, tolerance)) throw new ArgumentException("center 0.0 is not a multiple of step from min away"); +#endif + Min = min; + Center = 0.0; + Max = max; + Step = step; + Tolerance = tolerance; + } + + public bool IsValid(double value) + { + if (value + Tolerance < Min) return false; + if (value - Tolerance > Max) return false; + if (IsValid(value, Center, Step, Tolerance)) return true; + if (IsValid(value, Min, Step, Tolerance)) return true; + if (IsValid(value, Max, Step, Tolerance)) return true; + return false; + } + + public override string ToString() + { + return "[DT:" + Min + ";" + Center + ";" + Max + ";" + Step + ";" + Tolerance + "]"; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/DataConversion/DoubleTypeConverter.cs b/source/C#/UniTraX/Infrastructure/DataConversion/DoubleTypeConverter.cs new file mode 100644 index 0000000000000000000000000000000000000000..069cba4b0a66c5e2269856691c5fb2c1069b4a8b --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/DataConversion/DoubleTypeConverter.cs @@ -0,0 +1,88 @@ +namespace Qbb.Infrastructure.DataConversion +{ + using Qbb.Core.Partition; + using Qbb.Core.Specification; + using System; + + public class DoubleTypeConverter : ITypeConverter, IFieldType + { + public static DoubleTypeConverter WholeNumber = new DoubleTypeConverter(DoubleType.WholeNumber); + public static DoubleTypeConverter Integer = new DoubleTypeConverter(DoubleType.Integer); + public static DoubleTypeConverter TwoDecimal = new DoubleTypeConverter(DoubleType.TwoDecimal); + public static DoubleTypeConverter LongLat = new DoubleTypeConverter(DoubleType.LongLat); + + private static long ToSystem(double value, double center, double step) + { + var system = (value - center) / step; + if (system >= 0.0) system += 0.0000001; + else system -= 0.0000001; + return (long)system; + } + + private static double ToType(long value, double center, double step, bool larger) + { + var tolerance = 0.0000000000001; +#if DEBUG + if (step / 1000.0 <= tolerance) throw new Exception("step is too small"); +#endif + var result = center + value * step; + if (larger) return result + tolerance; + return result - tolerance; + } + + public IType Type { get; private set; } + public Interval System { get; private set; } + + public DoubleTypeConverter(IType type) + { +#if DEBUG + if (type == null) throw new ArgumentNullException(nameof(type)); +#endif + Type = type; + System = new Interval(ToSystem(Type.Min, Type.Center, Type.Step), ToSystem(Type.Max, Type.Center, Type.Step)); + } + + public long ToSystem(double value) + { +#if DEBUG + if (!Type.IsValid(value)) throw new ArgumentException("value not valid in type"); +#endif + var system = ToSystemUnchecked(value); +#if DEBUG + if (!System.Encloses(system)) throw new Exception("should not happen"); +#endif + return system; + } + + public double ToType(long value, bool larger) + { +#if DEBUG + if (!System.Encloses(value)) throw new ArgumentException("value not valid in system"); +#endif + var type = ToTypeUnchecked(value, larger); +#if DEBUG + if (!Type.IsValid(type)) throw new Exception("should not happen"); +#endif + return type; + } + + public long ToSystemUnchecked(double value) + { + return ToSystem(value, Type.Center, Type.Step); + } + + public double ToTypeUnchecked(long value, bool larger) + { + return ToType(value, Type.Center, Type.Step, larger); + } + + public override string ToString() + { + return "[DTC:" + Type.ToString() + ";" + System.ToString() + "]"; + } + + public Interval Bounds => System; + public Func Convert => ToType; + public Func ConvertUnchecked => ToTypeUnchecked; + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/DataConversion/IType.cs b/source/C#/UniTraX/Infrastructure/DataConversion/IType.cs new file mode 100644 index 0000000000000000000000000000000000000000..3dfa39783a77890ae4eee5298204dbe655b164cb --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/DataConversion/IType.cs @@ -0,0 +1,15 @@ +namespace Qbb.Infrastructure.DataConversion +{ + using System; + + public interface IType where T : IComparable + { + T Min { get; } + T Center { get; } + T Max { get; } + T Step { get; } + T Tolerance { get; } + bool IsValid(T value); + string ToString(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/DataConversion/ITypeConverter.cs b/source/C#/UniTraX/Infrastructure/DataConversion/ITypeConverter.cs new file mode 100644 index 0000000000000000000000000000000000000000..4375af27d6bdd6d0b9c2aaa602816911ee44ccaf --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/DataConversion/ITypeConverter.cs @@ -0,0 +1,14 @@ +namespace Qbb.Infrastructure.DataConversion +{ + using Qbb.Core.Partition; + using System; + + public interface ITypeConverter where T : IComparable + { + IType Type { get; } + Interval System { get; } + long ToSystem(T value); + T ToType(long value, bool larger); + string ToString(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/DataLoading/DataTableProvider.cs b/source/C#/UniTraX/Infrastructure/DataLoading/DataTableProvider.cs new file mode 100644 index 0000000000000000000000000000000000000000..39a95003cb5835f68d709a19985d1b128be2d398 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/DataLoading/DataTableProvider.cs @@ -0,0 +1,120 @@ +namespace Qbb.Infrastructure.DataLoading +{ + using Qbb.Infrastructure.BudgetProvider; + using System; + using System.Data; + using System.IO; + + public class DataTableProvider + { + private readonly string DataFilename; + private IBudgetProvider BudgetProvider; + private readonly string[] ColumnNames; + private readonly string BudgetColumnName; + private readonly string[] StringColumnNames; + + public DataTableProvider(string dataFilename, IBudgetProvider budgetProvider, string[] columnNames, string budgetColumnName, string[] stringColumnNames) + { +#if DEBUG + if (dataFilename == null) throw new ArgumentNullException(nameof(dataFilename)); + if (columnNames == null) throw new ArgumentNullException(nameof(columnNames)); + if (columnNames.Length == 0) throw new ArgumentException("column names is empty"); + if (budgetColumnName == null) throw new ArgumentNullException(nameof(budgetColumnName)); +#endif + DataFilename = dataFilename; + BudgetProvider = budgetProvider; + ColumnNames = columnNames; + BudgetColumnName = budgetColumnName; + StringColumnNames = stringColumnNames; + } + + public DataTable GetDataTable() + { + Console.WriteLine("Loading data into memory."); + var dataTable = new DataTable(); + int rowCount = 0; + var topPos = 0; + Console.Write("Loaded rows: " + rowCount); + foreach (var c in ColumnNames) + { + DataColumn column; + if (c.Equals(BudgetColumnName) && BudgetProvider == null) continue; + else if (StringColumnNames != null && Array.IndexOf(StringColumnNames, c) > -1) + { + column = new DataColumn + { + DataType = System.Type.GetType("System.String"), + ColumnName = c + }; + } + else + { + column = new DataColumn + { + DataType = System.Type.GetType("System.Double"), + ColumnName = c + }; + } + dataTable.Columns.Add(column); + } + using (var reader = new StreamReader(DataFilename)) + { + while (!reader.EndOfStream) + { + var line = reader.ReadLine(); + var values = line.Split(';'); + try + { + var row = dataTable.NewRow(); + for (var i = 0; i < ColumnNames.Length; i++) + { + var c = ColumnNames[i]; + var v = values[i]; + v = v.Replace("\"", ""); + { + + } + if (c.Equals(BudgetColumnName)) + { + if (BudgetProvider == null) continue; + else row[c] = BudgetProvider.NextBudget(); + } + else if (StringColumnNames != null && Array.IndexOf(StringColumnNames, c) > -1) + { + row[c] = v; + } + else + { + row[c] = Double.Parse(v); + } + } + dataTable.Rows.Add(row); + } + catch (FormatException ex) + { + topPos = Console.CursorTop; + Console.SetCursorPosition(13, topPos); + Console.Write(rowCount); + Console.WriteLine(); + Console.WriteLine("DataTableProvider caught FormatException: " + ex.Message + " In line: " + line); + Console.Write("Loaded rows: " + rowCount); + continue; + } + rowCount++; + if (rowCount % 10000 == 0) + { + topPos = Console.CursorTop; + Console.SetCursorPosition(13, topPos); + Console.Write(rowCount); + } + } + topPos = Console.CursorTop; + Console.SetCursorPosition(13, topPos); + Console.Write(rowCount); + } + Console.WriteLine(); + Console.WriteLine("Loaded data into memory."); + return dataTable; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/DataLoading/TableDisk.cs b/source/C#/UniTraX/Infrastructure/DataLoading/TableDisk.cs new file mode 100644 index 0000000000000000000000000000000000000000..3498f7d02a5c0053decf5213e3e63e305cd819c8 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/DataLoading/TableDisk.cs @@ -0,0 +1,99 @@ +namespace Qbb.Infrastructure.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using System; + using System.Collections.Generic; + using System.Data; + using System.Data.SqlClient; + + public class TableDisk + { + public static Index CreateDefaultIndex(Table table, string name) + { +#if DEBUG + if (table == null) throw new ArgumentNullException(nameof(table)); + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + var index = new Index(table, name) + { + IndexType = IndexType.NonClusteredIndex, + IsUnique = false, + PadIndex = true, + NoAutomaticRecomputation = true, + IgnoreDuplicateKeys = false, + DisallowRowLocks = true, + DisallowPageLocks = true, + FillFactor = Convert.ToByte(100), + IndexKeyType = IndexKeyType.None + }; + return index; + } + + public static Table CreateTable(Database database, bool withBudget, string databaseName, string tableName, string[] columnNames, string primaryKeyName, string budgetColumnName, Dictionary specialColumns) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); + if (tableName == null) throw new ArgumentNullException(nameof(tableName)); + if (columnNames == null) throw new ArgumentNullException(nameof(columnNames)); + if (columnNames.Length == 0) throw new ArgumentException("column names is empty"); + if (primaryKeyName == null) throw new ArgumentNullException(nameof(primaryKeyName)); + if (budgetColumnName == null) throw new ArgumentNullException(nameof(budgetColumnName)); +#endif + Console.WriteLine(databaseName + ": Creating table " + tableName + " " + (withBudget ? "with" : "without") + " budget."); + var table = new Table(database, tableName); + foreach (var c in columnNames) + { + if (c.Equals(budgetColumnName) && !withBudget) continue; + else if (specialColumns != null && specialColumns.ContainsKey(c)) + { + table.Columns.Add(new Column(table, c, specialColumns[c]) + { + Nullable = false + }); + } + else + { + table.Columns.Add(new Column(table, c, DataType.Float) + { + Nullable = false + }); + } + } + var indexPk = new Index(table, "PK_" + databaseName + "_" + tableName) + { + IndexType = IndexType.ClusteredIndex, + IsUnique = true, + PadIndex = true, + NoAutomaticRecomputation = true, + IgnoreDuplicateKeys = false, + DisallowRowLocks = true, + DisallowPageLocks = true, + FillFactor = Convert.ToByte(100), + IndexKeyType = IndexKeyType.DriPrimaryKey + }; + indexPk.IndexedColumns.Add(new IndexedColumn(indexPk, primaryKeyName, false)); + table.Indexes.Add(indexPk); + table.Create(); + indexPk.PhysicalPartitions[0].DataCompression = DataCompressionType.Page; + indexPk.Alter(); + Console.WriteLine(databaseName + ": Created table " + tableName); + return table; + } + + public static void FillTable(DataTable dataTable, SqlConnection sqlConnection, string tableName) + { +#if DEBUG + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); + if (sqlConnection == null) throw new ArgumentNullException(nameof(sqlConnection)); + if (tableName == null) throw new ArgumentNullException(nameof(tableName)); +#endif + SqlBulkCopy sqlbc = new SqlBulkCopy(sqlConnection, SqlBulkCopyOptions.TableLock | SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.CheckConstraints, null) + { + DestinationTableName = "[" + tableName + "]", + BulkCopyTimeout = 600 + }; + sqlbc.WriteToServer(dataTable); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/DataLoading/TableMem.cs b/source/C#/UniTraX/Infrastructure/DataLoading/TableMem.cs new file mode 100644 index 0000000000000000000000000000000000000000..37f295d5f3a5c9d33a2ea93dd92dc67700a445de --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/DataLoading/TableMem.cs @@ -0,0 +1,95 @@ +namespace Qbb.Infrastructure.DataLoading +{ + using Microsoft.SqlServer.Management.Smo; + using System; + using System.Collections.Generic; + using System.Data; + using System.Data.SqlClient; + + public class TableMem + { + public static Index CreateDefaultIndex(Table table, string name) + { +#if DEBUG + if (table == null) throw new ArgumentNullException(nameof(table)); + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + var index = new Index(table, name) + { + IndexType = IndexType.NonClusteredIndex, + IndexKeyType = IndexKeyType.None + }; + return index; + } + + public static Table CreateTableInit(Database database, bool withBudget, string databaseName, string tableName, string[] columnNames, string primaryKeyName, string budgetColumnName, Dictionary specialColumns) + { +#if DEBUG + if (database == null) throw new ArgumentNullException(nameof(database)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); + if (tableName == null) throw new ArgumentNullException(nameof(tableName)); + if (columnNames == null) throw new ArgumentNullException(nameof(columnNames)); + if (columnNames.Length == 0) throw new ArgumentException("column names is empty"); + if (primaryKeyName == null) throw new ArgumentNullException(nameof(primaryKeyName)); + if (budgetColumnName == null) throw new ArgumentNullException(nameof(budgetColumnName)); +#endif + Console.WriteLine(databaseName + ": Creating table " + tableName + " " + (withBudget ? "with" : "without") + " budget."); + var table = new Table(database, tableName) + { + IsMemoryOptimized = true, + Durability = DurabilityType.SchemaOnly + }; + foreach (var c in columnNames) + { + if (c.Equals(budgetColumnName) && !withBudget) continue; + else if (specialColumns != null && specialColumns.ContainsKey(c)) + { + table.Columns.Add(new Column(table, c, specialColumns[c]) + { + Nullable = false + }); + } + else + { + table.Columns.Add(new Column(table, c, DataType.Float) + { + Nullable = false + }); + } + } + var indexPk = new Index(table, "PK_" + databaseName + "_" + tableName) + { + IndexType = IndexType.NonClusteredIndex, + IndexKeyType = IndexKeyType.DriPrimaryKey + }; + indexPk.IndexedColumns.Add(new IndexedColumn(indexPk, primaryKeyName, false)); + table.Indexes.Add(indexPk); + return table; + } + + public static void CreateTableFinish(Table table, string tableName) + { +#if DEBUG + if (table == null) throw new ArgumentNullException(nameof(table)); + if (tableName == null) throw new ArgumentNullException(nameof(tableName)); +#endif + table.Create(); + Console.WriteLine("TableMem: Created table " + tableName); + } + + public static void FillTable(DataTable dataTable, SqlConnection sqlConnection, string tableName) + { +#if DEBUG + if (dataTable == null) throw new ArgumentNullException(nameof(dataTable)); + if (sqlConnection == null) throw new ArgumentNullException(nameof(sqlConnection)); + if (tableName == null) throw new ArgumentNullException(nameof(tableName)); +#endif + SqlBulkCopy sqlbc = new SqlBulkCopy(sqlConnection, SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.CheckConstraints, null) + { + DestinationTableName = "[" + tableName + "]", + BulkCopyTimeout = 600 + }; + sqlbc.WriteToServer(dataTable); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Experiments/BKQueryableFactory.cs b/source/C#/UniTraX/Infrastructure/Experiments/BKQueryableFactory.cs new file mode 100644 index 0000000000000000000000000000000000000000..2f6329ff535ace864bfe746474022ebe16f6873f --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Experiments/BKQueryableFactory.cs @@ -0,0 +1,29 @@ +namespace Qbb.Infrastructure.ExperimentUtils +{ + using Qbb.Core.Bookkeeper; + using Qbb.Core.BoundedData; + using Qbb.Core.History; + using Qbb.Core.Partition; + using System; + using System.Collections.Generic; + + public class BKQueryableFactory + { + private readonly BoundedDataAccess BoundedDataAccess; + + public BKQueryableFactory(BoundedDataAccess boundedDataAccess) + { +#if DEBUG + if (boundedDataAccess == null) throw new ArgumentNullException(nameof(boundedDataAccess)); +#endif + BoundedDataAccess = boundedDataAccess; + } + + public BKQueryable GenerateBKQueryable() + { + var boundary = new Partition(new HashSet() { BoundedDataAccess.Record.Boundary }); + return new BKQueryable(BoundedDataAccess.GetQueryable(), new Bookkeeper(boundary, BoundedDataAccess, + new InMemoryHistory(BoundedDataAccess.Record.GetBudgetIndex())), boundary); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Experiments/ExperimentUtils.cs b/source/C#/UniTraX/Infrastructure/Experiments/ExperimentUtils.cs new file mode 100644 index 0000000000000000000000000000000000000000..3b66f2e21caba25fd4396d0c2529d5adae9f94e3 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Experiments/ExperimentUtils.cs @@ -0,0 +1,196 @@ +namespace Qbb.Infrastructure.ExperimentUtils +{ + using Qbb.Core.Utils; + using Qbb.Infrastructure.Provisioning; + using Qbb.Queries.Core; + using Qbb.Queries.Recording; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + + public class ExperimentUtils + { + public static void SetupExperimentDirect(String instanceDirName, String experimentName) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); +#endif + Directory.CreateDirectory(Path.Combine(instanceDirName, experimentName)); + } + + public static double[][] RunExperimentDirect(String instanceDirName, String experimentName, IExperimentServer server, IQueryable queryable, List> queries, int run, string databaseName) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); +#endif + var experimentDirName = Path.Combine(instanceDirName, experimentName); + var results = new double[queries.Count()][]; + System.Threading.Thread.Sleep(1000); + Recorder.Reset(); + Recorder.StartGlobalTimer(); +#if RECORD + var timer = Recorder.StartTimer(); +#endif + for (int i = 0; i < queries.Count(); i++) results[i] = queries[i].ApplyTo(queryable); +#if RECORD + var time = Recorder.StopTimer(timer); + Console.WriteLine("Took time ms: " + time); +#endif + Recorder.StopGlobalTimer(); + Recorder.ToFile(Path.Combine(experimentDirName, "run" + run + "_results.csv")); + FileUtils.WriteLineToFile("0," + queryable.Count(), Path.Combine(experimentDirName, "run" + run + "_budgetUsed.csv")); + return results; + } + + public static double[][][] RepeatExperimentDirect(String instanceDirName, String experimentName, IExperimentServer server, IQueryable queryable, List> queries, string databaseName, int repeats) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); + if (repeats < 1) throw new ArgumentException("repeats is invalid"); +#endif + var results = new double[repeats][][]; + for (int i = 0; i < repeats; i++) + { + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": STARTED"); + results[i] = RunExperimentDirect(instanceDirName, experimentName, server, queryable, queries, i, databaseName); + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": FINISHED"); + } + return results; + } + + public static void SetupExperimentPINQ(String instanceDirName, String experimentName) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); +#endif + Directory.CreateDirectory(Path.Combine(instanceDirName, experimentName)); + } + + public static double[][] RunExperimentPINQ(String instanceDirName, String experimentName, IExperimentServer server, PINQueryableFactory queryableFactoryPINQ, List> queries, double defaultTotalBudget, int run, string databaseName) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (queryableFactoryPINQ == null) throw new ArgumentNullException(nameof(queryableFactoryPINQ)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); +#endif + var experimentDirName = Path.Combine(instanceDirName, experimentName); + var queryables = queryableFactoryPINQ.GeneratePINQueryable(); + var queryable = queryables.Item1; + var queryablePINQ = queryables.Item2; + var results = new double[queries.Count()][]; + System.Threading.Thread.Sleep(1000); + Recorder.Reset(); + Recorder.StartGlobalTimer(); +#if RECORD + var timer = Recorder.StartTimer(); +#endif + for (int i = 0; i < queries.Count(); i++) results[i] = queries[i].ApplyTo(queryablePINQ); +#if RECORD + var time = Recorder.StopTimer(timer); + Console.WriteLine("Took time ms: " + time); +#endif + Recorder.StopGlobalTimer(); + Recorder.ToFile(Path.Combine(experimentDirName, "run" + run + "_results.csv")); + FileUtils.WriteLineToFile((defaultTotalBudget - queryablePINQ.remaining()) + "," + queryable.Count(), Path.Combine(experimentDirName, "run" + run + "_budgetUsed.csv")); + return results; + } + + public static double[][][] RepeatExperimentPINQ(String instanceDirName, String experimentName, IExperimentServer server, PINQueryableFactory queryableFactoryPINQ, List> queries, double defaultTotalBudget, string databaseName, int repeats) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (queryableFactoryPINQ == null) throw new ArgumentNullException(nameof(queryableFactoryPINQ)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); + if (repeats < 1) throw new ArgumentException("repeats is invalid"); +#endif + var results = new double[repeats][][]; + for (int i = 0; i < repeats; i++) + { + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": STARTED"); + results[i] = RunExperimentPINQ(instanceDirName, experimentName, server, queryableFactoryPINQ, queries, defaultTotalBudget, i, databaseName); + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": FINISHED"); + } + return results; + } + + public static void SetupExperimentBK(String instanceDirName, String experimentName) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); +#endif + Directory.CreateDirectory(Path.Combine(instanceDirName, experimentName)); + } + + public static double[][] RunExperimentBK(String instanceDirName, String experimentName, IExperimentServer server, BKQueryableFactory queryableFactory, List> queries, int run, string databaseName) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (queryableFactory == null) throw new ArgumentNullException(nameof(queryableFactory)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); +#endif + var experimentDirName = Path.Combine(instanceDirName, experimentName); + var queryable = queryableFactory.GenerateBKQueryable(); + var results = new double[queries.Count()][]; + System.Threading.Thread.Sleep(1000); + Recorder.Reset(); + Recorder.StartGlobalTimer(); +#if RECORD + var timer = Recorder.StartTimer(); +#endif + for (int i = 0; i < queries.Count(); i++) results[i] = queries[i].ApplyTo(queryable); +#if RECORD + var time = Recorder.StopTimer(timer); + Console.WriteLine("Took time ms: " + time); +#endif + Recorder.StopGlobalTimer(); + Recorder.ToFile(Path.Combine(experimentDirName, "run" + run + "_results.csv")); + queryable.CleanHistory(); + queryable.WriteDetailedBudgetUseToFile(Path.Combine(experimentDirName, "run" + run + "_budgetUsed.csv")); + return results; + } + + public static double[][][] RepeatExperimentBK(String instanceDirName, String experimentName, IExperimentServer server, BKQueryableFactory queryableFactory, List> queries, string databaseName, int repeats) + { +#if DEBUG + if (instanceDirName == null) throw new ArgumentNullException(nameof(instanceDirName)); + if (experimentName == null) throw new ArgumentNullException(nameof(experimentName)); + if (server == null) throw new ArgumentNullException(nameof(server)); + if (queryableFactory == null) throw new ArgumentNullException(nameof(queryableFactory)); + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); + if (repeats < 1) throw new ArgumentException("repeats is invalid"); +#endif + var results = new double[repeats][][]; + for (int i = 0; i < repeats; i++) + { + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": STARTED"); + results[i] = RunExperimentBK(instanceDirName, experimentName, server, queryableFactory, queries, i, databaseName); + Console.WriteLine("SubExperiment " + experimentName + " Run " + i + ": FINISHED"); + } + return results; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Experiments/PINQueryableFactory.cs b/source/C#/UniTraX/Infrastructure/Experiments/PINQueryableFactory.cs new file mode 100644 index 0000000000000000000000000000000000000000..ec601021e442047cdb1b554f939684817350e39a --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Experiments/PINQueryableFactory.cs @@ -0,0 +1,26 @@ +namespace Qbb.Infrastructure.ExperimentUtils +{ + using PINQ; + using System; + using System.Linq; + + public class PINQueryableFactory + { + private readonly IQueryable Queryable; + private readonly double TotalBudget; + + public PINQueryableFactory(IQueryable queryable, double totalBudget) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Queryable = queryable; + TotalBudget = totalBudget; + } + + public Tuple, PINQueryable> GeneratePINQueryable() + { + return new Tuple, PINQueryable>(Queryable, new PINQueryable(Queryable, new PINQAgentBudget(TotalBudget))); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/DatabaseServer.cs b/source/C#/UniTraX/Infrastructure/Provisioning/DatabaseServer.cs new file mode 100644 index 0000000000000000000000000000000000000000..6d97a8c02dc2cec67dc1476e371ea70470cb2f54 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/DatabaseServer.cs @@ -0,0 +1,293 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using Microsoft.SqlServer.Management.Common; + using Microsoft.SqlServer.Management.Smo; + using System; + + public class DatabaseServer + { + private ServerConnection Connection; + private Server Server; + + public DatabaseServer(string host, string username, string password) + { +#if DEBUG + if (host == null) throw new ArgumentNullException(nameof(host)); + if (username == null) throw new ArgumentNullException(nameof(username)); + if (password == null) throw new ArgumentNullException(nameof(password)); +#endif + Connection = new ServerConnection(host); + Connection.LoginSecure = false; + Connection.Login = username; + Connection.Password = password; + Server = new Server(Connection); + } + + public void Drop(string databaseName) + { +#if DEBUG + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); +#endif + Console.WriteLine("Dropping Database " + databaseName); + try + { + Server.KillAllProcesses(databaseName); + Server.KillDatabase(databaseName); + Console.WriteLine("Dropped Database " + databaseName); + } + catch (SmoException smoex) + { + HandleSMOException(smoex, true); + } + catch (Exception ex) + { + HandleException(ex, true); + } + } + + public void CreateOnDisk(string databaseName, string sysdataFileName, string logFileName) + { +#if DEBUG + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); + if (sysdataFileName == null) throw new ArgumentNullException(nameof(sysdataFileName)); + if (logFileName == null) throw new ArgumentNullException(nameof(logFileName)); +#endif + Console.WriteLine("Creating On-Disk Database " + databaseName); + try + { + var database = new Database(Server, databaseName) + { + ContainmentType = ContainmentType.None, + Collation = "Latin1_General_100_CS_AS_KS_WS_SC", + CompatibilityLevel = CompatibilityLevel.Version140, + AnsiNullDefault = false, + AnsiWarningsEnabled = false, + ArithmeticAbortEnabled = false, + AutoClose = false, + AutoShrink = false, + AutoUpdateStatisticsEnabled = true, + CloseCursorsOnCommitEnabled = false, + LocalCursorsDefault = false, + ConcatenateNullYieldsNull = false, + NumericRoundAbortEnabled = false, + QuotedIdentifiersEnabled = false, + RecursiveTriggersEnabled = false, + BrokerEnabled = false, + AutoUpdateStatisticsAsync = false, + DateCorrelationOptimization = false, + Trustworthy = false, + IsParameterizationForced = false, + IsReadCommittedSnapshotOn = false, + HonorBrokerPriority = false, + RecoveryModel = RecoveryModel.Simple, + UserAccess = DatabaseUserAccess.Multiple, + PageVerify = PageVerify.None, + DatabaseOwnershipChaining = false, + FilestreamNonTransactedAccess = FilestreamNonTransactedAccessType.Off, + TargetRecoveryTime = 60, + DelayedDurability = DelayedDurability.Forced, + IsVarDecimalStorageFormatEnabled = true, + LegacyCardinalityEstimation = DatabaseScopedConfigurationOnOff.Off, + LegacyCardinalityEstimationForSecondary = DatabaseScopedConfigurationOnOff.Primary, + MaxDop = 1, + MaxDopForSecondary = 1, + ParameterSniffing = DatabaseScopedConfigurationOnOff.Off, + ParameterSniffingForSecondary = DatabaseScopedConfigurationOnOff.Primary, + QueryOptimizerHotfixes = DatabaseScopedConfigurationOnOff.Off, + QueryOptimizerHotfixesForSecondary = DatabaseScopedConfigurationOnOff.Primary + }; + var primaryFileGroup = new FileGroup(database, "PRIMARY", FileGroupType.RowsFileGroup); + var sysdataFile = new DataFile(primaryFileGroup, databaseName + "_SysData", sysdataFileName) + { + Size = 8192, + Growth = 65536, + GrowthType = FileGrowthType.KB, + IsPrimaryFile = true + }; + primaryFileGroup.Files.Add(sysdataFile); + database.FileGroups.Add(primaryFileGroup); + var logFile = new LogFile(database, databaseName + "_Log", logFileName) + { + Size = 8192, + Growth = 65536, + GrowthType = FileGrowthType.KB + }; + database.LogFiles.Add(logFile); + database.Create(); + database.SetSnapshotIsolation(true); + database.QueryStoreOptions.DesiredState = QueryStoreOperationMode.Off; + database.SetOwner("sa"); + database.Alter(); + Console.WriteLine("Created On-Disk Database " + databaseName); + } + catch (SmoException smoex) + { + HandleSMOException(smoex, true); + } + catch (Exception ex) + { + HandleException(ex, true); + } + } + + public void CreateInMem(string databaseName, string sysdataFileName, string logFileName, string memoptDirName) + { +#if DEBUG + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); + if (sysdataFileName == null) throw new ArgumentNullException(nameof(sysdataFileName)); + if (logFileName == null) throw new ArgumentNullException(nameof(logFileName)); + if (memoptDirName == null) throw new ArgumentNullException(nameof(memoptDirName)); +#endif + Console.WriteLine("Creating In-Mem Database " + databaseName); + try + { + var database = new Database(Server, databaseName) + { + ContainmentType = ContainmentType.None, + Collation = "Latin1_General_100_CS_AS_KS_WS_SC", + CompatibilityLevel = CompatibilityLevel.Version140, + AnsiNullDefault = false, + AnsiWarningsEnabled = false, + ArithmeticAbortEnabled = false, + AutoClose = false, + AutoShrink = false, + AutoUpdateStatisticsEnabled = true, + CloseCursorsOnCommitEnabled = false, + LocalCursorsDefault = false, + ConcatenateNullYieldsNull = false, + NumericRoundAbortEnabled = false, + QuotedIdentifiersEnabled = false, + RecursiveTriggersEnabled = false, + BrokerEnabled = false, + AutoUpdateStatisticsAsync = false, + DateCorrelationOptimization = false, + Trustworthy = false, + IsParameterizationForced = false, + IsReadCommittedSnapshotOn = false, + HonorBrokerPriority = false, + RecoveryModel = RecoveryModel.Simple, + UserAccess = DatabaseUserAccess.Multiple, + PageVerify = PageVerify.None, + DatabaseOwnershipChaining = false, + FilestreamNonTransactedAccess = FilestreamNonTransactedAccessType.Off, + TargetRecoveryTime = 60, + DelayedDurability = DelayedDurability.Forced, + IsVarDecimalStorageFormatEnabled = true, + LegacyCardinalityEstimation = DatabaseScopedConfigurationOnOff.Off, + LegacyCardinalityEstimationForSecondary = DatabaseScopedConfigurationOnOff.Primary, + MaxDop = 1, + MaxDopForSecondary = 1, + ParameterSniffing = DatabaseScopedConfigurationOnOff.Off, + ParameterSniffingForSecondary = DatabaseScopedConfigurationOnOff.Primary, + QueryOptimizerHotfixes = DatabaseScopedConfigurationOnOff.Off, + QueryOptimizerHotfixesForSecondary = DatabaseScopedConfigurationOnOff.Primary + }; + var primaryFileGroup = new FileGroup(database, "PRIMARY", FileGroupType.RowsFileGroup); + var sysdataFile = new DataFile(primaryFileGroup, databaseName + "_SysData", sysdataFileName) + { + Size = 8192, + Growth = 65536, + GrowthType = FileGrowthType.KB, + IsPrimaryFile = true + }; + primaryFileGroup.Files.Add(sysdataFile); + database.FileGroups.Add(primaryFileGroup); + var logFile = new LogFile(database, databaseName + "_Log", logFileName) + { + Size = 8192, + Growth = 65536, + GrowthType = FileGrowthType.KB + }; + database.LogFiles.Add(logFile); + var memoptFileGroup = new FileGroup(database, "MEMOPT", FileGroupType.MemoryOptimizedDataFileGroup); + var memoptDir = new DataFile(memoptFileGroup, databaseName + "_MemOpt", memoptDirName); + memoptFileGroup.Files.Add(memoptDir); + database.FileGroups.Add(memoptFileGroup); + database.Create(); + database.SetSnapshotIsolation(true); + database.QueryStoreOptions.DesiredState = QueryStoreOperationMode.Off; + database.SetOwner("sa"); + database.Alter(); + Console.WriteLine("Created In-Mem Database " + databaseName); + } + catch (SmoException smoex) + { + HandleSMOException(smoex, true); + } + catch (Exception ex) + { + HandleException(ex, true); + } + } + + public Database GetDatabase(string name) + { + try + { + var db = Server.Databases[name]; + if (db == null) throw new ProvisioningException("Could not retrieve database " + name); + return db; + } + catch (SmoException smoex) + { + HandleSMOException(smoex, true); + } + catch (Exception ex) + { + HandleException(ex, true); + } + return null; + } + + public void MakeDatabaseReadonly(string name) + { + Console.WriteLine("Making Database " + name + " read-only."); + try + { + var db = Server.Databases[name]; + if (db == null) throw new ProvisioningException("Could not retrieve database " + name); + db.DatabaseOptions.ReadOnly = true; + db.Alter(TerminationClause.RollbackTransactionsImmediately); + } + catch (SmoException smoex) + { + HandleSMOException(smoex, false); + } + catch (Exception ex) + { + HandleException(ex, true); + } + Console.WriteLine("Made Database " + name + " read-only."); + } + + private void HandleSMOException(SmoException smoex, bool throwProvisioningException) + { +#if DEBUG + if (smoex == null) throw new ArgumentNullException(nameof(smoex)); +#endif + Console.WriteLine("This is an SMO Exception"); + Console.WriteLine(smoex.Message); + Console.WriteLine(smoex.StackTrace); + Exception ex; + ex = smoex; + while (!object.ReferenceEquals(ex.InnerException, (null))) + { + Console.WriteLine(ex.InnerException.Message); + Console.WriteLine(ex.InnerException.StackTrace); + ex = ex.InnerException; + } + if (throwProvisioningException) throw new ProvisioningException("DatabaseServer caught SMO exception."); + } + + private void HandleException(Exception ex, bool throwProvisioningException) + { +#if DEBUG + if (ex == null) throw new ArgumentNullException(nameof(ex)); +#endif + Console.WriteLine("This is not an SMO exception."); + Console.WriteLine(ex.Message); + Console.WriteLine(ex.StackTrace); + if (throwProvisioningException) throw new ProvisioningException("DatabaseServer caught non-SMO exception."); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/DatabaseServerInterface.cs b/source/C#/UniTraX/Infrastructure/Provisioning/DatabaseServerInterface.cs new file mode 100644 index 0000000000000000000000000000000000000000..7bf4854ab77022263d5636bb00b3bf7d3dcb7335 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/DatabaseServerInterface.cs @@ -0,0 +1,79 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using System; + using System.Data.SqlClient; + + public class DatabaseServerInterface + { + private readonly string Datasource; + private readonly string InitialCatalog; + private readonly string Username; + private readonly string Password; + private SqlConnection Connection; + + public DatabaseServerInterface(string datasource, string initialCatalog, string username, string password) + { +#if DEBUG + if (datasource == null) throw new ArgumentNullException(nameof(datasource)); + if (initialCatalog == null) throw new ArgumentNullException(nameof(initialCatalog)); + if (username == null) throw new ArgumentNullException(nameof(username)); + if (password == null) throw new ArgumentNullException(nameof(password)); +#endif + Datasource = datasource; + InitialCatalog = initialCatalog; + Username = username; + Password = password; + RenewConnection(); + } + + private void RenewConnection() + { + if (Connection != null) + { + try + { + Connection.Close(); + Connection.Dispose(); + } + catch (Exception ex) + { + Console.WriteLine("Database error:"); + Console.WriteLine(ex.Message); + } + } + SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(); + builder["Data Source"] = Datasource; + builder["Initial Catalog"] = InitialCatalog; + builder["User ID"] = Username; + builder["Password"] = Password; + Connection = new SqlConnection(builder.ToString()); + } + + public void Execute(string command) + { + try + { + Connection.Open(); + var sqlCommand = new SqlCommand(command, Connection); + var dataReader = sqlCommand.ExecuteReader(); + while (dataReader.Read()) + { + var values = new Object[dataReader.FieldCount]; + var num = dataReader.GetValues(values); + for (var i = 0; i < num; i++) Console.Write("Value " + values[i] + ";"); + Console.WriteLine(); + } + dataReader.Close(); + sqlCommand.Dispose(); + Connection.Close(); + } + catch (Exception ex) + { + Console.WriteLine("Database error:"); + Console.WriteLine(ex.Message); + RenewConnection(); + throw new ProvisioningException("DatabaseServerInterface caught exception."); + } + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/Hosts/ExampleDBServer.cs b/source/C#/UniTraX/Infrastructure/Provisioning/Hosts/ExampleDBServer.cs new file mode 100644 index 0000000000000000000000000000000000000000..94f6d20af055ebbd3b2aee83766f081c9ed30353 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/Hosts/ExampleDBServer.cs @@ -0,0 +1,148 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using Microsoft.SqlServer.Management.Smo; + using System; + using System.Collections.Generic; + using System.Data.SqlClient; + + public class ExampleDBServer : IExperimentServer + { + private readonly string Host; + private readonly string Username; + private readonly string Password; + private readonly string DatabaseUsername; + private readonly string DatabasePassword; + private readonly ManagedServer Server; + private readonly DatabaseServer DatabaseServer; + private readonly string Hostname; + private readonly string RamDiskSize; + private readonly List PotentialDatabases; + + public ExampleDBServer() + { + Host = @"server0815.example.com"; + Username = @"SERVER0815\Administrator"; + Password = @"password"; + DatabaseUsername = @"sa"; + DatabasePassword = @"password"; + Server = new ManagedServer(Host, Username, Password); + DatabaseServer = new DatabaseServer(Host, DatabaseUsername, DatabasePassword); + Hostname = "server0815"; + RamDiskSize = "131072MB"; + PotentialDatabases = new List(); + PotentialDatabases.Add("mobility-disk"); + PotentialDatabases.Add("mobility-mem"); + PotentialDatabases.Add("financial-disk"); + PotentialDatabases.Add("financial-mem"); + PotentialDatabases.Add("medical-disk"); + PotentialDatabases.Add("medical-mem"); + } + + public void CreateRamDisk() + { + RamDisk.Create(Host, Username, Password, Hostname, RamDiskSize); + } + + public void RemoveRamDisk() + { + RamDisk.Remove(Host, Username, Password, Hostname); + } + + public Database GetDatabase(string name) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + return DatabaseServer.GetDatabase(name); + } + + public string GetDatabaseConnectionString(string databaseName) + { +#if DEBUG + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); +#endif + return @"Data Source=" + Host + @";Initial Catalog=" + databaseName + @";User ID=" + DatabaseUsername + @";Password=" + DatabasePassword; + } + + public SqlConnection GetSqlConnection(string databaseName) + { +#if DEBUG + if (databaseName == null) throw new ArgumentNullException(nameof(databaseName)); +#endif + return new SqlConnection(GetDatabaseConnectionString(databaseName)); + } + + public void StartDBService() + { + Server.StartService(@"MSSQL$MSSQLSERVER2017"); + } + + public void StopDBService() + { + Server.StopService(@"MSSQL$MSSQLSERVER2017"); + } + + public bool DBServiceIsRunning() + { + return Server.ServiceIsRunning(@"MSSQL$MSSQLSERVER2017"); + } + + public bool DBServiceIsStopped() + { + return Server.ServiceIsStopped(@"MSSQL$MSSQLSERVER2017"); + } + + public void CreateDatabaseOnDisk(string name) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + DatabaseServer.CreateOnDisk(name, @"E:\Program Files\Microsoft SQL Server\MSSQL14.MSSQLSERVER2017\MSSQL\Data\" + name + @".mdf", @"E:\Program Files\Microsoft SQL Server\MSSQL14.MSSQLSERVER2017\MSSQL\Data\" + name + @".ldf"); + } + + public void CreateDatabaseInMem(string name) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + DatabaseServer.CreateInMem(name, @"E:\Program Files\Microsoft SQL Server\MSSQL14.MSSQLSERVER2017\MSSQL\Data\" + name + @"_sysdata.mdf", @"E:\Program Files\Microsoft SQL Server\MSSQL14.MSSQLSERVER2017\MSSQL\Data\" + name + @"_log.ldf", @"E:\Program Files\Microsoft SQL Server\MSSQL14.MSSQLSERVER2017\MSSQL\Data\" + name + @"_memopt"); + } + + public void MakeDatabaseReadonly(string name) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + DatabaseServer.MakeDatabaseReadonly(name); + } + + public void Wipe() + { + if (DBServiceIsRunning()) + { + foreach (var db in PotentialDatabases) + { + try + { + DatabaseServer.Drop(db); + } + catch (ProvisioningException ex) + { + Console.WriteLine("Could not drop database " + db + ", caught exception: " + ex.Message); + } + } + StopDBService(); + } + RemoveRamDisk(); + } + + public void Setup() + { + if (!DBServiceIsStopped()) throw new ProvisioningException("Cannot Setup() server0815.example.com: DB Service is already running."); + CreateRamDisk(); + PowerShellInterface shell = new PowerShellInterface(); + shell.ExecuteRemote(@"New-Item -path 'E:\Program Files\Microsoft SQL Server\MSSQL14.MSSQLSERVER2017\MSSQL\Data' -type directory", Host, Username, Password, true); + StartDBService(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/IExperimentServer.cs b/source/C#/UniTraX/Infrastructure/Provisioning/IExperimentServer.cs new file mode 100644 index 0000000000000000000000000000000000000000..b0678c9cd8a6a2f0beee3cae5870d1003b738aa9 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/IExperimentServer.cs @@ -0,0 +1,17 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using Microsoft.SqlServer.Management.Smo; + using System.Data.SqlClient; + + public interface IExperimentServer + { + void CreateDatabaseOnDisk(string name); + void CreateDatabaseInMem(string name); + void MakeDatabaseReadonly(string name); + void Wipe(); + void Setup(); + Database GetDatabase(string name); + string GetDatabaseConnectionString(string databaseName); + SqlConnection GetSqlConnection(string databaseName); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/ManagedServer.cs b/source/C#/UniTraX/Infrastructure/Provisioning/ManagedServer.cs new file mode 100644 index 0000000000000000000000000000000000000000..d34a7ebe87ff0a30af5f57c7418b28dd9167dfa6 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/ManagedServer.cs @@ -0,0 +1,155 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using Microsoft.SqlServer.Management.Smo; + using Microsoft.SqlServer.Management.Smo.Wmi; + using System; + + public class ManagedServer + { + private ManagedComputer Host; + + public ManagedServer (string host, string username, string password) + { +#if DEBUG + if (host == null) throw new ArgumentNullException(nameof(host)); + if (username == null) throw new ArgumentNullException(nameof(username)); + if (password == null) throw new ArgumentNullException(nameof(password)); +#endif + Host = new ManagedComputer(host, username, password); + Host.ConnectionSettings.ProviderArchitecture = ProviderArchitecture.Use64bit; + } + + public void ListServices() + { + try + { + Console.WriteLine("Listing Services"); + foreach (Service s in Host.Services) Console.WriteLine("Service: DisplayName " + s.DisplayName + " Name " + s.Name + " Description " + s.Description); + Console.WriteLine("Listed Services"); + } + catch (SmoException smoex) + { + HandleSMOException(smoex); + } + catch (Exception ex) + { + HandleException(ex); + } + } + + public void StartService(string name) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + try + { + Console.WriteLine("Starting Service " + name); + var service = Host.Services[name]; + if (service.ServiceState != ServiceState.Running) service.Start(); + while (service.ServiceState != ServiceState.Running) + { + service.Refresh(); + } + Console.WriteLine("Started Service " + name); + } + catch (SmoException smoex) + { + HandleSMOException(smoex); + } + catch (Exception ex) + { + HandleException(ex); + } + } + + public void StopService(string name) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); +#endif + try + { + Console.WriteLine("Stopping Service " + name); + var service = Host.Services[name]; + if (service.ServiceState != ServiceState.Stopped) service.Stop(); + while (service.ServiceState != ServiceState.Stopped) service.Refresh(); + Console.WriteLine("Stopped Service " + name); + } + catch (SmoException smoex) + { + HandleSMOException(smoex); + } + catch (Exception ex) + { + HandleException(ex); + } + } + + public bool ServiceIsRunning(string name) + { + try + { + Console.WriteLine("Reading State of Service " + name); + if (Host.Services[name].ServiceState != ServiceState.Running) return false; + } + catch (SmoException smoex) + { + HandleSMOException(smoex); + } + catch (Exception ex) + { + HandleException(ex); + } + return true; + } + + public bool ServiceIsStopped(string name) + { + try + { + Console.WriteLine("Reading State of Service " + name); + if (Host.Services[name].ServiceState != ServiceState.Stopped) return false; + } + catch (SmoException smoex) + { + HandleSMOException(smoex); + } + catch (Exception ex) + { + HandleException(ex); + } + return true; + } + + private void HandleSMOException(SmoException smoex) + { +#if DEBUG + if (smoex == null) throw new ArgumentNullException(nameof(smoex)); +#endif + Console.WriteLine("This is an SMO Exception"); + Console.WriteLine(smoex.Message); + Console.WriteLine(smoex.StackTrace); + Exception ex; + ex = smoex; + while (!object.ReferenceEquals(ex.InnerException, (null))) + { + Console.WriteLine(ex.InnerException.Message); + Console.WriteLine(ex.InnerException.StackTrace); + ex = ex.InnerException; + } + throw new ProvisioningException("ManagedServer caught SMO exception."); + } + + private void HandleException(Exception ex) + { +#if DEBUG + if (ex == null) throw new ArgumentNullException(nameof(ex)); +#endif + Console.WriteLine("This is not an SMO exception."); + Console.WriteLine(ex.Message); + Console.WriteLine(ex.StackTrace); + throw new ProvisioningException("ManagedServer caught non-SMO exception."); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/PowerShellInterface.cs b/source/C#/UniTraX/Infrastructure/Provisioning/PowerShellInterface.cs new file mode 100644 index 0000000000000000000000000000000000000000..9b9cd5168f58613657ffe1bf3e9236615f718e4d --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/PowerShellInterface.cs @@ -0,0 +1,58 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using System; + using System.Collections.ObjectModel; + using System.Management.Automation; + + public class PowerShellInterface + { + private PowerShell Shell; + + public PowerShellInterface() { } + + public void Execute(string script, bool stopAtNonTerminatingErrors) + { +#if DEBUG + if (script == null) throw new ArgumentNullException(nameof(script)); +#endif + Console.WriteLine("Running PowerShell script: " + script); + try + { + Shell = PowerShell.Create(); + Shell.AddScript(script); + Collection results = Shell.Invoke(); + Console.WriteLine("Output:"); + foreach (var psObject in results) Console.WriteLine(psObject); + var hasErrors = Shell.Streams.Error.Count > 0; + if (hasErrors) Console.WriteLine("Non-terminating errors:"); + foreach (ErrorRecord err in Shell.Streams.Error) Console.WriteLine(err.ToString()); + if (hasErrors && stopAtNonTerminatingErrors) throw new ProvisioningException("PowerShellInterface caught non-terminating errors."); + Console.WriteLine("Ran PowerShell script"); + Shell.Dispose(); + } + catch (RuntimeException ex) + { + Console.WriteLine("Terminating error:"); + Console.WriteLine(ex.Message); + Console.WriteLine(ex.StackTrace); + if (Shell != null) Shell.Dispose(); + throw new ProvisioningException("PowerShellInterface caught RuntimeException."); + } + } + + public void ExecuteRemote(string script, string host, string username, string password, bool stopAtNonTerminatingErrors) + { +#if DEBUG + if (script == null) throw new ArgumentNullException(nameof(script)); + if (host == null) throw new ArgumentNullException(nameof(host)); + if (username == null) throw new ArgumentNullException(nameof(username)); + if (password == null) throw new ArgumentNullException(nameof(password)); +#endif + string s = @"$Pass = ConvertTo-SecureString -AsPlainText " + password + + " -Force; $Cred = New-Object System.Management.Automation.PSCredential -ArgumentList " + username + + ",$Pass; Invoke-Command -Credential $Cred -ComputerName " + host + + " -ScriptBlock {" + script + "} | Out-Null"; + Execute(s, stopAtNonTerminatingErrors); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/ProvisioningException.cs b/source/C#/UniTraX/Infrastructure/Provisioning/ProvisioningException.cs new file mode 100644 index 0000000000000000000000000000000000000000..b6c4136fabf67d8fb8636773fa58179f6e43a98b --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/ProvisioningException.cs @@ -0,0 +1,9 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using System; + + public class ProvisioningException : Exception + { + public ProvisioningException(string message) : base(message) { } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Infrastructure/Provisioning/RamDisk.cs b/source/C#/UniTraX/Infrastructure/Provisioning/RamDisk.cs new file mode 100644 index 0000000000000000000000000000000000000000..8cf65d532173e43300abccf02bb12d1a854fec58 --- /dev/null +++ b/source/C#/UniTraX/Infrastructure/Provisioning/RamDisk.cs @@ -0,0 +1,55 @@ +namespace Qbb.Infrastructure.Provisioning +{ + using System; + + public class RamDisk + { + public static void Create(string host, string username, string password, string hostname, string size) + { +#if DEBUG + if (host == null) throw new ArgumentNullException(nameof(host)); + if (username == null) throw new ArgumentNullException(nameof(username)); + if (password == null) throw new ArgumentNullException(nameof(password)); + if (hostname == null) throw new ArgumentNullException(nameof(hostname)); + if (size == null) throw new ArgumentNullException(nameof(size)); +#endif + Console.WriteLine("Creating RamDisk on " + host + " with username " + username + " password " + password + " hostname " + hostname + " and size " + size); + PowerShellInterface shell = new PowerShellInterface(); + shell.ExecuteRemote(@"New-IscsiVirtualDisk -Path 'ramdisk:dbdisk.vhdx' -Size " + size, host, username, password, true); + shell.ExecuteRemote(@"New-IscsiServerTarget -TargetName 'dbdisk' -InitiatorIds 'IQN:iqn.1991-05.com.microsoft:" + host + @"'", host, username, password, true); + shell.ExecuteRemote(@"Add-IscsiVirtualDiskTargetMapping -TargetName 'dbdisk' -Path 'ramdisk:dbdisk.vhdx'", host, username, password, true); + shell.ExecuteRemote(@"New-IscsiTargetPortal -TargetPortalAddress '" + host + @"' -TargetPortalPortNumber 3260", host, username, password, true); + shell.ExecuteRemote(@"$address = (Get-InitiatorPort); $address | Select NodeAddress; Set-InitiatorPort -NodeAddress $address.NodeAddress -NewNodeAddress 'iqn.1991-05.com.microsoft:" + host + "'", host, username, password, true); + shell.ExecuteRemote(@"Get-IscsiTarget", host, username, password, true); + shell.ExecuteRemote(@"Connect-IscsiTarget -NodeAddress 'iqn.1991-05.com.microsoft:" + hostname + @"-dbdisk-target' -IsPersistent $False", host, username, password, true); + shell.ExecuteRemote(@"Initialize-Disk -Number 1", host, username, password, true); + shell.ExecuteRemote(@"New-Partition -DiskNumber 1 -UseMaximumSize -DriveLetter E -Alignment 1048576", host, username, password, true); + shell.ExecuteRemote(@"Format-Volume -DriveLetter E -AllocationUnitSize 65536 -FileSystem 'NTFS'", host, username, password, true); + shell.ExecuteRemote(@"Remove-NTFSAccess -Path 'E:\' -Account 'BUILTIN\Users' -AccessRights AppendData", host, username, password, true); + shell.ExecuteRemote(@"Remove-NTFSAccess -Path 'E:\' -Account 'BUILTIN\Users' -AccessRights CreateFiles", host, username, password, true); + shell.ExecuteRemote(@"Remove-NTFSAccess -Path 'E:\' -Account 'BUILTIN\Users' -AccessRights ReadAndExecute", host, username, password, true); + shell.ExecuteRemote(@"Add-NTFSAccess -Path 'E:\' -Account 'BUILTIN\Users' -AccessRights FullControl -AppliesTo ThisFolderSubfoldersAndFiles", host, username, password, true); + Console.WriteLine("Created RamDisk on " + host); + } + + public static void Remove(string host, string username, string password, string hostname) + { +#if DEBUG + if (host == null) throw new ArgumentNullException(nameof(host)); + if (username == null) throw new ArgumentNullException(nameof(username)); + if (password == null) throw new ArgumentNullException(nameof(password)); + if (hostname == null) throw new ArgumentNullException(nameof(hostname)); +#endif + Console.WriteLine("Removing RamDisk on " + host + " with username " + username + " password " + password + " and hostname " + hostname); + PowerShellInterface shell = new PowerShellInterface(); + shell.ExecuteRemote(@"Get-IscsiTarget", host, username, password, false); + shell.ExecuteRemote(@"Disconnect-IscsiTarget -NodeAddress 'iqn.1991-05.com.microsoft:" + hostname + @"-dbdisk-target' -Confirm:$false", host, username, password, false); + shell.ExecuteRemote(@"Remove-IscsiTargetPortal -TargetPortalAddress 'localhost' -Confirm:$false", host, username, password, false); + shell.ExecuteRemote(@"Remove-IscsiTargetPortal -TargetPortalAddress '" + hostname + @"' -Confirm:$false", host, username, password, false); + shell.ExecuteRemote(@"Remove-IscsiVirtualDiskTargetMapping -TargetName 'dbdisk' -Path 'ramdisk:dbdisk.vhdx'", host, username, password, false); + shell.ExecuteRemote(@"Remove-IscsiServerTarget -TargetName 'dbdisk'", host, username, password, false); + shell.ExecuteRemote(@"Remove-IscsiVirtualDisk -Path 'ramdisk:dbdisk.vhdx'", host, username, password, false); + Console.WriteLine("Removed RamDisk on " + host); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/Aggregate.cs b/source/C#/UniTraX/Queries/Aggregation/Aggregate.cs new file mode 100644 index 0000000000000000000000000000000000000000..5d9979d28af80601ead73ba1726449fb348fb4bf --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/Aggregate.cs @@ -0,0 +1,173 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Linq; + + public abstract class Aggregate : IQuery + { + public string Name { get; private set; } + public long Epsilon { get; private set; } + public Func ConvertBudget { get; private set; } + public IPINQCondition PINQCondition { get; private set; } + public int CleanThreshold { get; private set; } + + public Aggregate(string name, long epsilon, Func convertBudget, IPINQCondition pinqCondition, int cleanThreshold) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); + if (epsilon < 0) throw new ArgumentException("epsilon is not valid"); + if (convertBudget == null) throw new ArgumentNullException(nameof(convertBudget)); + if (pinqCondition == null) throw new ArgumentNullException(nameof(pinqCondition)); +#endif + Name = name; + Epsilon = epsilon; + ConvertBudget = convertBudget; + PINQCondition = pinqCondition; + CleanThreshold = cleanThreshold; + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif +#if RECORD + var setupTime = Recorder.StopGlobalTimer(); + Recorder.StartGlobalTimer(); +#endif + var queryables = PINQCondition.ApplyTo(queryable); + var aggregates = new double[queryables.Length]; + for (int i = 0; i < queryables.Length; i++) + { + aggregates[i] = SpecificApplyTo(queryables[i]); +#if RECORD + var executeTime = Recorder.StopGlobalTimer(); + Log(queryable, aggregates[i], setupTime, executeTime, 0.0, CleanThreshold); + setupTime = 0.0; + Recorder.StartGlobalTimer(); +#endif + } + return aggregates; + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif +#if RECORD + var setupTime = Recorder.StopGlobalTimer(); + Recorder.StartGlobalTimer(); +#endif + var queryables = PINQCondition.ApplyTo(queryable); + var aggregates = new double[queryables.Length]; + for (int i = 0; i < queryables.Length; i++) + { + aggregates[i] = SpecificApplyTo(queryables[i]); +#if RECORD + var executeTime = Recorder.StopGlobalTimer(); + Log(queryable, aggregates[i], setupTime, executeTime, 0.0, CleanThreshold); + setupTime = 0.0; + Recorder.StartGlobalTimer(); +#endif + } + return aggregates; + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif +#if RECORD + var setupTime = Recorder.StopGlobalTimer(); + var setupNumberPartitions = queryable.Bookkeeper.NumberPartitions(); + Recorder.StartGlobalTimer(); +#endif + PINQueryable[] queryables = null; + var bkTime = 0.0; + if (Epsilon > 0L) + { + var pinqQueryable = queryable.AsPinq(Epsilon); +#if RECORD + bkTime = Recorder.StopGlobalTimer(); + Recorder.StartGlobalTimer(); +#endif + queryables = PINQCondition.ApplyTo(pinqQueryable); + } + else queryables = new PINQueryable[] { new PINQueryable(Enumerable.Empty().AsQueryable(), new PINQAgent()) }; + var aggregates = new double[queryables.Length]; +#if RECORD + var executeTime = 0.0; +#endif + for (int i = 0; i < queryables.Length; i++) + { + aggregates[i] = SpecificApplyTo(queryables[i]); +#if RECORD + executeTime = Recorder.StopGlobalTimer(); + if (i < queryables.Length - 1) + { + Log(queryable, aggregates[i], setupTime, executeTime + bkTime, bkTime, 0.0, CleanThreshold); + setupTime = 0.0; + bkTime = 0.0; + } + Recorder.StartGlobalTimer(); +#endif + } + var executeNumberPartitions = queryable.Bookkeeper.NumberPartitions(); + Teardown(queryable, executeNumberPartitions, CleanThreshold); +#if RECORD + var teardownTime = Recorder.StopGlobalTimer(); + var teardownNumberPartitions = queryable.Bookkeeper.NumberPartitions(); + Recorder.Add(new EntryNumberPartitions(setupNumberPartitions, executeNumberPartitions, teardownNumberPartitions)); + Log(queryable, aggregates[aggregates.Length - 1], setupTime, executeTime + bkTime, bkTime, teardownTime, CleanThreshold); + Recorder.StartGlobalTimer(); +#endif + return aggregates; + } + + public virtual void Teardown(BKQueryable queryable, int executeNumberPartitions, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (executeNumberPartitions - queryable.Bookkeeper.NumberPartitionsAfterClean() > cleanThreshold) queryable.CleanHistory(); + } + + public virtual void Log(IQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Recorder.Add(new EntryAggregate(Name, "", -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public virtual void Log(PINQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Recorder.Add(new EntryAggregate(Name, "", -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public virtual void Log(BKQueryable queryable, double aggregate, double setupTime, double executeTime, double bkExecuteTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Recorder.Add(new EntryAggregate(Name, "", Epsilon, aggregate, setupTime, executeTime, bkExecuteTime, teardownTime, cleanThreshold)); + } + + public abstract double SpecificApplyTo(IQueryable queryable); + + public abstract double SpecificApplyTo(PINQueryable queryable); + + public abstract IQuery Clone(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/Average.cs b/source/C#/UniTraX/Queries/Aggregation/Average.cs new file mode 100644 index 0000000000000000000000000000000000000000..91660d8115ea72f5e41b50c28a24bab79e7ac8ae --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/Average.cs @@ -0,0 +1,96 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Linq; + using System.Linq.Expressions; + + public class Average : Aggregate + { + private class PINQAverage + { + public static double Calculate(PINQueryable queryable, long epsilon, Func convertBudget, Field field, double scaleValue) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (epsilon < 0) throw new ArgumentException("epsilon is not valid"); + if (convertBudget == null) throw new ArgumentNullException(nameof(convertBudget)); + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue == 0) throw new ArgumentException("scaleValue is not valid"); +#endif + Expression> exp = Expression.Lambda>(Expression.Divide(field.Access.Body, Expression.Constant(scaleValue)), field.Access.Parameters); + return queryable.NoisyAverage(convertBudget(epsilon, false), exp) * scaleValue; + } + } + + private readonly Field Field; + private readonly double ScaleValue; + + public Average(long epsilon, Func convertBudget, IPINQCondition pinqCondition, Field field, double scaleValue, int cleanThreshold) : + base("Average", epsilon, convertBudget, pinqCondition, cleanThreshold) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue == 0) throw new ArgumentException("scaleValue is not valid"); +#endif + Field = field; + ScaleValue = scaleValue; + } + + public override double SpecificApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + return queryable.Average(Field.Access); + } + + public override double SpecificApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + return PINQAverage.Calculate(queryable, Epsilon, ConvertBudget, Field, ScaleValue); + } + + public override void Log(IQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(PINQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(BKQueryable queryable, double aggregate, double setupTime, double executeTime, double bkExecuteTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, Epsilon, aggregate, setupTime, executeTime, bkExecuteTime, teardownTime, cleanThreshold)); + } + + public override IQuery Clone() + { + return new Average(Epsilon, ConvertBudget, PINQCondition, Field, ScaleValue, CleanThreshold); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/Count.cs b/source/C#/UniTraX/Queries/Aggregation/Count.cs new file mode 100644 index 0000000000000000000000000000000000000000..42caf7d554d3b9e94ed60c57b8bf6843c30ceb85 --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/Count.cs @@ -0,0 +1,50 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Queries.Core; + using System; + using System.Linq; + + public class Count : Aggregate + { + private class PINQCount + { + public static double Calculate(PINQueryable queryable, long epsilon, Func convertBudget) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (epsilon < 0) throw new ArgumentException("epsilon is not valid"); + if (convertBudget == null) throw new ArgumentNullException(nameof(convertBudget)); +#endif + return queryable.NoisyCount(convertBudget(epsilon, false)); + } + } + + public Count(long epsilon, Func convertBudget, IPINQCondition pinqCondition, int cleanThreshold) : + base("Count", epsilon, convertBudget, pinqCondition, cleanThreshold) + { + + } + + public override double SpecificApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return queryable.Count(); + } + + public override double SpecificApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return PINQCount.Calculate(queryable, Epsilon, ConvertBudget); + } + + public override IQuery Clone() + { + return new Count(Epsilon, ConvertBudget, PINQCondition, CleanThreshold); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/Dummy.cs b/source/C#/UniTraX/Queries/Aggregation/Dummy.cs new file mode 100644 index 0000000000000000000000000000000000000000..3b165ce874ab99cc39bc1367bbc2c9633dbe7c59 --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/Dummy.cs @@ -0,0 +1,48 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Queries.Core; + using System; + using System.Linq; + + public class Dummy : Aggregate + { + private readonly bool Run; + + public Dummy(long epsilon, Func convertBudget, IPINQCondition pinqCondition, bool run, int cleanThreshold) : + base("Dummy", run ? epsilon : 0L, convertBudget, pinqCondition, cleanThreshold) + { + Run = run; + } + + public override IQuery Clone() + { + return new Dummy(Epsilon, ConvertBudget, PINQCondition, Run, CleanThreshold); + } + + public override double SpecificApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return 0.0; + } + + public override double SpecificApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return 0.0; + } + + public override void Teardown(BKQueryable queryable, int executeNumberPartitions, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (Run && executeNumberPartitions - queryable.Bookkeeper.NumberPartitionsAfterClean() > cleanThreshold) queryable.CleanHistory(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/ExponentialMechanism.cs b/source/C#/UniTraX/Queries/Aggregation/ExponentialMechanism.cs new file mode 100644 index 0000000000000000000000000000000000000000..118e85f5eb2a0172d1b76467654cc17c52cf7344 --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/ExponentialMechanism.cs @@ -0,0 +1,72 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Core.Specification; + using Qbb.Queries.Core; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + + public class ExponentialMechanism : Aggregate + { + private class PINQExponentialMechanism + { + public static double Calculate(PINQueryable queryable, long epsilon, Func convertBudget, + IQueryable values, Expression> scoreFunction) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (epsilon < 0) throw new ArgumentException("epsilon is not valid"); + if (convertBudget == null) throw new ArgumentNullException(nameof(convertBudget)); + if (values == null) throw new ArgumentNullException(nameof(values)); + if (scoreFunction == null) throw new ArgumentNullException(nameof(scoreFunction)); +#endif + var target = queryable.ExponentialMechanism(convertBudget(epsilon, false), values, scoreFunction); + return values.Select((x, i) => new { Value = x, Index = i }).Where(t => t.Value.Equals(target)).First().Index; + } + } + + private readonly IQueryable Values; + private readonly Expression> ScoreFunction; + + public ExponentialMechanism(long epsilon, Func convertBudget, IPINQCondition pinqCondition, IQueryable values, Expression> scoreFunction, int cleanThreshold) : + base("ExponentialMechanism", epsilon, convertBudget, pinqCondition, cleanThreshold) + { +#if DEBUG + if (values == null) throw new ArgumentNullException(nameof(values)); + if (scoreFunction == null) throw new ArgumentNullException(nameof(scoreFunction)); +#endif + Values = values; + ScoreFunction = scoreFunction; + } + + public override double SpecificApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Values == null) throw new ArgumentNullException(nameof(Values)); + if (ScoreFunction == null) throw new ArgumentNullException(nameof(ScoreFunction)); +#endif + var compiled = ScoreFunction.Compile(); + var scores = Values.Select(r => new KeyValuePair(r, queryable.Select(x => compiled.Invoke(x, r)).Sum())).ToArray(); + var target = scores.Aggregate((x, y) => x.Value > y.Value ? x : y).Key; + return Values.Select((x, i) => new { Value = x, Index = i }).Where(t => t.Value.Equals(target)).First().Index; + } + + public override double SpecificApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Values == null) throw new ArgumentNullException(nameof(Values)); + if (ScoreFunction == null) throw new ArgumentNullException(nameof(ScoreFunction)); +#endif + return PINQExponentialMechanism.Calculate(queryable, Epsilon, ConvertBudget, Values, ScoreFunction); + } + + public override IQuery Clone() + { + return new ExponentialMechanism(Epsilon, ConvertBudget, PINQCondition, Values, ScoreFunction, CleanThreshold); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/Median.cs b/source/C#/UniTraX/Queries/Aggregation/Median.cs new file mode 100644 index 0000000000000000000000000000000000000000..078bc8277bde74c598ae147d715e5fb503673c5d --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/Median.cs @@ -0,0 +1,103 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Linq; + using System.Linq.Expressions; + + public class Median : Aggregate + { + private class PINQMedian + { + public static double Calculate(PINQueryable queryable, long epsilon, Func convertBudget, + Field field, double shiftValue, double scaleValue) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (epsilon < 0) throw new ArgumentException("epsilon is not valid"); + if (convertBudget == null) throw new ArgumentNullException(nameof(convertBudget)); + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue == 0) throw new ArgumentException("scaleValue is not valid"); +#endif + Expression> exp = Expression.Lambda>(Expression.Divide(Expression.Add( + field.Access.Body, Expression.Constant(shiftValue)), Expression.Constant(scaleValue)),field.Access.Parameters); + return queryable.NoisyMedian(convertBudget(epsilon, false), exp); + } + } + + private readonly Field Field; + private readonly double ShiftValue; + private readonly double ScaleValue; + + public Median(long epsilon, Func convertBudget, IPINQCondition pinqCondition, Field field, double shiftValue, double scaleValue, int cleanThreshold) : + base("Median", epsilon, convertBudget, pinqCondition, cleanThreshold) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue == 0) throw new ArgumentException("scaleValue is not valid"); +#endif + Field = field; + ShiftValue = shiftValue; + ScaleValue = scaleValue; + } + + public override double SpecificApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + var projectedQueryable = queryable.Select(Field.Access); + var orderedQueryable = projectedQueryable.OrderBy(x => x); + var list = orderedQueryable.ToList(); + return (list.Skip(list.Count / 2).First() + ShiftValue) / ScaleValue; + } + + public override double SpecificApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + return PINQMedian.Calculate(queryable, Epsilon, ConvertBudget, Field, ShiftValue, ScaleValue); + } + + public override void Log(IQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(PINQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(BKQueryable queryable, double aggregate, double setupTime, double executeTime, double bkExecuteTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, Epsilon, aggregate, setupTime, executeTime, bkExecuteTime, teardownTime, cleanThreshold)); + } + + public override IQuery Clone() + { + return new Median(Epsilon, ConvertBudget, PINQCondition, Field, ShiftValue, ScaleValue, CleanThreshold); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/OrderStatistic.cs b/source/C#/UniTraX/Queries/Aggregation/OrderStatistic.cs new file mode 100644 index 0000000000000000000000000000000000000000..b82a34b343ddeac85bc059e69f06efb6e55493aa --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/OrderStatistic.cs @@ -0,0 +1,104 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Linq; + using System.Linq.Expressions; + + public class OrderStatistic : Aggregate + { + private class PINQOrderStatistic + { + public static double Calculate(PINQueryable queryable, long epsilon, Func convertBudget, + Field field, double shiftValue, double scaleValue, double fraction) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (epsilon < 0L) throw new ArgumentException("epsilon is not valid"); + if (convertBudget == null) throw new ArgumentNullException(nameof(convertBudget)); + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue == 0) throw new ArgumentException("scaleValue is not valid"); + if (fraction < 0 || 1 < fraction) throw new ArgumentException("fraction is not valid"); +#endif + Expression> exp = Expression.Lambda>(Expression.Divide(Expression.Add(field.Access.Body, Expression.Constant(shiftValue)), Expression.Constant(scaleValue)), field.Access.Parameters); + return queryable.NoisyOrderStatistic(convertBudget(epsilon, false), fraction, exp); + } + } + + private readonly Field Field; + private readonly double ShiftValue; + private readonly double ScaleValue; + private readonly double Fraction; + + public OrderStatistic(long epsilon, Func convertBudget, IPINQCondition pinqCondition, Field field, double shiftValue, double scaleValue, double fraction, int cleanThreshold) : + base("OrderStatistics", epsilon, convertBudget, pinqCondition, cleanThreshold) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue == 0) throw new ArgumentException("scaleValue is not valid"); + if (fraction < 0.0 || 1.0 < fraction) throw new ArgumentException("fraction is not valid"); +#endif + Field = field; + ShiftValue = shiftValue; + ScaleValue = scaleValue; + Fraction = fraction; + } + + public override double SpecificApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var projectedQueryable = queryable.Select(Field.Access); + var orderedQueryable = projectedQueryable.OrderBy(x => x); + var count = orderedQueryable.Count(); + return (orderedQueryable.Skip(Convert.ToInt32(Fraction * count)).First() + ShiftValue) / ScaleValue; + } + + public override double SpecificApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return PINQOrderStatistic.Calculate(queryable, Epsilon, ConvertBudget, Field, ShiftValue, ScaleValue, Fraction); + } + + public override void Log(IQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(PINQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(BKQueryable queryable, double aggregate, double setupTime, double executeTime, double bkExecuteTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, Epsilon, aggregate, setupTime, executeTime, bkExecuteTime, teardownTime, cleanThreshold)); + } + + public override IQuery Clone() + { + return new OrderStatistic(Epsilon, ConvertBudget, PINQCondition, Field, ShiftValue, ScaleValue, Fraction, CleanThreshold); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Aggregation/Sum.cs b/source/C#/UniTraX/Queries/Aggregation/Sum.cs new file mode 100644 index 0000000000000000000000000000000000000000..571189ea01ae0f2ffb6e0b7745de954b75768b2b --- /dev/null +++ b/source/C#/UniTraX/Queries/Aggregation/Sum.cs @@ -0,0 +1,95 @@ +namespace Qbb.Queries.Aggregation +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using Qbb.Queries.Core; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Linq; + using System.Linq.Expressions; + + public class Sum : Aggregate + { + private class PINQSum + { + public static double Calculate(PINQueryable queryable, long epsilon, Func convertBudget, + Field field, double scaleValue) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (epsilon < 0) throw new ArgumentException("epsilon is not valid"); + if (convertBudget == null) throw new ArgumentNullException(nameof(convertBudget)); + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue <= 0) throw new ArgumentException("scaleValue is not valid"); +#endif + Expression> exp = Expression.Lambda>(Expression.Divide(field.Access.Body, Expression.Constant(scaleValue)), field.Access.Parameters); + return queryable.NoisySum(convertBudget(epsilon, false), exp) * scaleValue; + } + } + + private readonly Field Field; + private readonly double ScaleValue; + + public Sum(long epsilon, Func convertBudget, IPINQCondition pinqCondition, Field field, double scaleValue, int cleanThreshold) : + base("Sum", epsilon, convertBudget, pinqCondition, cleanThreshold) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (scaleValue <= 0) throw new ArgumentException("scaleValue is not valid"); +#endif + Field = field; + ScaleValue = scaleValue; + } + + public override double SpecificApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return queryable.Sum(Field.Access); + } + + public override double SpecificApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + return PINQSum.Calculate(queryable, Epsilon, ConvertBudget, Field, ScaleValue); + } + + public override void Log(IQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(PINQueryable queryable, double aggregate, double setupTime, double executeTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, -1L, aggregate, setupTime, executeTime, 0.0, teardownTime, cleanThreshold)); + } + + public override void Log(BKQueryable queryable, double aggregate, double setupTime, double executeTime, double bkExecuteTime, double teardownTime, int cleanThreshold) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); + if (Field == null) throw new ArgumentNullException(nameof(Field)); +#endif + Recorder.Add(new EntryAggregate(Name, Field.Name, Epsilon, aggregate, setupTime, executeTime, bkExecuteTime, teardownTime, cleanThreshold)); + } + + public override IQuery Clone() + { + return new Sum(Epsilon, ConvertBudget, PINQCondition, Field, ScaleValue, CleanThreshold); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/AnalyticBase.cs b/source/C#/UniTraX/Queries/Core/AnalyticBase.cs new file mode 100644 index 0000000000000000000000000000000000000000..dcfafd8e38ac9638e8dd823059d50bde0a2f69c7 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/AnalyticBase.cs @@ -0,0 +1,108 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; +#if RECORD + using Qbb.Queries.Recording; +#endif +#if DEBUG + using System; +#endif + using System.Linq; + + public class AnalyticBase : IQuery + { + public long Id { get; private set; } + public string Message { get; private set; } + public IQuery Query { get; private set; } + + public AnalyticBase(long id, string message, IQuery query) + { +#if DEBUG + if (id < 0) throw new ArgumentException("id is not valid"); + if (message == null) throw new ArgumentNullException(nameof(message)); + if (query == null) throw new ArgumentNullException(nameof(query)); +#endif + Id = id; + Message = message; + Query = query; + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif +#if RECORD + var timer = Recorder.StartTimer(); +#endif + var values = Query.ApplyTo(queryable); +#if RECORD + var time = Recorder.StopTimer(timer); + Log(queryable, time); +#endif + return values; + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif +#if RECORD + var timer = Recorder.StartTimer(); +#endif + var values = Query.ApplyTo(queryable); +#if RECORD + var time = Recorder.StopTimer(timer); + Log(queryable, time); +#endif + return values; + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif +#if RECORD + var timer = Recorder.StartTimer(); +#endif + var values = Query.ApplyTo(queryable); +#if RECORD + var time = Recorder.StopTimer(timer); + Log(queryable, time); +#endif + return values; + } + + public void Log(IQueryable queryable, double time) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Recorder.Add(new EntryAnalytic(Id, time, Message)); + } + + public void Log(PINQueryable queryable, double time) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Recorder.Add(new EntryAnalytic(Id, time, Message)); + } + + public void Log(BKQueryable queryable, double time) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Recorder.Add(new EntryAnalytic(Id, time, Message)); + } + + public IQuery Clone() + { + return new AnalyticBase(Id, Message, Query); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/AnalyticBaseFactory.cs b/source/C#/UniTraX/Queries/Core/AnalyticBaseFactory.cs new file mode 100644 index 0000000000000000000000000000000000000000..f127081085a6659dd4681e9724175627f4710345 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/AnalyticBaseFactory.cs @@ -0,0 +1,47 @@ +namespace Qbb.Queries.Core +{ + using System; + + public abstract class AnalyticBaseFactory + { + public long ID { get; private set; } + public string QueryMessage { get; private set; } + public bool UseEvenly { get; private set; } + public bool BudgetOverRuntime { get; private set; } + public int CleanThreshold { get; private set; } + public long[] Epsilons { get; private set; } + + public AnalyticBaseFactory(long id, string queryMessage, bool useEvenly, bool budgetOverRuntime, int cleanThreshold, long[] epsilons) + { +#if DEBUG + if (epsilons == null) throw new ArgumentNullException(nameof(epsilons)); + foreach (long epsilon in epsilons) if (epsilon < 0L) throw new ArgumentException("epsilon below zero"); +#endif + ID = id; + QueryMessage = queryMessage; + UseEvenly = useEvenly; + BudgetOverRuntime = budgetOverRuntime; + CleanThreshold = cleanThreshold; + Epsilons = epsilons; + } + + public IQuery GetAnalyticBaseDirect() + { + return new AnalyticBase(ID, QueryMessage, GenerateQueryDirect()); + } + + public IQuery GetAnalyticBasePINQ() + { + return new AnalyticBase(ID, QueryMessage, GenerateQueryPINQ()); + } + + public IQuery GetAnalyticBaseBK() + { + return new AnalyticBase(ID, QueryMessage, GenerateQueryBK()); + } + + public abstract IQuery GenerateQueryDirect(); + public abstract IQuery GenerateQueryPINQ(); + public abstract IQuery GenerateQueryBK(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/AnalyticBaseProvider.cs b/source/C#/UniTraX/Queries/Core/AnalyticBaseProvider.cs new file mode 100644 index 0000000000000000000000000000000000000000..3e4528d32b2f16702bb64a53b4bb77fdb2e44b4d --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/AnalyticBaseProvider.cs @@ -0,0 +1,40 @@ +namespace Qbb.Queries.Core +{ + using System; + using System.Collections.Generic; + + public class AnalyticBaseProvider + { + private readonly List> Factories; + + public AnalyticBaseProvider(List> factories) + { +#if DEBUG + if (factories == null) throw new ArgumentNullException(nameof(factories)); + if (factories.Count < 1) throw new ArgumentException("need at least one factory"); +#endif + Factories = factories; + } + + public List> GetQueriesDirect() + { + var queries = new List>(); + foreach (var factory in Factories) queries.Add(factory.GetAnalyticBaseDirect()); + return queries; + } + + public List> GetQueriesPINQ() + { + var queries = new List>(); + foreach (var factory in Factories) queries.Add(factory.GetAnalyticBasePINQ()); + return queries; + } + + public List> GetQueriesBK() + { + var queries = new List>(); + foreach (var factory in Factories) queries.Add(factory.GetAnalyticBaseBK()); + return queries; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/Conditional.cs b/source/C#/UniTraX/Queries/Core/Conditional.cs new file mode 100644 index 0000000000000000000000000000000000000000..f8677cc5c2257828057bc11151d6a74fbeb7b820 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/Conditional.cs @@ -0,0 +1,110 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Linq; + + public class Conditional : IQuery + { + private readonly Func EvalFunction; + private readonly IQuery QueryCondition; + private readonly IQuery QueryTrue; + private readonly IQuery QueryFalse; + + public Conditional(Func evalFunction, IQuery queryCondition, IQuery queryTrue, IQuery queryFalse) + { +#if DEBUG + if (queryCondition == null) throw new ArgumentNullException(nameof(queryCondition)); + if (queryTrue == null) throw new ArgumentNullException(nameof(queryTrue)); +#endif + EvalFunction = evalFunction; + QueryCondition = queryCondition; + QueryTrue = queryTrue; + QueryFalse = queryFalse; + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + double[][] results = new double[3][]; + results[0] = QueryCondition.ApplyTo(queryable); + if (EvalFunction(results[0])) + { + results[1] = new double[1] { 1.0 }; + results[2] = QueryTrue.ApplyTo(queryable); + } + else if (QueryFalse != null) + { + results[1] = new double[1] { 0.0 }; + results[2] = QueryFalse.ApplyTo(queryable); + } + else + { + results[1] = new double[1] { 0.0 }; + results[2] = new double[0]; + } + return results.SelectMany(item => item).ToArray(); + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + double[][] results = new double[3][]; + results[0] = QueryCondition.ApplyTo(queryable); + if (EvalFunction(results[0])) + { + results[1] = new double[1] { 1.0 }; + results[2] = QueryTrue.ApplyTo(queryable); + } + else if (QueryFalse != null) + { + results[1] = new double[1] { 0.0 }; + results[2] = QueryFalse.ApplyTo(queryable); + } + else + { + results[1] = new double[1] { 0.0 }; + results[2] = new double[0]; + } + return results.SelectMany(item => item).ToArray(); + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + double[][] results = new double[3][]; + results[0] = QueryCondition.ApplyTo(queryable); + if (EvalFunction(results[0])) + { + results[1] = new double[1] { 1.0 }; + results[2] = QueryTrue.ApplyTo(queryable); + } + else if (QueryFalse != null) + { + results[1] = new double[1] { 0.0 }; + results[2] = QueryFalse.ApplyTo(queryable); + } + else + { + results[1] = new double[1] { 0.0 }; + results[2] = new double[0]; + } + return results.SelectMany(item => item).ToArray(); + } + + public IQuery Clone() + { + return new Conditional(EvalFunction, QueryCondition, QueryTrue, QueryFalse); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/IPINQCondition.cs b/source/C#/UniTraX/Queries/Core/IPINQCondition.cs new file mode 100644 index 0000000000000000000000000000000000000000..882e56df3331ce683c19a66cc7b3924a9b0920d7 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/IPINQCondition.cs @@ -0,0 +1,13 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using System.Linq; + + public interface IPINQCondition + { + IQueryable[] ApplyTo(IQueryable queryable); + PINQueryable[] ApplyTo(PINQueryable queryable); + + IPINQCondition Clone(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/IQuery.cs b/source/C#/UniTraX/Queries/Core/IQuery.cs new file mode 100644 index 0000000000000000000000000000000000000000..ce8c269fe5ce577aa37a84f1ecd9b3ae276c02e4 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/IQuery.cs @@ -0,0 +1,14 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using System.Linq; + + public interface IQuery + { + double[] ApplyTo(IQueryable queryable); + double[] ApplyTo(PINQueryable queryable); + double[] ApplyTo(BKQueryable queryable); + IQuery Clone(); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/ISelectProvider.cs b/source/C#/UniTraX/Queries/Core/ISelectProvider.cs new file mode 100644 index 0000000000000000000000000000000000000000..702f45cac092fd8d66c046e601a06f313458fb8b --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/ISelectProvider.cs @@ -0,0 +1,10 @@ +namespace Qbb.Queries.Core +{ + using System; + using System.Linq.Expressions; + + public interface ISelectProvider + { + Expression> GetSelectorExpression(int[] selectIndexes); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/Merger.cs b/source/C#/UniTraX/Queries/Core/Merger.cs new file mode 100644 index 0000000000000000000000000000000000000000..6a71dde567d84ab8c12c6adb38cfac55bd9cbda5 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/Merger.cs @@ -0,0 +1,106 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using System; + using System.Linq; + + public class Merger : IQuery + { + private IQueryable Queryable; + private PINQueryable PINQueryable; + private BKQueryable BKQueryable; + + private Merger DerivedFrom; + + public Merger Derive() + { + var ret = new Merger + { + DerivedFrom = this + }; + return ret; + } + + public IQueryable GetQueryable() + { +#if DEBUG + if (DerivedFrom != null && Queryable != null) throw new ArgumentException("Cannot be both set."); +#endif + if (DerivedFrom != null) return DerivedFrom.GetQueryable(); + return Queryable; + } + + public PINQueryable GetPINQueryable() + { +#if DEBUG + if (DerivedFrom != null && PINQueryable != null) throw new ArgumentException("Cannot be both set."); +#endif + if (DerivedFrom != null) return DerivedFrom.GetPINQueryable(); + return PINQueryable; + } + + public BKQueryable GetBKQueryable() + { +#if DEBUG + if (DerivedFrom != null && BKQueryable != null) throw new ArgumentException("Cannot be both set."); +#endif + if (DerivedFrom != null) return DerivedFrom.GetBKQueryable(); + return BKQueryable; + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + Queryable = queryable; + return null; + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + PINQueryable = queryable; + return null; + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + BKQueryable = queryable; + return null; + } + + public IQuery Clone() + { + var ret = new Merger + { + Queryable = Queryable, + PINQueryable = PINQueryable, + BKQueryable = BKQueryable, + DerivedFrom = DerivedFrom + }; + return ret; + } + + public Merger Merge(Merger other) + { + var ret = new Merger(); + if (GetQueryable() == null) ret.Queryable = other.GetQueryable(); + else if (other.GetQueryable() != null) ret.Queryable = GetQueryable().Union(other.GetQueryable()); + else ret.Queryable = GetQueryable(); + if (GetPINQueryable() == null) ret.PINQueryable = other.GetPINQueryable(); + else if (other.GetPINQueryable() != null) ret.PINQueryable = GetPINQueryable().Union(other.GetPINQueryable()); + else ret.PINQueryable = GetPINQueryable(); + if (GetBKQueryable() == null) ret.BKQueryable = other.GetBKQueryable(); + else if (other.GetBKQueryable() != null) ret.BKQueryable = GetBKQueryable().Union(other.GetBKQueryable()); + else ret.BKQueryable = GetBKQueryable(); + return ret; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/Multi.cs b/source/C#/UniTraX/Queries/Core/Multi.cs new file mode 100644 index 0000000000000000000000000000000000000000..b85a77997d83b6aec55fa348dad0b7d02198d82a --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/Multi.cs @@ -0,0 +1,60 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Collections.Generic; + using System.Linq; + + public class Multi : IQuery + { + private readonly List> Queries; + + public Multi(List> queries) + { +#if DEBUG + if (queries == null) throw new ArgumentNullException(nameof(queries)); + if (!queries.Any()) throw new ArgumentException("no queries provided"); +#endif + Queries = queries; + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var results = new double[Queries.Count()][]; + for (int i = 0; i < Queries.Count(); i++) results[i] = Queries[i].ApplyTo(queryable); + return results.SelectMany(item => item).ToArray(); + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var results = new double[Queries.Count()][]; + for (int i = 0; i < Queries.Count(); i++) results[i] = Queries[i].ApplyTo(queryable); + return results.SelectMany(item => item).ToArray(); + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var results = new double[Queries.Count()][]; + for (int i = 0; i < Queries.Count(); i++) results[i] = Queries[i].ApplyTo(queryable); + return results.SelectMany(item => item).ToArray(); + } + + public IQuery Clone() + { + return new Multi(Queries); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/PINQDummy.cs b/source/C#/UniTraX/Queries/Core/PINQDummy.cs new file mode 100644 index 0000000000000000000000000000000000000000..b429b502eb76b7352d3a172b2eda300d6d631726 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/PINQDummy.cs @@ -0,0 +1,23 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using System.Linq; + + public class PINQDummy : IPINQCondition + { + public IQueryable[] ApplyTo(IQueryable queryable) + { + return new IQueryable[] { queryable }; + } + + public PINQueryable[] ApplyTo(PINQueryable queryable) + { + return new PINQueryable[] { queryable }; + } + + public IPINQCondition Clone() + { + return new PINQDummy(); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/Partitioning.cs b/source/C#/UniTraX/Queries/Core/Partitioning.cs new file mode 100644 index 0000000000000000000000000000000000000000..665a36e5677f0a42f615aa20a897dd022d1d8709 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/Partitioning.cs @@ -0,0 +1,445 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using Qbb.Core.Utils; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + + public class Part + { + public long Low { get; private set; } + public long High { get; private set; } + public int MapTo { get; private set; } + + public Part(long low, long high, int mapTo) + { + Low = low; + High = high; + MapTo = mapTo; + } + } + + public class Dimension + { + public Field Field { get; private set; } + public List Parts { get; private set; } + + public Dimension(Field field, List parts) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (parts == null) throw new ArgumentNullException(nameof(parts)); + if (!parts.Any()) throw new ArgumentException("no parts provided"); +#endif + Field = field; + Parts = parts; + } + + public Dimension Clone() + { + return new Dimension(Field, Parts); + } + } + + public class Partitioning : IQuery + { + public static List, Merger, int>> GeneratePartitionsStep(Queue> dimensions) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + Dimension dimension = dimensions.Dequeue(); + List, Merger, int>> partitions = null; + if (dimensions.Any()) partitions = GeneratePartitions(dimensions); + int resultCapacity = dimension.Parts.Count(); + if (partitions != null) resultCapacity *= partitions.Count(); + var result = new List, Merger, int>>(resultCapacity); + foreach (Part p in dimension.Parts) + { + if (partitions != null) foreach (Tuple, Merger, int> t in partitions) result.Add(new Tuple, Merger, int>( + new WhereRange(dimension.Field, p.Low, p.High, t.Item1), t.Item2.Derive(), t.Item3 + p.MapTo)); + else + { + var merger = new Merger(); + result.Add(new Tuple, Merger, int>( + new WhereRange(dimension.Field, p.Low, p.High, merger), merger, p.MapTo)); + } + } + return result; + } + + public static List, Merger, int>> GeneratePartitions(Queue> dimensions) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + var dimensionsTmp = new Queue>(); + foreach (Dimension d in dimensions) dimensionsTmp.Enqueue(d.Clone()); + return GeneratePartitionsStep(dimensionsTmp); + } + + public static List, Merger, int>> GenerateInversePartitionsStep(Queue> dimensions, int ignoreIndex) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + Dimension dimension = dimensions.Dequeue(); + List, Merger, int>> inversePartitions = null; + if (dimensions.Any()) inversePartitions = GenerateInversePartitionsStep(dimensions, ignoreIndex); + if (inversePartitions == null) inversePartitions = new List, Merger, int>>(); + if (dimension.Field.Index == ignoreIndex) return inversePartitions; + List inverseParts = InverseParts(dimension.Field, dimension.Parts); + foreach (Part p in inverseParts) + { + var merger = new Merger(); + inversePartitions.Add(new Tuple, Merger, int>(new WhereRange(dimension.Field, p.Low, p.High, merger), merger, p.MapTo)); + } + return inversePartitions; + } + + public static List, Merger, int>> GenerateInversePartitions(Queue> dimensions, int ignoreIndex) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + var dimensionsTmp = new Queue>(); + foreach (Dimension d in dimensions) dimensionsTmp.Enqueue(d.Clone()); + return GenerateInversePartitionsStep(dimensionsTmp, ignoreIndex); + } + + public static SortedDictionary>, List>>> MergePartitions(List, Merger, int>> partitions) + { +#if DEBUG + if (partitions == null) throw new ArgumentNullException(nameof(partitions)); + if (!partitions.Any()) throw new ArgumentException("partitions is empty"); +#endif + var result = new SortedDictionary>, List>>>(); + foreach (Tuple, Merger, int> p in partitions) + { + if (result.TryGetValue(p.Item3, out Tuple>, List>> t)) + { + t.Item1.Add(p.Item1); + t.Item2.Add(p.Item2); + } + else result.Add(p.Item3, new Tuple>, List>>(new List>() { p.Item1 }, new List>() { p.Item2 })); + } + return result; + } + + public static List InverseParts(Field field, List parts) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (parts == null) throw new ArgumentNullException(nameof(parts)); +#endif + var inverseParts = new List(); + long nextOfPreviousHigh = field.FieldType.Bounds.Low; + bool nophIsNaN = false; + foreach (Part p in parts) + { + if (nextOfPreviousHigh < p.Low) inverseParts.Add(new Part(nextOfPreviousHigh, p.Low - 1L, -1)); + if (p.High < field.FieldType.Bounds.High) nextOfPreviousHigh = p.High + 1L; + else + { + nextOfPreviousHigh = 0L; + nophIsNaN = true; + break; + } + } + if (!nophIsNaN) inverseParts.Add(new Part(nextOfPreviousHigh, field.FieldType.Bounds.High, -1)); + return inverseParts; + } + + public static List BothParts(Field field, List parts) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (parts == null) throw new ArgumentNullException(nameof(parts)); +#endif + var bothParts = new List(); + long nextOfPreviousHigh = field.FieldType.Bounds.Low; + bool nophIsNaN = false; + foreach (Part p in parts) + { + if (nextOfPreviousHigh < p.Low) bothParts.Add(new Part(nextOfPreviousHigh, p.Low - 1, -1)); + if (p.High < field.FieldType.Bounds.High) nextOfPreviousHigh = p.High + 1; + else + { + nextOfPreviousHigh = 0L; + nophIsNaN = true; + } + bothParts.Add(p); + if (nophIsNaN) break; + } + if (!nophIsNaN) bothParts.Add(new Part(nextOfPreviousHigh, field.FieldType.Bounds.High, -1)); + return bothParts; + } + + public static Expression GenerateKeyExpression(List parts, Expression accessBody, Func convert) + { +#if DEBUG + if (parts == null) throw new ArgumentNullException(nameof(parts)); + if (!parts.Any()) throw new ArgumentException("parts is empty"); + if (accessBody == null) throw new ArgumentNullException(nameof(accessBody)); + if (convert == null) throw new ArgumentNullException(nameof(convert)); +#endif + if (parts.Count == 1) return Expression.Constant(parts.First().MapTo); + var left = new List(); + var right = new List(); + var i = 0; + for (; i < parts.Count / 2; i++) left.Add(parts[i]); + for (; i < parts.Count; i++) right.Add(parts[i]); + return Expression.Condition(Expression.LessThan(accessBody, Expression.Constant(convert(right.First().Low, false))), GenerateKeyExpression(left, accessBody, convert), GenerateKeyExpression(right, accessBody, convert)); + } + + public static Expression> GenerateKeyExpression(Queue> dimensions, ParameterExpression parameter) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + Dimension dimension = dimensions.Dequeue(); + var currentParameter = parameter; + if (currentParameter == null) currentParameter = Expression.Parameter(typeof(R)); + var accessVisitor = new ReplaceExpressionVisitor(dimension.Field.Access.Parameters[0], currentParameter); + var accessBody = accessVisitor.Visit(dimension.Field.Access.Body); + var dimensionExpression = GenerateKeyExpression(BothParts(dimension.Field, dimension.Parts), accessBody, dimension.Field.FieldType.Convert); + Expression result = dimensionExpression; + if (dimensions.Any()) + { + Expression> lowerKeyExpression = GenerateKeyExpression(dimensions, currentParameter); + result = Expression.Condition(Expression.LessThan(dimensionExpression, Expression.Constant(0)), + dimensionExpression, + Expression.Condition(Expression.LessThan(lowerKeyExpression.Body, Expression.Constant(0)), + lowerKeyExpression.Body, + Expression.Add(dimensionExpression, lowerKeyExpression.Body))); + } + return Expression.Lambda>(result, currentParameter); + } + + public static Expression> GenerateKeyExpression(Queue> dimensions) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + var dimensionsTmp = new Queue>(); + foreach (Dimension d in dimensions) dimensionsTmp.Enqueue(d.Clone()); + return GenerateKeyExpression(dimensionsTmp, null); + } + + public static int[] GenerateKeys(Queue> dimensions) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + var keySet = new SortedSet() { -1 }; + var keyList = new List(); + foreach (Dimension d in dimensions) + { + var tmpList = keyList; + if (tmpList.Count > 0) + { + keyList = new List(d.Parts.Count() * tmpList.Count()); + foreach (Part p in d.Parts) foreach (int i in tmpList) keyList.Add(p.MapTo + i); + } + else + { + keyList = new List(d.Parts.Count()); + foreach (Part p in d.Parts) keyList.Add(p.MapTo); + } + } + foreach (int i in keyList) keySet.Add(i); + return keySet.ToArray(); + } + + public static SortedDictionary>, List>>> GenerateInversePartitionMap(Queue> dimensions) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + var dimensionsTmp = new Queue>(); + foreach (Dimension d in dimensions) dimensionsTmp.Enqueue(d.Clone()); + var inversePartitions = GenerateInversePartitions(dimensionsTmp, -1); + if (!inversePartitions.Any()) return null; + return MergePartitions(inversePartitions); + } + + public static SortedDictionary>, List>>> GeneratePartitionMap(Queue> dimensions) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); +#endif + var dimensionsTmp = new Queue>(); + foreach (Dimension d in dimensions) dimensionsTmp.Enqueue(d.Clone()); + return MergePartitions(GeneratePartitions(dimensionsTmp)); + } + + private readonly int[] Keys; + private readonly Expression> KeyExpression; + private readonly SortedDictionary>, List>>> PartitionMap; + private readonly SortedDictionary>, List>>> InversePartitionMap; + private readonly Queue> Dimensions; + private readonly IQuery ForEach; + private readonly IQuery ForOthers; + private readonly bool BudgetOverRuntime; + + public Partitioning(Queue> dimensions, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) + { +#if DEBUG + if (dimensions == null) throw new ArgumentNullException(nameof(dimensions)); + if (!dimensions.Any()) throw new ArgumentException("dimensions is empty"); + if (forEach == null) throw new ArgumentNullException(nameof(forEach)); +#endif + Keys = GenerateKeys(dimensions); + KeyExpression = GenerateKeyExpression(dimensions); + PartitionMap = GeneratePartitionMap(dimensions); + if (ForOthers != null) InversePartitionMap = GenerateInversePartitionMap(dimensions); + Dimensions = dimensions; + ForEach = forEach; + ForOthers = forOthers; + BudgetOverRuntime = budgetOverRuntime; + } + + public Partitioning(Queue> dimensions, IQuery forEach, IQuery forOthers, bool budgetOverRuntime, int[] keys, Expression> keyExpression) : + this(dimensions, forEach, forOthers, budgetOverRuntime) + { +#if DEBUG + if (keys == null) throw new ArgumentNullException(nameof(keys)); + if (!keys.Any()) throw new ArgumentException(); + if (keyExpression == null) throw new ArgumentNullException(nameof(keyExpression)); +#endif + Keys = keys; + KeyExpression = keyExpression; + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var queryableMap = new SortedDictionary>(); + foreach (KeyValuePair>, List>>> kv in PartitionMap) + { + foreach (WhereRange wr in kv.Value.Item1) wr.ApplyTo(queryable); + var merger = new Merger(); + foreach (Merger m in kv.Value.Item2) merger = merger.Merge(m); + var q = merger.GetQueryable(); + if (BudgetOverRuntime) q = q.ToArray().AsQueryable(); + queryableMap.Add(kv.Key, q); + } + + var results = new double[PartitionMap.Count()][]; + var count = 0; + foreach (KeyValuePair> kv in queryableMap) + { + var r = ForEach.Clone().ApplyTo(kv.Value); + results[count] = new double[r.Length + 1]; + results[count][0] = kv.Key; + for (int i = 1; i < r.Length + 1; i++) results[count][i] = r[i - 1]; + count++; + } + if (ForOthers != null && InversePartitionMap != null) foreach (KeyValuePair>, List>>> kv in InversePartitionMap) + ForOthers.Clone().ApplyTo(Enumerable.Empty().AsQueryable()); + return results.SelectMany(item => item).ToArray(); + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (BudgetOverRuntime) + { + var queryables = queryable.Partition(Keys, KeyExpression); + var results = new List(); + for (int i = 0; i < Keys.Count(); i++) + { + if (Keys[i] < 0) continue; + var r = ForEach.Clone().ApplyTo(queryables[Keys[i]]); + var res = new double[r.Count() + 1]; + res[0] = Keys[i]; + for (int j = 0; j < r.Count(); j++) res[j + 1] = r[j]; + results.Add(res); + } + if (ForOthers != null && InversePartitionMap != null) foreach (KeyValuePair>, List>>> kv in InversePartitionMap) + ForOthers.Clone().ApplyTo(Enumerable.Empty().AsQueryable()); + return results.SelectMany(item => item).ToArray(); + } + else + { + var results = new double[PartitionMap.Count()][]; + var count = 0; + foreach (KeyValuePair>, List>>> kv in PartitionMap) + { + foreach (WhereRange wr in kv.Value.Item1) wr.ApplyTo(queryable); + var merger = new Merger(); + foreach (Merger m in kv.Value.Item2) merger = merger.Merge(m); + var r = ForEach.Clone().ApplyTo(merger.GetPINQueryable()); + results[count] = new double[r.Length + 1]; + results[count][0] = kv.Key; + for (int i = 0; i < r.Length; i++) results[count][i + 1] = r[i]; + count++; + } + if (ForOthers != null && InversePartitionMap != null) foreach (KeyValuePair>, List>>> kv in InversePartitionMap) + ForOthers.Clone().ApplyTo(Enumerable.Empty().AsQueryable()); + return results.SelectMany(item => item).ToArray(); + } + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + var queryableMap = new SortedDictionary>(); + foreach (KeyValuePair>, List>>> kv in PartitionMap) + { + foreach (WhereRange wr in kv.Value.Item1) wr.ApplyTo(queryable); + var merger = new Merger(); + foreach (Merger m in kv.Value.Item2) merger = merger.Merge(m); + var q = merger.GetBKQueryable(); + if (BudgetOverRuntime) q = q.Materialize(); + queryableMap.Add(kv.Key, q); + } + + var results = new double[PartitionMap.Count()][]; + var count = 0; + foreach (KeyValuePair> kv in queryableMap) + { + var r = ForEach.Clone().ApplyTo(kv.Value); + results[count] = new double[r.Length + 1]; + results[count][0] = kv.Key; + for (int i = 1; i < r.Length + 1; i++) results[count][i] = r[i - 1]; + count++; + } + if (ForOthers != null && InversePartitionMap != null) + { + foreach (KeyValuePair>, List>>> kv in InversePartitionMap) + { + foreach (WhereRange r in kv.Value.Item1) r.ApplyTo(queryable); + var merger = new Merger(); + foreach (Merger m in kv.Value.Item2) merger = merger.Merge(m); + ForOthers.Clone().ApplyTo(merger.GetBKQueryable()); + } + } + return results.SelectMany(item => item).ToArray(); + } + + public IQuery Clone() + { + return new Partitioning(Dimensions, ForEach, ForOthers, BudgetOverRuntime, Keys, KeyExpression); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/Partitioning2DGeo.cs b/source/C#/UniTraX/Queries/Core/Partitioning2DGeo.cs new file mode 100644 index 0000000000000000000000000000000000000000..7200b546fabfc48181dd9400bea054aa357b9d72 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/Partitioning2DGeo.cs @@ -0,0 +1,154 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using Qbb.Core.Utils; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + + public class Partition2DGeo : IQuery + { + public static (int[], Expression>) KeysAndKeyExpression(Field fieldLat, long startLat, long intervalLat, int numberIntervalsLat, + Field fieldLong, long startLong, long intervalLong, int numberIntervalsLong) + { +#if DEBUG + if (fieldLat == null) throw new ArgumentNullException(nameof(fieldLat)); + if (intervalLat < 0) throw new ArgumentException("intervalLat must be positive"); + if (numberIntervalsLat < 1) throw new ArgumentException("numberIntervalsLat must be above zero"); + if (fieldLong == null) throw new ArgumentNullException(nameof(fieldLong)); + if (intervalLong < 0) throw new ArgumentException("intervalLong must be positive"); + if (numberIntervalsLong < 1) throw new ArgumentException("numberIntervalsLong must be above zero"); +#endif + var maxNumber = numberIntervalsLat; + if (numberIntervalsLong > maxNumber) maxNumber = numberIntervalsLong; + var oom = Math.Pow(10.0, Math.Ceiling(Math.Log10(maxNumber) + Double.Epsilon)); + int offset = Convert.ToInt32(oom); + + var keys = new List { -1 }; + for (int i = 0; i < numberIntervalsLat; i++) for (int j = 0; j < numberIntervalsLong; j++) keys.Add((offset + i) * offset * 10 + 2 * offset + j); + var currentParameter = Expression.Parameter(typeof(R)); + var accessVisitorA = new ReplaceExpressionVisitor(fieldLat.Access.Parameters[0], currentParameter); + var accessVisitorB = new ReplaceExpressionVisitor(fieldLong.Access.Parameters[0], currentParameter); + var accessBodyA = accessVisitorA.Visit(fieldLat.Access.Body); + var accessBodyB = accessVisitorB.Visit(fieldLong.Access.Body); + var startDA = fieldLat.FieldType.Convert(startLat, false); + var startDB = fieldLong.FieldType.Convert(startLong, false); + var endDA = fieldLat.FieldType.Convert(startLat + (intervalLat + 1L) * numberIntervalsLat - 1L, true); + var endDB = fieldLong.FieldType.Convert(startLong + (intervalLong + 1L) * numberIntervalsLong - 1L, true); + var singleStepA = fieldLat.FieldType.Convert(1L, true); + var singleStepB = fieldLong.FieldType.Convert(1L, true); + var subtractA = startDA - 0.5 * singleStepA; + var subtractB = startDB - 0.5 * singleStepB; + double intervalDA = fieldLat.FieldType.ConvertUnchecked(intervalLat + 1L, true); + double intervalDB = fieldLat.FieldType.ConvertUnchecked(intervalLong + 1L, true); + Expression currentExpression = Expression.Add( + Expression.Multiply(Expression.Constant(offset * 10), Expression.Add(Expression.Constant(offset), Expression.Convert( + Expression.Divide(Expression.Subtract(accessBodyA, Expression.Constant(subtractA)), Expression.Constant(intervalDA)), typeof(int)))), + Expression.Add(Expression.Constant(2 * offset), Expression.Convert( + Expression.Divide(Expression.Subtract(accessBodyB, Expression.Constant(subtractB)), Expression.Constant(intervalDB)), typeof(int)))); + currentExpression = Expression.Condition(Expression.GreaterThan(accessBodyA, Expression.Constant(endDA)), Expression.Constant(-1), currentExpression); + currentExpression = Expression.Condition(Expression.GreaterThan(accessBodyB, Expression.Constant(endDB)), Expression.Constant(-1), currentExpression); + currentExpression = Expression.Condition(Expression.LessThan(accessBodyA, Expression.Constant(startDA)), Expression.Constant(-1), currentExpression); + currentExpression = Expression.Condition(Expression.LessThan(accessBodyB, Expression.Constant(startDB)), Expression.Constant(-1), currentExpression); + return (keys.ToArray(), Expression.Lambda>(currentExpression, currentParameter)); + } + + private readonly Field FieldLat; + private readonly long StartLat; + private readonly long IntervalLat; + private readonly int NumberIntervalsLat; + private readonly Field FieldLong; + private readonly long StartLong; + private readonly long IntervalLong; + private readonly int NumberIntervalsLong; + private readonly IQuery ForEach; + private readonly IQuery ForOthers; + private readonly Partitioning Partitioning; + private readonly bool BudgetOverRuntime; + + public Partition2DGeo(Field fieldLat, long startLat, long intervalLat, int numberIntervalsLat, + Field fieldLong, long startLong, long intervalLong, int numberIntervalsLong, + IQuery forEach, IQuery forOthers, bool budgetOverRuntime) + { +#if DEBUG + if (fieldLat == null) throw new ArgumentNullException(nameof(fieldLat)); + if (intervalLat < 0) throw new ArgumentException("intervalLat below zero"); + if (numberIntervalsLat < 1) throw new ArgumentException("numberIntervalsLat below one"); + if (fieldLong == null) throw new ArgumentNullException(nameof(fieldLong)); + if (intervalLong < 0) throw new ArgumentException("intervalLong below zero"); + if (numberIntervalsLong < 1) throw new ArgumentException("numberIntervalsLong below one"); + if (forEach == null) throw new ArgumentNullException(nameof(forEach)); +#endif + FieldLat = fieldLat; + StartLat = startLat; + IntervalLat = intervalLat; + NumberIntervalsLat = numberIntervalsLat; + FieldLong = fieldLong; + StartLong = startLong; + IntervalLong = intervalLong; + NumberIntervalsLong = numberIntervalsLong; + ForEach = forEach; + ForOthers = forOthers; + BudgetOverRuntime = budgetOverRuntime; + + var maxNumber = NumberIntervalsLat; + if (NumberIntervalsLong > maxNumber) maxNumber = NumberIntervalsLong; + var oom = Math.Pow(10.0, Math.Ceiling(Math.Log10(maxNumber) + Double.Epsilon)); + int offset = Convert.ToInt32(oom); + + long startLatAmount = StartLat; + long intervalLatAmount = IntervalLat + 1L; + long startLongAmount = StartLong; + long intervalLongAmount = IntervalLong + 1L; + + var partsLat = new List(numberIntervalsLat); + for (int i = 0; i < numberIntervalsLat; i++) partsLat.Add(new Part(startLatAmount + intervalLatAmount * i, startLatAmount + intervalLatAmount * (i + 1) - 1L, (offset + i) * offset * 10)); + var partsLong = new List(numberIntervalsLong); + for (int i = 0; i < numberIntervalsLong; i++) partsLong.Add(new Part(startLongAmount + intervalLongAmount * i, startLongAmount + intervalLongAmount * (i + 1) - 1L, 2 * offset + i)); + + var dimensions = new Queue>(); + dimensions.Enqueue(new Dimension(FieldLong, partsLong)); + dimensions.Enqueue(new Dimension(FieldLat, partsLat)); + var keysAndKeyExpression = KeysAndKeyExpression(fieldLat, startLat, intervalLat, numberIntervalsLat, fieldLong, startLong, intervalLong, numberIntervalsLong); + Partitioning = new Partitioning(dimensions, ForEach, ForOthers, BudgetOverRuntime, keysAndKeyExpression.Item1, keysAndKeyExpression.Item2); + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (FieldLat == null) throw new ArgumentNullException(nameof(FieldLat)); + if (FieldLong == null) throw new ArgumentNullException(nameof(FieldLong)); + return Partitioning.ApplyTo(queryable); + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (FieldLat == null) throw new ArgumentNullException(nameof(FieldLat)); + if (FieldLong == null) throw new ArgumentNullException(nameof(FieldLong)); + return Partitioning.ApplyTo(queryable); + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (FieldLat == null) throw new ArgumentNullException(nameof(FieldLat)); + if (FieldLong == null) throw new ArgumentNullException(nameof(FieldLong)); + return Partitioning.ApplyTo(queryable); + } + + public IQuery Clone() + { + return new Partition2DGeo(FieldLat, StartLat, IntervalLat, NumberIntervalsLat, FieldLong, StartLong, IntervalLong, NumberIntervalsLong, ForEach, ForOthers, BudgetOverRuntime); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/PerNumbers.cs b/source/C#/UniTraX/Queries/Core/PerNumbers.cs new file mode 100644 index 0000000000000000000000000000000000000000..306ba3d130396f43da5461755c106513695c2218 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/PerNumbers.cs @@ -0,0 +1,146 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using Qbb.Core.Utils; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + + public class PerNumbers : IQuery + { + public static (int[], Expression>) KeysAndKeyExpression(Field field, long start, long interval, int numberIntervals, int mapOffset) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (interval < 0) throw new ArgumentException("interval must be positive"); + if (numberIntervals < 1) throw new ArgumentException("numberIntervals must be above zero"); +#endif + var keys = new List { -1 }; + for (int i = 0; i < numberIntervals; i++) keys.Add(i + mapOffset); + var currentParameter = Expression.Parameter(typeof(R)); + var accessVisitor = new ReplaceExpressionVisitor(field.Access.Parameters[0], currentParameter); + var accessBody = accessVisitor.Visit(field.Access.Body); + var startD = field.FieldType.Convert(start, false); + var endD = field.FieldType.Convert(start + (interval + 1L) * numberIntervals - 1L, true); + var singleStep = field.FieldType.Convert(1L, true); + var subtract = startD - 0.5 * singleStep; + double intervalD = field.FieldType.ConvertUnchecked(interval + 1L, true); + Expression currentExpression = null; + currentExpression = Expression.Convert(Expression.Divide(Expression.Subtract(accessBody, Expression.Constant(subtract)), Expression.Constant(intervalD)), typeof(int)); + if (mapOffset != 0) currentExpression = Expression.Add(currentExpression, Expression.Constant(mapOffset)); + currentExpression = Expression.Condition(Expression.GreaterThan(accessBody, Expression.Constant(endD)), Expression.Constant(-1), currentExpression); + currentExpression = Expression.Condition(Expression.LessThan(accessBody, Expression.Constant(startD)), Expression.Constant(-1), currentExpression); + return (keys.ToArray(), Expression.Lambda>(currentExpression, currentParameter)); + } + + private readonly Field Field; + private readonly List Parts; + private readonly IQuery ForEach; + private readonly IQuery ForOthers; + private readonly Partitioning Partitioning; + private readonly bool BudgetOverRuntime; + + private PerNumbers(Field field, IQuery forEach, IQuery forOthers) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (forEach == null) throw new ArgumentNullException(nameof(forEach)); +#endif + Field = field; + ForEach = forEach; + ForOthers = forOthers; + } + + public PerNumbers(Field field, List parts, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : this(field, forEach, forOthers) + { +#if DEBUG + if (parts == null) throw new ArgumentNullException(nameof(parts)); + foreach (Part part in parts) + { + if (field != null && !field.FieldType.Bounds.Encloses(part.Low)) throw new ArgumentException("part low not valid in field bounds"); + if (field != null && !field.FieldType.Bounds.Encloses(part.High)) throw new ArgumentException("part high not valid in field bounds"); + if (part.MapTo < 0) throw new ArgumentException("mapTo below zero"); + } +#endif + Parts = parts; + var dimensions = new Queue>(); + dimensions.Enqueue(new Dimension(Field, parts)); + BudgetOverRuntime = budgetOverRuntime; + Partitioning = new Partitioning(dimensions, ForEach, ForOthers, BudgetOverRuntime); + } + + public PerNumbers(Field field, long start, long interval, int numberIntervals, int mapOffset, IQuery forEach, IQuery forOthers, bool budgetOverRuntime) : + this(field, forEach, forOthers) + { +#if DEBUG + if (interval < 0) throw new ArgumentException("interval below zero"); + if (numberIntervals < 1) throw new ArgumentException("numberIntervals below one"); + if (mapOffset < 0) throw new ArgumentException("mapOffset below zero"); + if (!field.FieldType.Bounds.Encloses(start)) throw new ArgumentException("start not valied in field step"); + var value = start; + for (int i = 0; i < numberIntervals; i++) + { + value += interval; + if (!field.FieldType.Bounds.Encloses(value)) throw new ArgumentException("value is not valid in fieldB step at interval " + i); + if (i < numberIntervals - 1) + { + if (!field.FieldType.Bounds.Encloses(value + 1)) throw new ArgumentException("value has no next at interval " + i); + value = value + 1; + } + } +#endif + var parts = new List(numberIntervals); + var begin = start; + for (int i = 0; i < numberIntervals; i++) + { + var end = begin + interval; + parts.Add(new Part(begin, end, mapOffset + i)); + if (i < numberIntervals - 1) begin = end + 1; + } + Parts = parts; + var dimensions = new Queue>(); + dimensions.Enqueue(new Dimension(Field, parts)); + BudgetOverRuntime = budgetOverRuntime; + + var keysAndKeyExpression = KeysAndKeyExpression(field, start, interval, numberIntervals, mapOffset); + Partitioning = new Partitioning(dimensions, ForEach, ForOthers, BudgetOverRuntime, keysAndKeyExpression.Item1, keysAndKeyExpression.Item2); +// More expensive +// Partitioning = new Partitioning(dimensions, ForEach, ForOthers, BudgetOverRuntime); + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (Field == null) throw new ArgumentNullException(nameof(Field)); + return Partitioning.ApplyTo(queryable); + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (Field == null) throw new ArgumentNullException(nameof(Field)); + return Partitioning.ApplyTo(queryable); + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (Field == null) throw new ArgumentNullException(nameof(Field)); + return Partitioning.ApplyTo(queryable); + } + + public IQuery Clone() + { + return new PerNumbers(Field, Parts, ForEach, ForOthers, BudgetOverRuntime); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/Select.cs b/source/C#/UniTraX/Queries/Core/Select.cs new file mode 100644 index 0000000000000000000000000000000000000000..e082fb3ee07fbdf2798eb99d421ccd9a98ae80c6 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/Select.cs @@ -0,0 +1,44 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using System; + using System.Linq; + using System.Linq.Expressions; + + public class Select : IQuery + { + private readonly Expression> SelectorExpression; + private readonly IQuery Afterwards; + + public Select(Expression> selectorExpression, IQuery afterwards) + { +#if DEBUG + if (selectorExpression == null) throw new ArgumentNullException(nameof(selectorExpression)); + if (afterwards == null) throw new ArgumentNullException(nameof(afterwards)); +#endif + SelectorExpression = selectorExpression; + Afterwards = afterwards; + } + + public double[] ApplyTo(IQueryable queryable) + { + return Afterwards.ApplyTo(queryable.Select(SelectorExpression)); + } + + public double[] ApplyTo(PINQueryable queryable) + { + return Afterwards.ApplyTo(queryable.Select(SelectorExpression)); + } + + public double[] ApplyTo(BKQueryable queryable) + { + throw new InvalidOperationException("Bookkeeper does not support selects"); + } + + public IQuery Clone() + { + return new Select(SelectorExpression, Afterwards); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/WhereBudget.cs b/source/C#/UniTraX/Queries/Core/WhereBudget.cs new file mode 100644 index 0000000000000000000000000000000000000000..6a3d3bc1b6aaa5638c5e3b3c0abb52dc12fb05e5 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/WhereBudget.cs @@ -0,0 +1,53 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Specification; + using System; + using System.Linq; + + public class WhereBudget : IQuery + { + private readonly long BudgetAvailable; + private readonly Field FieldBudget; + private readonly IQuery Afterwards; + + public WhereBudget(long budgetAvailable, Field fieldBudget, IQuery afterwards) + { +#if DEBUG + if (budgetAvailable < 0) throw new ArgumentException("budgetAvailable is not valid"); + if (afterwards == null) throw new ArgumentNullException(nameof(afterwards)); +#endif + BudgetAvailable = budgetAvailable; + FieldBudget = fieldBudget; + Afterwards = afterwards; + } + + public double[] ApplyTo(IQueryable queryable) + { + return Afterwards.ApplyTo(queryable); + } + + public double[] ApplyTo(PINQueryable queryable) + { + return Afterwards.ApplyTo(queryable); + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (FieldBudget == null) throw new ArgumentNullException(nameof(FieldBudget)); +#endif + var maxUsage = queryable.MaxUsage(); + var budgetAvailable = BudgetAvailable; + var budgetLow = maxUsage + budgetAvailable; + var range = new WhereRange(FieldBudget, budgetLow, FieldBudget.FieldType.Bounds.High, Afterwards); + return range.ApplyTo(queryable); + } + + public IQuery Clone() + { + return new WhereBudget(BudgetAvailable, FieldBudget, Afterwards); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Core/WhereRange.cs b/source/C#/UniTraX/Queries/Core/WhereRange.cs new file mode 100644 index 0000000000000000000000000000000000000000..53e3639b5939c6e2dcbbcd756f116dffa261e329 --- /dev/null +++ b/source/C#/UniTraX/Queries/Core/WhereRange.cs @@ -0,0 +1,91 @@ +namespace Qbb.Queries.Core +{ + using PINQ; + using Qbb.Core.Bookkeeper; + using Qbb.Core.Partition; + using Qbb.Core.Specification; +#if RECORD + using Qbb.Queries.Recording; +#endif + using System; + using System.Linq; + using System.Linq.Expressions; + + public class WhereRange : IQuery + { + public static Expression> GenerateExpression(Field field, long low, long high) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); +#endif + if (low <= field.FieldType.Bounds.Low && high >= field.FieldType.Bounds.High) return null; + Expression> expHigh = Expression.Lambda>(Expression.LessThanOrEqual(field.Access.Body, Expression.Constant(field.FieldType.Convert(high, true))), field.Access.Parameters); + if (low <= field.FieldType.Bounds.Low) return expHigh; + Expression> expLow = Expression.Lambda>(Expression.GreaterThanOrEqual(field.Access.Body, Expression.Constant(field.FieldType.Convert(low, false))), field.Access.Parameters); + if (high >= field.FieldType.Bounds.High) return expLow; + return Expression.Lambda>(Expression.And(expLow.Body, expHigh.Body), field.Access.Parameters); + } + + private readonly Field Field; + private readonly long Low; + private readonly long High; + private readonly IQuery Afterwards; + + public WhereRange(Field field, long low, long high, IQuery afterwards) + { +#if DEBUG + if (field == null) throw new ArgumentNullException(nameof(field)); + if (high < low) throw new ArgumentException("high below low"); + if (afterwards == null) throw new ArgumentNullException(nameof(afterwards)); +#endif + Field = field; + Low = low; + High = high; + Afterwards = afterwards; + } + + public double[] ApplyTo(IQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (Field == null) throw new ArgumentNullException(nameof(Field)); + var newQueryable = queryable.Where(GenerateExpression(Field, Low, High)); +#if RECORD + Recorder.Add(new EntryRange(Field.Name, Low, High)); +#endif + return Afterwards.ApplyTo(newQueryable); + } + + public double[] ApplyTo(PINQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (Field == null) throw new ArgumentNullException(nameof(Field)); + var newQueryable = queryable.Where(GenerateExpression(Field, Low, High)); +#if RECORD + Recorder.Add(new EntryRange(Field.Name, Low, High)); +#endif + return Afterwards.ApplyTo(newQueryable); + } + + public double[] ApplyTo(BKQueryable queryable) + { +#if DEBUG + if (queryable == null) throw new ArgumentNullException(nameof(queryable)); +#endif + if (Field == null) throw new ArgumentNullException(nameof(Field)); + var newQueryable = queryable.Apply(x => Partition.LessEqual(Partition.GreaterEqual(x, new IndexedValue(Field.Index, Low), Field.FieldType.Bounds), new IndexedValue(Field.Index, High))); +#if RECORD + Recorder.Add(new EntryRange(Field.Name, Low, High)); +#endif + return Afterwards.ApplyTo(newQueryable); + } + + public IQuery Clone() + { + return new WhereRange(Field, Low, High, Afterwards); + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Recorder/EntryAggregate.cs b/source/C#/UniTraX/Queries/Recorder/EntryAggregate.cs new file mode 100644 index 0000000000000000000000000000000000000000..ed7af88b6ab0be0e59faf90795b1751a879a1a93 --- /dev/null +++ b/source/C#/UniTraX/Queries/Recorder/EntryAggregate.cs @@ -0,0 +1,42 @@ +namespace Qbb.Queries.Recording +{ + using System; + + public class EntryAggregate : EntryTimed + { + public string Name { get; private set; } + public string Message { get; private set; } + public long Epsilon { get; private set; } + public double Aggregate { get; private set; } + public int CleanThreshold { get; private set; } + + public EntryAggregate(string name, string message, long epsilon, double aggregate, double setupTime, double executeTime, double bkExecuteTime, double teardownTime, int cleanThreshold) : base(setupTime, executeTime, bkExecuteTime, teardownTime) + { +#if DEBUG + if (name == null) throw new ArgumentNullException(nameof(name)); + if (message == null) throw new ArgumentNullException(nameof(message)); +#endif + Name = name; + Message = message; + Epsilon = epsilon; + Aggregate = aggregate; + CleanThreshold = cleanThreshold; + } + + public override bool SameAs(object comparand) + { + if (comparand == null) return false; + if (!GetType().Equals(comparand.GetType())) return false; + EntryAggregate c = (EntryAggregate)comparand; + if (!Name.Equals(c.Name)) return false; + if (!Message.Equals(c.Message)) return false; + if (CleanThreshold != c.CleanThreshold) return false; + return true; + } + + public override string ToString() + { + return "Aggregate;" + Name + ";" + Message + ";" + Epsilon + ";" + Aggregate + ";" + base.ToString() + ";" + CleanThreshold; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Recorder/EntryAnalytic.cs b/source/C#/UniTraX/Queries/Recorder/EntryAnalytic.cs new file mode 100644 index 0000000000000000000000000000000000000000..14d08dfd2f9143921040cf99f60986de31834428 --- /dev/null +++ b/source/C#/UniTraX/Queries/Recorder/EntryAnalytic.cs @@ -0,0 +1,34 @@ +namespace Qbb.Queries.Recording +{ + using System; + + public class EntryAnalytic : IEntry + { + public long Id { get; private set; } + public double Time { get; private set; } + public string Message { get; private set; } + + public EntryAnalytic(long id, double time, string message) + { +#if DEBUG + if (time < 0) throw new ArgumentException("time is not valid"); + if (message == null) throw new ArgumentNullException(nameof(message)); +#endif + Id = id; + Time = time; + Message = message; + } + + public bool SameAs(object comparand) + { + if (comparand == null) return false; + if (!GetType().Equals(comparand.GetType())) return false; + return true; + } + + public override string ToString() + { + return "Analytic;" + Id + ";" + Time + ";" + Message; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Recorder/EntryNumberPartitions.cs b/source/C#/UniTraX/Queries/Recorder/EntryNumberPartitions.cs new file mode 100644 index 0000000000000000000000000000000000000000..df8f3ff93fcfe1856259803581bc9598eb4bbd4e --- /dev/null +++ b/source/C#/UniTraX/Queries/Recorder/EntryNumberPartitions.cs @@ -0,0 +1,39 @@ +namespace Qbb.Queries.Recording +{ + using System; + + public class EntryNumberPartitions : IEntry + { + private readonly int SetupNumberPartitions; + private readonly int ExecuteNumberPartitions; + private readonly int TeardownNumberPartitions; + + public EntryNumberPartitions(int setupNumberPartitions, int executeNumberPartitions, int teardownNumberPartitions) + { +#if DEBUG + if (setupNumberPartitions < 0) throw new ArgumentException("setupNumberPartitions is not valid"); + if (executeNumberPartitions < 0) throw new ArgumentException("executeNumberPartitions is not valid"); + if (teardownNumberPartitions < 0) throw new ArgumentException("teardownNumberPartitions is not valid"); +#endif + SetupNumberPartitions = setupNumberPartitions; + ExecuteNumberPartitions = executeNumberPartitions; + TeardownNumberPartitions = teardownNumberPartitions; + } + + public bool SameAs(object comparand) + { + if (comparand == null) return false; + if (!GetType().Equals(comparand.GetType())) return false; + EntryNumberPartitions c = (EntryNumberPartitions)comparand; + if (!SetupNumberPartitions.Equals(c.SetupNumberPartitions)) return false; + if (!ExecuteNumberPartitions.Equals(c.ExecuteNumberPartitions)) return false; + if (!TeardownNumberPartitions.Equals(c.TeardownNumberPartitions)) return false; + return true; + } + + public override string ToString() + { + return "NumberPartitions;" + SetupNumberPartitions + ";" + ExecuteNumberPartitions + ";" + TeardownNumberPartitions; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Recorder/EntryRange.cs b/source/C#/UniTraX/Queries/Recorder/EntryRange.cs new file mode 100644 index 0000000000000000000000000000000000000000..9ccf21d83547d0836b4a99a01bc625f943cd4fbb --- /dev/null +++ b/source/C#/UniTraX/Queries/Recorder/EntryRange.cs @@ -0,0 +1,38 @@ +namespace Qbb.Queries.Recording +{ + using System; + + public class EntryRange : IEntry + { + public string ColumnName { get; private set; } + public long Low { get; private set; } + public long High { get; private set; } + + public EntryRange(string columnName, long low, long high) + { +#if DEBUG + if (columnName == null) throw new ArgumentNullException(nameof(columnName)); + if (high < low) throw new ArgumentException("high below low"); +#endif + ColumnName = columnName; + Low = low; + High = high; + } + + public bool SameAs(object comparand) + { + if (comparand == null) return false; + if (!GetType().Equals(comparand.GetType())) return false; + EntryRange c = (EntryRange)comparand; + if (!ColumnName.Equals(c.ColumnName)) return false; + if (!Low.Equals(c.Low)) return false; + if (!High.Equals(c.High)) return false; + return true; + } + + public override string ToString() + { + return "Range;" + ColumnName + ";" + Low + ";" + High; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Recorder/EntryTimed.cs b/source/C#/UniTraX/Queries/Recorder/EntryTimed.cs new file mode 100644 index 0000000000000000000000000000000000000000..2a0f53b192fe4b0f000ffd9c4c2bac1e9d903bae --- /dev/null +++ b/source/C#/UniTraX/Queries/Recorder/EntryTimed.cs @@ -0,0 +1,38 @@ +namespace Qbb.Queries.Recording +{ + using System; + + public class EntryTimed : IEntry + { + public double SetupTime { get; private set; } + public double ExecuteTime { get; private set; } + public double BKExecuteTime { get; private set; } + public double TeardownTime { get; private set; } + + public EntryTimed(double setupTime, double executeTime, double bkExecuteTime, double teardownTime) + { +#if DEBUG + if (setupTime < 0) throw new ArgumentException("setupTime is not valid"); + if (executeTime < 0) throw new ArgumentException("executeTime is not valid"); + if (bkExecuteTime < 0) throw new ArgumentException("bkExecuteTime is not valid"); + if (teardownTime < 0) throw new ArgumentException("teardownTime is not valid"); +#endif + SetupTime = setupTime; + ExecuteTime = executeTime; + BKExecuteTime = bkExecuteTime; + TeardownTime = teardownTime; + } + + public virtual bool SameAs(object comparand) + { + if (comparand == null) return false; + if (!GetType().Equals(comparand.GetType())) return false; + return true; + } + + public override string ToString() + { + return SetupTime + ";" + ExecuteTime + ";" + BKExecuteTime + ";" + TeardownTime; + } + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Recorder/IEntry.cs b/source/C#/UniTraX/Queries/Recorder/IEntry.cs new file mode 100644 index 0000000000000000000000000000000000000000..85f7eee22252990bab607361aac44e3fa41470ec --- /dev/null +++ b/source/C#/UniTraX/Queries/Recorder/IEntry.cs @@ -0,0 +1,7 @@ +namespace Qbb.Queries.Recording +{ + public interface IEntry + { + bool SameAs(object comparand); + } +} \ No newline at end of file diff --git a/source/C#/UniTraX/Queries/Recorder/Recorder.cs b/source/C#/UniTraX/Queries/Recorder/Recorder.cs new file mode 100644 index 0000000000000000000000000000000000000000..6630e664130a55e57be9c2432b5b498eb4899eb5 --- /dev/null +++ b/source/C#/UniTraX/Queries/Recorder/Recorder.cs @@ -0,0 +1,103 @@ +namespace Qbb.Queries.Recording +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + + public class Recorder + { +#if RECORD + public static List Entries = new List(); + private static Stopwatch GlobalTimer = null; + + public static void Add(IEntry entry) + { + Entries.Add(entry); + } + + public static void PrintStatus() + { + Console.WriteLine("Recording is ACTIVE"); + } + + public static void StartGlobalTimer() + { +#if DEBUG + if (GlobalTimer != null) throw new InvalidOperationException("Global timer already running"); +#endif + GlobalTimer = StartTimer(); + } + + public static double StopGlobalTimer() + { +#if DEBUG + if (GlobalTimer == null) throw new InvalidOperationException("Global timer not running"); +#endif + var time = StopTimer(GlobalTimer); + GlobalTimer = null; + return time; + } + + public static Stopwatch StartTimer() + { + Stopwatch timer = new Stopwatch(); + timer.Start(); + return timer; + } + + public static double StopTimer(Stopwatch timer) + { + timer.Stop(); + return timer.Elapsed.TotalMilliseconds; + } + + public static void Reset() + { + Entries = new List(); + } + + public static void ToFile(string filename) + { + ToFile(filename, X => true); + } + + public static void ToFile(string filename, Func filter) + { + Console.Write("Reporting to file ... "); + using (var file = new System.IO.StreamWriter(filename)) + { + for (var i = 0; i < Entries.Count; i++) + { + if (filter(Entries[i])) file.WriteLine(Entries[i].ToString()); + } + } + Console.WriteLine("Done!"); + } +#else + public static List Entries; + + public static void Add(IEntry entry) {} + + public static void PrintStatus() + { + Console.WriteLine("Recording is INACTIVE"); + } + + public static Stopwatch StartTimer() + { + return null; + } + + public static double StopTimer(Stopwatch timer) + { + return -1.0; + } + + public static void Reset() {} + + public static void ToFile(string filename) {} + + public static void ToFile(string filename, Func filter) {} +#endif + } +} \ No newline at end of file diff --git a/source/java/logConverter/pom.xml b/source/java/logConverter/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..89e24cb0963402c9c9e14ece99b0dba83f23e420 --- /dev/null +++ b/source/java/logConverter/pom.xml @@ -0,0 +1,26 @@ + + 4.0.0 + unitrax + logConverter + 0.0.1-SNAPSHOT + + src + + + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + + + + org.apache.commons + commons-math3 + 3.6.1 + + + \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/Data.java b/source/java/logConverter/src/logConverter/Data.java new file mode 100644 index 0000000000000000000000000000000000000000..783e1cebab00a28b1917b94cfe5b271ea8584b8e --- /dev/null +++ b/source/java/logConverter/src/logConverter/Data.java @@ -0,0 +1,336 @@ +package logConverter; + +import java.security.InvalidParameterException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; + +import org.apache.commons.math3.util.Pair; + +public class Data { + + public static List generateRowsDouble(List> columns) { + if (columns == null) throw new NullPointerException("columns is null"); + Integer size = null; + for (List column : columns) { + if (column == null) continue; + if (column.isEmpty()) throw new IllegalArgumentException("column is empty"); + if (size != null && !size.equals(column.size())) throw new IllegalArgumentException("columns are not same size"); + size = column.size(); + } + List rows = new LinkedList<>(); + for (int i = 0; i < size; i++) { + double[] row = new double[columns.size()]; + for (int j = 0; j < columns.size(); j++) { + List list = columns.get(j); + if (list == null) row[j] = Double.NaN; + else row[j] = list.get(i); + } + rows.add(row); + } + return rows; + } + + public static List generateRowsValue(List> columns, boolean ste) { + if (columns == null) throw new NullPointerException("columns is null"); + Integer size = null; + for (int i = 0; i < columns.size(); i++) { + List column = columns.get(i); + if (column == null) continue; + if (column.isEmpty()) throw new IllegalArgumentException("column is empty"); + if (size != null && !size.equals(column.size())) { + throw new IllegalArgumentException("columns are not same size, was " + column.size() + " should be " + size); + } + size = column.size(); + } + int rowSize = columns.size(); + if (ste) rowSize *= 2; + List rows = new LinkedList<>(); + for (int i = 0; i < size; i++) { + double[] row = new double[rowSize]; + for (int j = 0; j < columns.size(); j++) { + List list = columns.get(j); + if (ste) { + if (list == null) { + row[j * 2] = Double.NaN; + row[j * 2 + 1] = Double.NaN; + } else { + row[j * 2] = list.get(i).getValue(); + row[j * 2 + 1] = list.get(i).getStE(); + } + } else { + if (list == null) row[j] = Double.NaN; + else row[j] = list.get(i).getValue(); + } + } + rows.add(row); + } + return rows; + } + + public static int[] getSortFilter(List input, int sortIndex) { + List> list = new ArrayList>(input.size()); + for (int i = 0; i < input.size(); i++) list.add(new Pair(input.get(i)[sortIndex], i)); + Collections.sort(list, new Comparator>() { + @Override + public int compare(Pair o1, Pair o2) { + return Double.compare(o1.getFirst(), o2.getFirst()); + } + }); + int[] result = new int[list.size()]; + for (int i = 0; i < list.size(); i++) result[i] = list.get(i).getSecond(); + return result; + } + + public static int[] generatePercent(int totalNumber, double rate) { + Random r = new Random(); + List list = new LinkedList(); + for (int i = 0; i < totalNumber; i++) if (r.nextDouble() < rate) list.add(i); + int[] result = new int[list.size()]; + for (int i = 0; i < list.size(); i++) result[i] = list.get(i); + return result; + } + + public static List filterBy(List input, int[] filterBy) { + List result = new ArrayList(filterBy.length); + for (int i = 0; i < filterBy.length; i++) { + if (filterBy[i] > input.size() - 1) break; + result.add(input.get(filterBy[i])); + } + return result; + } + + public static List valueDifferences(Log compareTo, Log[] logs) { + List> list = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) list.add(Log.valueDifferences(compareTo, logs[i])); + return generateRowsDouble(list); + } + + public static List executeTimeDifferencesPercent(Log compareTo, Log[] logs) { + List> list = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) list.add(Log.executeTimeDifferencesPercent(compareTo, logs[i])); + return generateRowsDouble(list); + } + + public static List cdf(List input, double start, double end, int numIntervals, boolean log, int logSteps) { + int numSystems = input.get(0).length; + double interval = (end - start) / new Double(numIntervals); + double totalCount = 0.0; + double[][] counts = new double[numSystems][]; + for (int i = 0; i < input.size(); i++) { + totalCount++; + for (int j = 0; j < numSystems; j++) { + int index = -1; + double value = input.get(i)[j]; + if (Double.isNaN(value)) continue; + if (value < start) index = 0; + else if (value >= end) index = numIntervals + 1; + else index = new Double((value - start) / interval).intValue() + 1; + if (counts[j] == null) counts[j] = new double[numIntervals + 2]; + counts[j][index]++; + } + } + + double[][][] valuedCounts = new double[numSystems][][]; + for (int i = 0; i < numSystems; i++) { + if (counts[i] == null) continue; + valuedCounts[i] = new double[counts[i].length][]; + for (int j = 0; j < counts[i].length; j++) { + valuedCounts[i][j] = new double[2]; + if (j == 0) valuedCounts[i][j][0] = start; + else if (j == counts[i].length - 1) valuedCounts[i][j][0] = end; + else valuedCounts[i][j][0] = start + (new Double(j + 1) - 0.5) * interval; + valuedCounts[i][j][1] = counts[i][j]; + } + } + + int count = numIntervals + 2; + if (log) { + count = 0; + int mult = 1; + int intervals = numIntervals; + while (intervals > 0) { + for (int i = 0; i < logSteps; i++) { + if (intervals <= 0) break; + count++; + intervals -= mult; + } + mult *= 1000; + } + count += 2; + double[][][] origCounts = valuedCounts; + valuedCounts = new double[origCounts.length][][]; + for (int i = 0; i < origCounts.length; i++) { + if (origCounts[i] == null) continue; + valuedCounts[i] = new double[count][]; + valuedCounts[i][0] = origCounts[i][0]; + valuedCounts[i][valuedCounts[i].length - 1] = origCounts[i][origCounts[i].length - 1]; + intervals = numIntervals; + mult = 1; + int indexNew = 1; + int indexOld = 1; + while (intervals > 0) { + for (int j = 0; j < logSteps; j++) { + if (intervals <= 0) break; + double value = 0.0; + int c = 0; + double num = 0.0; + for (int k = 0; k < mult; k++) { + if (intervals <= 0) break; + value += origCounts[i][indexOld][0]; + c += origCounts[i][indexOld][1]; + num++; + indexOld++; + intervals--; + } + valuedCounts[i][indexNew] = new double[2]; + valuedCounts[i][indexNew][0] = value / num; + valuedCounts[i][indexNew][1] = c; + indexNew++; + } + mult *= 1000; + } + } + } + + List result = new ArrayList(count); + double[] runningCounts = new double[numSystems]; + for (int i = 0; i < count; i++) { + double value = Double.NaN; + for (int j = 0; j < numSystems; j++) { + if (valuedCounts[j] == null) { + runningCounts[j] = Double.NaN; + continue; + } + if (Double.isNaN(value)) value = valuedCounts[j][i][0]; + else if (value != valuedCounts[j][i][0]) { + throw new InvalidParameterException("values should match " + value + " " + valuedCounts[j][i][0]); + } + runningCounts[j] += valuedCounts[j][i][1]; + } + double[] res = new double[numSystems + 1]; + for (int j = 0; j < numSystems; j++) { + if (Double.isNaN(runningCounts[j])) res[j + 1] = Double.NaN; + else res[j + 1] = runningCounts[j] / totalCount; + } + res[0] = value; + result.add(res); + } + return result; + } + + public static List relativeNumbers(List input, int relativeIndex) { + List result = new ArrayList(input.size()); + Iterator iterator = input.iterator(); + while (iterator.hasNext()) { + double[] row = iterator.next(); + if (Double.isNaN(row[relativeIndex])) throw new InvalidParameterException("relative values cannot be NaN"); + double[] res = new double[row.length]; + for (int i = 0; i < row.length; i++) { + if (Double.isNaN(row[i])) res[i] = Double.NaN; + else res[i] = row[i] / row[relativeIndex]; + } + result.add(res); + } + return result; + } + + public static List differenceNumbersAbs(List input, int differenceIndex) { + List result = new ArrayList(input.size()); + Iterator iterator = input.iterator(); + while (iterator.hasNext()) { + double[] row = iterator.next(); + if (Double.isNaN(row[differenceIndex])) throw new InvalidParameterException("difference values cannot be NaN"); + double[] res = new double[row.length]; + for (int i = 0; i < row.length; i++) { + if (Double.isNaN(row[i])) res[i] = Double.NaN; + else res[i] = Math.abs(row[i] - row[differenceIndex]); + } + result.add(res); + } + return result; + } + + public static List countedValuesToCdf(List> countedValues) { + double[] countsTotal = new double[countedValues.size()]; + LinkedList list = new LinkedList(); + for (int i = 0; i < countedValues.size(); i++) { + if (countedValues.get(i) == null) continue; + int index = list.size() / 2; + for (double[] kv : countedValues.get(i)) { + if (kv.length != 2) throw new IllegalArgumentException("counted values must be arrays length 2"); + while (index > 0 && kv[0] < list.get(index)[0]) index--; + while (index < list.size() && kv[0] > list.get(index)[0]) index++; + if (index < list.size() - 1 && kv[0] == list.get(index)[0]) list.get(index)[i + 1] += kv[1]; + else { + double[] insert = new double[countedValues.size() + 1]; + insert[0] = kv[0]; + insert[i + 1] = kv[1]; + list.add(index, insert); + } + countsTotal[i] += kv[1]; + } + } + double refCount = 0.0; + for (int i = 0; i < countsTotal.length; i++) { + if (countsTotal[i] == 0.0) continue; + if (refCount == 0.0) { + refCount = countsTotal[i]; + continue; + } + if (refCount != countsTotal[i]) throw new InvalidParameterException("total counts must be the same"); + } + double[] countsCurrent = new double[countedValues.size()]; + List result = new ArrayList(list.size() * 2); + result.add(new double[countedValues.size() + 1]); + Iterator iterator = list.iterator(); + double lastValue = 0.0; + while(iterator.hasNext()) { + double[] current = iterator.next(); + double[] result1 = new double[countedValues.size() + 1]; + result1[0] = current[0]; + lastValue = current[0]; + double[] result2 = new double[countedValues.size() + 1]; + result2[0] = current[0]; + for (int i = 1; i < current.length; i++) { + if (countsTotal[i - 1] == 0.0) result1[i] = Double.NaN; + else result1[i] = countsCurrent[i - 1] / countsTotal[i - 1]; + countsCurrent[i - 1] += current[i]; + if (countsTotal[i - 1] == 0.0) result2[i] = Double.NaN; + else result2[i] = countsCurrent[i - 1] / countsTotal[i - 1]; + } + result.add(result1); + result.add(result2); + } + double[] resultFinal = new double[countedValues.size() + 1]; + resultFinal[0] = lastValue; + for (int i = 1; i < resultFinal.length; i++) { + if (countsTotal[i - 1] == 0.0) resultFinal[i] = Double.NaN; + else resultFinal[i] = countsCurrent[i - 1] / countsTotal[i - 1]; + } + result.add(resultFinal); + return result; + } + + public static List> relativeBudgetNumbers(List> input, int relativeIndex) { + if (input.get(relativeIndex) == null) throw new NullPointerException("relative input is null"); + if (input.get(relativeIndex).size() != 1) throw new NullPointerException("relative input must have exactly one value"); + double relativeValue = input.get(relativeIndex).get(0)[0]; + List> result = new ArrayList<>(input.size()); + for (int i = 0; i < input.size(); i++) { + if (input.get(i) == null) { + result.add(null); + continue; + } + List res = new ArrayList<>(input.get(i).size()); + for (int j = 0; j < input.get(i).size(); j++) res.add(new double[] { input.get(i).get(j)[0] / relativeValue, input.get(i).get(j)[1] }); + result.add(res); + } + return result; + } + +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/DoubleComparator.java b/source/java/logConverter/src/logConverter/DoubleComparator.java new file mode 100644 index 0000000000000000000000000000000000000000..609c47c35c15b8005eae859bbc664dd2d78e13fe --- /dev/null +++ b/source/java/logConverter/src/logConverter/DoubleComparator.java @@ -0,0 +1,20 @@ +package logConverter; + +import java.util.Comparator; + +public class DoubleComparator implements Comparator { + + private double tolerance; + + public DoubleComparator(double tolerance) { + if (tolerance < 0.0) throw new IllegalArgumentException("tolerance is below zero"); + this.tolerance = tolerance; + } + + @Override + public int compare(Double o1, Double o2) { + if (o1 - o2 > tolerance) return 1; + if (o2 - o1 > tolerance) return -1; + return 0; + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/ExternalCommand.java b/source/java/logConverter/src/logConverter/ExternalCommand.java new file mode 100644 index 0000000000000000000000000000000000000000..5e2642fe21e28c61e38024e987b18705ec184bac --- /dev/null +++ b/source/java/logConverter/src/logConverter/ExternalCommand.java @@ -0,0 +1,17 @@ +package logConverter; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class ExternalCommand { + + public static void execute(List command) throws IOException, InterruptedException, ExecutionException { + ProcessBuilder builder = new ProcessBuilder(command); + builder.inheritIO(); + Process process = builder.start(); + int exitCode = process.waitFor(); + assert exitCode == 0; + process.destroy(); + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/Log.java b/source/java/logConverter/src/logConverter/Log.java new file mode 100644 index 0000000000000000000000000000000000000000..27baa0e039151f23ee2967c1725b975e7ebbf035 --- /dev/null +++ b/source/java/logConverter/src/logConverter/Log.java @@ -0,0 +1,219 @@ +package logConverter; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +public class Log { + + public static List rowsTotalTimes(Log[] logs, boolean ste) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getTotalTimes()); + } + return Data.generateRowsValue(lists, ste); + } + + public static List rowsSetupTimes(Log[] logs, boolean ste) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getSetupTimes()); + } + return Data.generateRowsValue(lists, ste); + } + + public static List rowsExecuteTimes(Log[] logs, boolean ste) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getExecuteTimes()); + } + return Data.generateRowsValue(lists, ste); + } + + public static List rowsTeardownTimes(Log[] logs, boolean ste) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getTeardownTimes()); + } + return Data.generateRowsValue(lists, ste); + } + + public static List rowsSetupPartitions(Log[] logs) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getSetupNumberPartitions()); + } + return Data.generateRowsDouble(lists); + } + + public static List rowsExecutePartitions(Log[] logs) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getExecuteNumberPartitions()); + } + return Data.generateRowsDouble(lists); + } + + public static List rowsTeardownPartitions(Log[] logs) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getTeardownNumberPartitions()); + } + return Data.generateRowsDouble(lists); + } + + public static List rowsValues(Log[] logs, boolean ste) { + List> lists = new ArrayList>(logs.length); + for (int i = 0; i < logs.length; i++) { + if (logs[i] == null) lists.add(null); + else lists.add(logs[i].getValues()); + } + return Data.generateRowsValue(lists, ste); + } + + public static Log average(List logs) { + if (logs == null) throw new NullPointerException("logs is null"); + if (logs.size() < 2) throw new IllegalArgumentException("need at least two logs for average"); + int size = logs.get(0).entries.size(); + for (Log log : logs) if (log.entries.size() != size) throw new IllegalArgumentException("logs must have same size"); + List resultEntries = new LinkedList<>(); + for (int i = 0; i < size; i++) { + List currentEntries = new LinkedList<>(); + for (Log log : logs) currentEntries.add(log.entries.get(i)); + resultEntries.add(LogEntryAnalytic.average(currentEntries)); + } + StringBuffer buf = new StringBuffer(); + buf.append("Average:"); + for (Log log : logs) buf.append(log.name + ";"); + return new Log(buf.toString(), resultEntries); + } + + public static Log averageSkipExtremes(List logs, int skip) { + if (logs == null) throw new NullPointerException("logs is null"); + if (logs.size() < 3) throw new IllegalArgumentException("need at least three logs"); + int size = logs.get(0).entries.size(); + for (Log log : logs) if (log.entries.size() != size) throw new IllegalArgumentException("logs must have same size"); + List resultEntries = new LinkedList<>(); + for (int i = 0; i < size; i++) { + List currentEntries = new LinkedList<>(); + for (Log log : logs) currentEntries.add(log.entries.get(i)); + resultEntries.add(LogEntryAnalytic.averageSkipExtremes(currentEntries, skip)); + } + StringBuffer buf = new StringBuffer(); + buf.append("Average:"); + for (Log log : logs) buf.append(log.name + ";"); + return new Log(buf.toString(), resultEntries); + } + + public static List valueDifferences(Log a, Log b) { + if (a == null) throw new NullPointerException("a is null"); + if (b == null) throw new NullPointerException("b is null"); + if (a.entries.size() != b.entries.size()) throw new IllegalArgumentException("logs must have same size"); + List list = new LinkedList<>(); + for (int i = 0; i < a.entries.size(); i++) list.addAll(LogEntryAnalytic.valueDifference(a.entries.get(i), b.entries.get(i))); + return list; + } + + public static List executeTimeDifferencesPercent(Log a, Log b) { + if (a == null) throw new NullPointerException("a is null"); + if (b == null) throw new NullPointerException("b is null"); + if (a.entries.size() != b.entries.size()) throw new IllegalArgumentException("logs must have same size"); + List list = new LinkedList<>(); + for (int i = 0; i < a.entries.size(); i++) list.addAll(LogEntryAnalytic.executeTimeDifferencePercent(a.entries.get(i), b.entries.get(i))); + return list; + } + + private String name; + private List entries; + + public Log(String name, List entries) { + if (name == null) throw new NullPointerException("name is null"); + if (entries == null) throw new NullPointerException("entries is null"); + this.name = name; + this.entries = entries; + } + + public List getEpsilons() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getEpsilons()); + return list; + } + + public List getValues() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getValues()); + return list; + } + + public List getTotalTimes() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getTotalTimes()); + return list; + } + + public List getSetupTimes() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getSetupTimes()); + return list; + } + + public List getExecuteTimes() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getExecuteTimes()); + return list; + } + + public List getBkExecuteTimes() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getBkExecuteTimes()); + return list; + } + + public List getTeardownTimes() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getTeardownTimes()); + return list; + } + + public List getSetupNumberPartitions() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getSetupNumberPartitions()); + return list; + } + + public List getExecuteNumberPartitions() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getExecuteNumberPartitions()); + return list; + } + + public List getTeardownNumberPartitions() { + List list = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) list.addAll(entry.getTeardownNumberPartitions()); + return list; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof Log)) return false; + Log other = (Log) object; + if (!name.equals(other.name)) return false; + for (LogEntryAnalytic entry : entries) if (!other.entries.contains(entry)) return false; + return true; + } + + @Override + public String toString() { + StringBuffer buf = new StringBuffer(); + buf.append("Log: name:" + name + ";"); + for (LogEntryAnalytic entry : entries) buf.append("\n" + entry.toString()); + return buf.toString(); + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/LogEntry.java b/source/java/logConverter/src/logConverter/LogEntry.java new file mode 100644 index 0000000000000000000000000000000000000000..bbd7ab86080d8e33e37b018d291839c7c0fafe03 --- /dev/null +++ b/source/java/logConverter/src/logConverter/LogEntry.java @@ -0,0 +1,28 @@ +package logConverter; + +public class LogEntry { + + private String name; + + public LogEntry(String name) { + if (name == null) throw new NullPointerException("name is null"); + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof LogEntry)) return false; + LogEntry other = (LogEntry) object; + if (!name.equals(other.name)) return false; + return true; + } + + @Override + public String toString() { + return "LogEntry: " + name; + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/LogEntryAggregate.java b/source/java/logConverter/src/logConverter/LogEntryAggregate.java new file mode 100644 index 0000000000000000000000000000000000000000..3d60d1af96a473ae60185970158a8376770449b0 --- /dev/null +++ b/source/java/logConverter/src/logConverter/LogEntryAggregate.java @@ -0,0 +1,250 @@ +package logConverter; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; + +public class LogEntryAggregate extends LogEntry { + + public static LogEntryAggregate average(List entries) { + if (entries == null) throw new NullPointerException("entries is null"); + if (entries.size() < 1) throw new IllegalArgumentException("need at least one entry for average"); + SummaryStatistics valueStats = new SummaryStatistics(); + SummaryStatistics epsilonStats = new SummaryStatistics(); + SummaryStatistics setupTimeStats = new SummaryStatistics(); + SummaryStatistics executeTimeStats = new SummaryStatistics(); + SummaryStatistics bkExecuteTimeStats = new SummaryStatistics(); + SummaryStatistics teardownTimeStats = new SummaryStatistics(); + SummaryStatistics totalTimeStats = new SummaryStatistics(); + String name = entries.get(0).getName(); + String message = entries.get(0).getMessage(); + LogEntryNumberPartitions numberPartitions = entries.get(0).numberPartitions; + for (LogEntryAggregate entry : entries) { + if (!name.equals(entry.getName())) throw new IllegalArgumentException("names must match"); + if (!message.equals(entry.getMessage())) throw new IllegalArgumentException("messages must match"); + if (!numberPartitions.equals(entry.numberPartitions)) { + throw new IllegalArgumentException("numberPartitions must match"); + } + epsilonStats.addValue(entry.epsilon.getValue()); + valueStats.addValue(entry.value.getValue()); + setupTimeStats.addValue(entry.setupTime.getValue()); + executeTimeStats.addValue(entry.executeTime.getValue()); + bkExecuteTimeStats.addValue(entry.bkExecuteTime.getValue()); + teardownTimeStats.addValue(entry.teardownTime.getValue()); + totalTimeStats.addValue(entry.totalTime.getValue()); + } + Value epsilon = new Value(epsilonStats.getMean(), epsilonStats.getStandardDeviation()); + Value value = new Value(valueStats.getMean(), valueStats.getStandardDeviation()); + Value setupTime = new Value(setupTimeStats.getMean(), setupTimeStats.getStandardDeviation()); + Value executeTime = new Value(executeTimeStats.getMean(), executeTimeStats.getStandardDeviation()); + Value bkExecuteTime = new Value(bkExecuteTimeStats.getMean(), bkExecuteTimeStats.getStandardDeviation()); + Value teardownTime = new Value(teardownTimeStats.getMean(), teardownTimeStats.getStandardDeviation()); + Value totalTime = new Value(totalTimeStats.getMean(), totalTimeStats.getStandardDeviation()); + return new LogEntryAggregate(name, message, epsilon, value, setupTime, executeTime, bkExecuteTime, teardownTime, totalTime, null, numberPartitions); + } + + public static LogEntryAggregate averageSkipExtremes(List entries, int skip) { + if (entries == null) throw new NullPointerException("entries is null"); + if (entries.size() < 1 + 2 * skip) throw new IllegalArgumentException("need at least three entries"); + LinkedList epsilonList = new LinkedList<>(); + LinkedList valueList = new LinkedList<>(); + LinkedList setupTimeList = new LinkedList<>(); + LinkedList executeTimeList = new LinkedList<>(); + LinkedList bkExecuteTimeList = new LinkedList<>(); + LinkedList teardownTimeList = new LinkedList<>(); + LinkedList totalTimeList = new LinkedList<>(); + String name = entries.get(0).getName(); + String message = entries.get(0).getMessage(); + LogEntryNumberPartitions numberPartitions = entries.get(0).numberPartitions; + for (LogEntryAggregate entry : entries) { + if (!name.equals(entry.getName())) throw new IllegalArgumentException("names must match"); + if (!message.equals(entry.getMessage())) throw new IllegalArgumentException("messages must match"); + if (!numberPartitions.equals(entry.numberPartitions)) { + throw new IllegalArgumentException("numberPartitions must match"); + } + epsilonList.add(entry.epsilon.getValue()); + valueList.add(entry.value.getValue()); + setupTimeList.add(entry.setupTime.getValue()); + executeTimeList.add(entry.executeTime.getValue()); + bkExecuteTimeList.add(entry.bkExecuteTime.getValue()); + teardownTimeList.add(entry.teardownTime.getValue()); + totalTimeList.add(entry.totalTime.getValue()); + } + Collections.sort(epsilonList); + Collections.sort(valueList); + Collections.sort(setupTimeList); + Collections.sort(executeTimeList); + Collections.sort(bkExecuteTimeList); + Collections.sort(teardownTimeList); + Collections.sort(totalTimeList); + for (int i = 0; i < skip; i++) epsilonList.removeFirst(); + for (int i = 0; i < skip; i++) valueList.removeFirst(); + for (int i = 0; i < skip; i++) setupTimeList.removeFirst(); + for (int i = 0; i < skip; i++) executeTimeList.removeFirst(); + for (int i = 0; i < skip; i++) bkExecuteTimeList.removeFirst(); + for (int i = 0; i < skip; i++) teardownTimeList.removeFirst(); + for (int i = 0; i < skip; i++) totalTimeList.removeFirst(); + for (int i = 0; i < skip; i++) epsilonList.removeLast(); + for (int i = 0; i < skip; i++) valueList.removeLast(); + for (int i = 0; i < skip; i++) setupTimeList.removeLast(); + for (int i = 0; i < skip; i++) executeTimeList.removeLast(); + for (int i = 0; i < skip; i++) bkExecuteTimeList.removeLast(); + for (int i = 0; i < skip; i++) teardownTimeList.removeLast(); + for (int i = 0; i < skip; i++) totalTimeList.removeLast(); + SummaryStatistics epsilonStats = new SummaryStatistics(); + SummaryStatistics valueStats = new SummaryStatistics(); + SummaryStatistics setupTimeStats = new SummaryStatistics(); + SummaryStatistics executeTimeStats = new SummaryStatistics(); + SummaryStatistics bkExecuteTimeStats = new SummaryStatistics(); + SummaryStatistics teardownTimeStats = new SummaryStatistics(); + SummaryStatistics totalTimeStats = new SummaryStatistics(); + for (Double value : epsilonList) epsilonStats.addValue(value); + for (Double value : valueList) valueStats.addValue(value); + for (Double value : setupTimeList) setupTimeStats.addValue(value); + for (Double value : executeTimeList) executeTimeStats.addValue(value); + for (Double value : bkExecuteTimeList) bkExecuteTimeStats.addValue(value); + for (Double value : teardownTimeList) teardownTimeStats.addValue(value); + for (Double value : totalTimeList) totalTimeStats.addValue(value); + Value epsilon = new Value(epsilonStats.getMean(), epsilonStats.getStandardDeviation()); + Value value = new Value(valueStats.getMean(), valueStats.getStandardDeviation()); + Value setupTime = new Value(setupTimeStats.getMean(), setupTimeStats.getStandardDeviation()); + Value executeTime = new Value(executeTimeStats.getMean(), executeTimeStats.getStandardDeviation()); + Value bkExecuteTime = new Value(bkExecuteTimeStats.getMean(), bkExecuteTimeStats.getStandardDeviation()); + Value teardownTime = new Value(teardownTimeStats.getMean(), teardownTimeStats.getStandardDeviation()); + Value totalTime = new Value(totalTimeStats.getMean(), totalTimeStats.getStandardDeviation()); + return new LogEntryAggregate(name, message, epsilon, value, setupTime, executeTime, bkExecuteTime, teardownTime, totalTime, null, numberPartitions); + } + + public static Double valueDifference(LogEntryAggregate a, LogEntryAggregate b) { + if (a == null) throw new NullPointerException("a is null"); + if (b == null) throw new NullPointerException("b is null"); + if (!a.getName().equals(b.getName())) throw new IllegalArgumentException("names must match: " + a.getName() + " " + b.getName()); + return a.value.getValue() - b.value.getValue(); + } + + public static Double executeTimeDifferencePercent(LogEntryAggregate a, LogEntryAggregate b) { + if (a == null) throw new NullPointerException("a is null"); + if (b == null) throw new NullPointerException("b is null"); + if (!a.getName().equals(b.getName())) throw new IllegalArgumentException("names must match: " + a.getName() + " " + b.getName()); + double aVal = a.executeTime.getValue(); + double bVal = b.executeTime.getValue(); + if (aVal == 0.0 || bVal == 0.0) return Double.NaN; + return ((aVal - bVal) / aVal) * 100.0; + } + + private String message; + private Value epsilon; + private Value value; + private Value totalTime; + private Value setupTime; + private Value executeTime; + private Value bkExecuteTime; + private Value teardownTime; + private List ranges; + private LogEntryNumberPartitions numberPartitions; + + public LogEntryAggregate(String name, String message, Value epsilon, Value value, Value setupTime, Value executeTime, Value bkExecuteTime, Value teardownTime, + Value totalTime, List ranges, LogEntryNumberPartitions numberPartitions) { + super(name); + if (message == null) throw new NullPointerException("message is null"); + if (epsilon == null) throw new NullPointerException("epsilon is null"); + if (value == null) throw new NullPointerException("value is null"); + if (setupTime == null) throw new NullPointerException("setupTime is null"); + if (executeTime == null) throw new NullPointerException("executeTime is null"); + if (bkExecuteTime == null) throw new NullPointerException("bkExecuteTime is null"); + if (teardownTime == null) throw new NullPointerException("teardownTime is null"); + if (totalTime == null) throw new NullPointerException("totalTime is null"); + if (numberPartitions == null) throw new NullPointerException("numberPartitions is null"); + this.message = message; + this.epsilon = epsilon; + this.value = value; + this.totalTime = totalTime; + this.setupTime = setupTime; + this.executeTime = executeTime; + this.bkExecuteTime = bkExecuteTime; + this.teardownTime = teardownTime; + this.ranges = ranges; + this.numberPartitions = numberPartitions; + } + + public String getMessage() { + return message; + } + + public Value getEpsilon() { + return epsilon; + } + + public Value getValue() { + return value; + } + + public Value getTotalTime() { + return totalTime; + } + + public Value getSetupTime() { + return setupTime; + } + + public Value getExecuteTime() { + return executeTime; + } + + public Value getBkExecuteTime() { + return bkExecuteTime; + } + + public Value getTeardownTime() { + return teardownTime; + } + + public double getSetupNumberPartitions() { + return numberPartitions.getSetupNumberPartitions(); + } + + public double getExecuteNumberPartitions() { + return numberPartitions.getExecuteNumberPartitions(); + } + + public double getTeardownNumberPartitions() { + return numberPartitions.getTeardownNumberPartitions(); + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof LogEntryAggregate)) return false; + if (!super.equals(object)) return false; + LogEntryAggregate other = (LogEntryAggregate) object; + if (!message.equals(other.message)) return false; + if (!epsilon.equals(other.epsilon) )return false; + if (!value.equals(other.value) )return false; + if (!setupTime.equals(other.setupTime)) return false; + if (!executeTime.equals(other.executeTime)) return false; + if (!bkExecuteTime.equals(other.bkExecuteTime)) return false; + if (!teardownTime.equals(other.teardownTime)) return false; + if ((ranges == null && other.ranges != null) || (ranges != null && other.ranges == null)) return false; + if (ranges != null && ranges.size() != other.ranges.size()) return false; + if (ranges != null) for (int i = 0; i < ranges.size(); i++) if (!ranges.get(i).equals(other.ranges.get(i))) return false; + if (!numberPartitions.equals(other.numberPartitions)) return false; + return true; + } + + @Override + public String toString() { + StringBuffer buf = new StringBuffer(); + buf.append(super.toString() + ";"); + buf.append("message:" + message + ";"); + buf.append("epsilon:" + epsilon.toString() + ";"); + buf.append("value:" + value.toString() + ";"); + buf.append("setupTime:" + setupTime.toString() + ";"); + buf.append("executeTime:" + executeTime.toString() + ";"); + buf.append("bkExecuteTime:" + bkExecuteTime.toString() + ";"); + buf.append("teardownTime:" + teardownTime.toString() + ";"); + buf.append("numberPartitions:" + numberPartitions.toString() + ";"); + if (ranges != null) for (LogEntry range : ranges) buf.append("\n\t\t" + range.toString()); + return buf.toString(); + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/LogEntryAnalytic.java b/source/java/logConverter/src/logConverter/LogEntryAnalytic.java new file mode 100644 index 0000000000000000000000000000000000000000..6025b26918481b6699d5ebcc8a858aa0aafa36e9 --- /dev/null +++ b/source/java/logConverter/src/logConverter/LogEntryAnalytic.java @@ -0,0 +1,188 @@ +package logConverter; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; + +public class LogEntryAnalytic extends LogEntry { + + public static LogEntryAnalytic average(List entries) { + if (entries == null) throw new NullPointerException("entries is null"); + if (entries.size() < 2) throw new IllegalArgumentException("need at least two entries for average"); + long id = entries.get(0).id; + String message = entries.get(0).message; + SummaryStatistics timeStats = new SummaryStatistics(); + for (LogEntryAnalytic entry : entries) { + if (entry.id != id) throw new IllegalArgumentException("ids of entries must match"); + if (!entry.message.equals(message)) throw new IllegalArgumentException("messages of entries must match"); + timeStats.addValue(entry.time.getValue()); + } + Value time = new Value(timeStats.getMean(), timeStats.getStandardDeviation()); + int size = entries.get(0).aggregates.size(); + for (LogEntryAnalytic entry : entries) { + if (entry.aggregates.size() != size) { + throw new IllegalArgumentException("queries must have same size"); + } + } + List resultQueries = new LinkedList<>(); + for (int i = 0; i < size; i++) { + List currentQueries = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) currentQueries.add(entry.aggregates.get(i)); + resultQueries.add(LogEntryAggregate.average(currentQueries)); + } + return new LogEntryAnalytic(id, time, message, resultQueries); + } + + public static LogEntryAnalytic averageSkipExtremes(List entries, int skip) { + if (entries == null) throw new NullPointerException("entries is null"); + if (entries.size() < 1 + 2 * skip) throw new IllegalArgumentException("need at least three entries"); + long id = entries.get(0).id; + String message = entries.get(0).message; + LinkedList values = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) { + if (entry.id != id) throw new IllegalArgumentException("ids of entries must match"); + if (!entry.message.equals(message)) throw new IllegalArgumentException("messages of entries must match"); + values.add(entry.time.getValue()); + } + Collections.sort(values); + for (int i = 0; i < skip; i++) values.removeFirst(); + for (int i = 0; i < skip; i++) values.removeLast(); + SummaryStatistics timeStats = new SummaryStatistics(); + for (Double value : values) timeStats.addValue(value); + Value time = new Value(timeStats.getMean(), timeStats.getStandardDeviation()); + int size = entries.get(0).aggregates.size(); + for (LogEntryAnalytic entry : entries) { + if (entry.aggregates.size() != size) { + throw new IllegalArgumentException("queries must have same size"); + } + } + List resultQueries = new LinkedList<>(); + for (int i = 0; i < size; i++) { + List currentQueries = new LinkedList<>(); + for (LogEntryAnalytic entry : entries) currentQueries.add(entry.aggregates.get(i)); + resultQueries.add(LogEntryAggregate.averageSkipExtremes(currentQueries, skip)); + } + return new LogEntryAnalytic(id, time, message, resultQueries); + } + + public static List valueDifference(LogEntryAnalytic a, LogEntryAnalytic b) { + if (a == null) throw new NullPointerException("a is null"); + if (b == null) throw new NullPointerException("b is null"); + if (a.aggregates.size() != b.aggregates.size()) throw new IllegalArgumentException("queries must have same size; was " + a.aggregates.size() + " vs. " + b.aggregates.size()); + List resultList = new LinkedList<>(); + for (int i = 0; i < a.aggregates.size(); i++) resultList.add(LogEntryAggregate.valueDifference(a.aggregates.get(i), b.aggregates.get(i))); + return resultList; + } + + public static List executeTimeDifferencePercent(LogEntryAnalytic a, LogEntryAnalytic b) { + if (a == null) throw new NullPointerException("a is null"); + if (b == null) throw new NullPointerException("b is null"); + if (a.aggregates.size() != b.aggregates.size()) throw new IllegalArgumentException("queries must have same size; was " + a.aggregates.size() + " vs. " + b.aggregates.size()); + List resultList = new LinkedList<>(); + for (int i = 0; i < a.aggregates.size(); i++) resultList.add(LogEntryAggregate.executeTimeDifferencePercent(a.aggregates.get(i), b.aggregates.get(i))); + return resultList; + } + + private long id; + private Value time; + private String message; + private List aggregates; + + public LogEntryAnalytic(long id, Value time, String message, List aggregates) { + super("Analytic"); + if (time == null) throw new NullPointerException("time is null"); + if (message == null) throw new NullPointerException("message is null"); + if (aggregates == null) throw new NullPointerException("aggregates is null"); + this.id = id; + this.time = time; + this.message = message; + this.aggregates = aggregates; + } + + public List getEpsilons() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getEpsilon()); + return list; + } + + public List getValues() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getValue()); + return list; + } + + public List getTotalTimes() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getTotalTime()); + return list; + } + + public List getSetupTimes() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getSetupTime()); + return list; + } + + public List getExecuteTimes() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getExecuteTime()); + return list; + } + + public List getBkExecuteTimes() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getBkExecuteTime()); + return list; + } + + public List getTeardownTimes() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getTeardownTime()); + return list; + } + + public List getSetupNumberPartitions() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getSetupNumberPartitions()); + return list; + } + + public List getExecuteNumberPartitions() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getExecuteNumberPartitions()); + return list; + } + + public List getTeardownNumberPartitions() { + List list = new LinkedList<>(); + for (LogEntryAggregate query : aggregates) list.add(query.getTeardownNumberPartitions()); + return list; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof LogEntryAnalytic)) return false; + if (!super.equals(object)) return false; + LogEntryAnalytic other = (LogEntryAnalytic) object; + if (id != other.id) return false; + if (!time.equals(other.time)) return false; + if (!message.equals(other.message)) return false; + if (aggregates.size() != other.aggregates.size()) return false; + for (int i = 0; i < aggregates.size(); i++) if (!aggregates.get(i).equals(other.aggregates.get(i))) return false; + return true; + } + + @Override + public String toString() { + StringBuffer buf = new StringBuffer(); + buf.append(super.toString() + ";"); + buf.append("id:" + id + ";"); + buf.append("time:" + time.toString() + ";"); + buf.append("message:" + message + ";"); + buf.append("\nqueries:"); + for (LogEntryAggregate query : aggregates) buf.append("\n\t" + query.toString()); + return buf.toString(); + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/LogEntryNumberPartitions.java b/source/java/logConverter/src/logConverter/LogEntryNumberPartitions.java new file mode 100644 index 0000000000000000000000000000000000000000..d41f454d0bb79c3fa8e9e75c229c8d8d0cfd97a8 --- /dev/null +++ b/source/java/logConverter/src/logConverter/LogEntryNumberPartitions.java @@ -0,0 +1,46 @@ +package logConverter; + +public class LogEntryNumberPartitions extends LogEntry { + + private int setupNumberPartitions; + private int executeNumberPartitions; + private int teardownNumberPartitions; + + public LogEntryNumberPartitions(int setupNumberPartitions, int executeNumberPartitions, int teardownNumberPartitions) { + super("NumberPartitions"); + if (setupNumberPartitions < 0) throw new IllegalArgumentException("setupNumberPartitions cannot be less than 0"); + if (executeNumberPartitions < 0) throw new IllegalArgumentException("executeNumberPartitions cannot be less than 0"); + if (teardownNumberPartitions < 0) throw new IllegalArgumentException("teardownNumberPartitions cannot be less than 0"); + this.setupNumberPartitions = setupNumberPartitions; + this.executeNumberPartitions = executeNumberPartitions; + this.teardownNumberPartitions = teardownNumberPartitions; + } + + public int getSetupNumberPartitions() { + return setupNumberPartitions; + } + + public int getExecuteNumberPartitions() { + return executeNumberPartitions; + } + + public int getTeardownNumberPartitions() { + return teardownNumberPartitions; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof LogEntryNumberPartitions)) return false; + if (!super.equals(object)) return false; + LogEntryNumberPartitions other = (LogEntryNumberPartitions) object; + if (setupNumberPartitions != other.setupNumberPartitions) return false; + if (executeNumberPartitions != other.executeNumberPartitions) return false; + if (teardownNumberPartitions != other.teardownNumberPartitions) return false; + return true; + } + + @Override + public String toString() { + return super.toString() + ";setupNumberPartitions:" + setupNumberPartitions + ";executeNumberPartitions:" + executeNumberPartitions + ";teardownNumberPartitions:" + teardownNumberPartitions; + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/LogEntryRange.java b/source/java/logConverter/src/logConverter/LogEntryRange.java new file mode 100644 index 0000000000000000000000000000000000000000..cf26b65c87ddb1cdf6ef82be8fc7b18f34ad9ba8 --- /dev/null +++ b/source/java/logConverter/src/logConverter/LogEntryRange.java @@ -0,0 +1,32 @@ +package logConverter; + +public class LogEntryRange extends LogEntry { + + private String fieldName; + private long low; + private long high; + + public LogEntryRange(String fieldName, long low, long high) { + super("Range"); + if (fieldName == null) throw new NullPointerException("fieldName is null"); + this.fieldName = fieldName; + this.low = low; + this.high = high; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof LogEntryRange)) return false; + if (!super.equals(object)) return false; + LogEntryRange other = (LogEntryRange) object; + if (!fieldName.equals(other.fieldName)) return false; + if (low != other.low) return false; + if (high != other.high) return false; + return true; + } + + @Override + public String toString() { + return super.toString() + ";fieldName:" + fieldName; + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/Output.java b/source/java/logConverter/src/logConverter/Output.java new file mode 100644 index 0000000000000000000000000000000000000000..de397ee094bfbfe75ecd19dc2bc68889477de3c2 --- /dev/null +++ b/source/java/logConverter/src/logConverter/Output.java @@ -0,0 +1,380 @@ +package logConverter; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class Output { + + private static String[] databases = new String[] { "Disk", "Mem" }; + private static String[] systems = new String[] { "Direct", "PINQ", "Always-BK", "Always-BK-Even", "Threshold-BK", "Threshold-BK-Even" }; + private static String[] kinds = new String[] { "Runtime", "Budget" }; + + private static String output = "evaluation01"; + private static int runs = 8; + private static int skip = 1; + private static double percentRate = 0.05; + private static String instanceName = "2200_2019-01-01_00-00-00-000"; + private static String instancePath = File.separator + "home" + File.separator + "user" + File.separator + "unitrax" + File.separator + + "experiments" + File.separator + "financial" + File.separator + "experiment" + File.separator + instanceName; + private static String scriptPath = File.separator + "home" + File.separator + "user" + File.separator + "unitrax" + File.separator + + "gitlab" + File.separator + "unitrax" + File.separator + "scripts" + File.separator + "plot" + File.separator + "java"; + private static String gnuplot = File.separator + "usr" + File.separator + "local" + File.separator + "bin" + File.separator + "gnuplot"; + + + private static String outputPath = instancePath + File.separator + "output" + File.separator + output; + private static String plotPath = instancePath + File.separator + "plot" + File.separator + output; + private static boolean filterDummies = true; + + private static String fileNameSortedAll = "sortedAll"; + private static String fileNameSortedPercent = "sortedPercent"; + private static String fileNameUnsortedAll = "unsortedAll"; + private static String fileNameUnsortedPercent = "unsortedPercent"; + private static String fileNameBudgets = "budgets"; + + private static String[] generateHeader(String[] additionalFirst, String[] databases, String[] systems, boolean ste, int skipSte) { + int additionalIndex = 0; + if (additionalFirst != null) additionalIndex = additionalFirst.length; + int numCols = databases.length * systems.length + additionalIndex; + String[] ret = new String[numCols]; + for (int i = 0; i < databases.length; i++) { + for (int j = 0; j < systems.length; j++) { + if (databases[i] != null && systems[j] != null) ret[i * systems.length + j + additionalIndex] = databases[i] + "-" + systems[j]; + else ret[i * systems.length + j + additionalIndex] = "missing"; + } + } + if (additionalFirst != null) for (int i = 0; i < additionalFirst.length; i++) ret[i] = additionalFirst[i]; + + if (ste) { + numCols += databases.length * systems.length + additionalIndex - skipSte; + String[] result = new String[numCols]; + int index = 0; + for (int i = 0; i < ret.length; i++) { + result[index] = ret[i]; + index++; + if (skipSte <= i) { + result[index] = "StE " + ret[i]; + index++; + } + } + return result; + } + return ret; + } + + private static String[] generateRelativeHeader(String[] additionalFirst, String[] databases, String[] systems, int relativeIndex, int skip) { + String[] headers = generateHeader(additionalFirst, databases, systems, false, 0); + String relativeHeader = headers[relativeIndex]; + for (int i = skip; i < headers.length; i++) { + if (headers[i].equals("missing")) continue; + headers[i] = headers[i] + "/" + relativeHeader; + } + return headers; + } + + private static void plot(String scriptName, String inputFileName, String outputFileName) throws IOException, InterruptedException, ExecutionException { + ExternalCommand.execute(Arrays.asList(gnuplot, "-c", scriptPath + File.separator + scriptName, + outputPath + File.separator + inputFileName, + plotPath + File.separator + outputFileName)); + } + + private static void outputBudgetUsedCDF(String kind) { + try { + List> budgetUsedLists = Parser.generateBudgetUsedLists(databases, systems, kind, instancePath); + List> budgetUsedListsRelative = Data.relativeBudgetNumbers(budgetUsedLists, 1); + List cdf = Data.countedValuesToCdf(budgetUsedListsRelative); + Result result = new Result(generateRelativeHeader(new String[] { "cdf" }, databases, systems, 1 + 1, 1), cdf); + result.writeTo(outputPath + File.separator + kind + "-" + fileNameBudgets + "-cdf.csv"); + plot("budgets.gp", kind + "-" + fileNameBudgets + "-cdf.csv", kind + "-" + fileNameBudgets + "-cdf.eps"); + } catch (IOException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (InterruptedException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (ExecutionException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } + } + + private static void outputValueDifferences(int[] percentIndexes, Log[][] logsForKind, String kind) { + try { + for (int i = 0; i < logsForKind.length; i++) { + String run = "run" + i; + if (i == 0) run = "average"; + Log[] logsRun = logsForKind[i]; + + List unsortedRows = Log.rowsValues(logsRun, false); + Result unsortedResultValues = new Result(generateHeader(null, databases, systems, false, 0), unsortedRows); + unsortedResultValues.writeTo(outputPath + File.separator + kind + "-" + run + "-Values-" + fileNameUnsortedAll + ".csv"); + unsortedResultValues.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-Values-" + fileNameUnsortedAll + ".csv"); + plot("values.gp", kind + "-" + run + "-Values-" + fileNameUnsortedAll + ".csv", kind + "-" + run + "-Values-" + fileNameUnsortedAll + ".eps"); + + List unsortedDifferenceRows = Data.differenceNumbersAbs(unsortedRows, 0); + Result unsortedResult = new Result(generateHeader(null, databases, systems, false, 0), unsortedDifferenceRows); + unsortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedAll + ".csv"); + unsortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedAll + ".csv"); + plot("values.gp", kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedAll + ".csv", kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedAll + ".eps"); + + List unsortedDifferenceRowsPercent = Data.filterBy(unsortedDifferenceRows, percentIndexes); + Result unsortedPercentResult = new Result(generateHeader(null, databases, systems, false, 0), unsortedDifferenceRowsPercent); + unsortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedPercent + ".csv"); + unsortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedPercent + ".csv"); + plot("values.gp", kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedPercent + ".csv", kind + "-" + run + "-ValueDifferences-" + fileNameUnsortedPercent + ".eps"); + + List unsortedCdfRows = Data.cdf(unsortedDifferenceRows, 0.0, 0.1, 10000000, true, 100); + Result unsortedCdfResult = new Result(generateRelativeHeader(new String[] { "cdf" }, databases, systems, 0 + 1, 1), unsortedCdfRows); + unsortedCdfResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ValueDifferences-cdf.csv"); + plot("valuesCdf.gp", kind + "-" + run + "-ValueDifferences-cdf.csv", kind + "-" + run + "-ValueDifferences-cdf.eps"); + + List unsortedRowsExecute = Log.rowsExecuteTimes(logsRun, true); + int[] sortFilter = Data.getSortFilter(unsortedRowsExecute, 0); + List sortedDifferenceRows = Data.filterBy(unsortedDifferenceRows, sortFilter); + Result sortedResult = new Result(generateHeader(null, databases, systems, false, 0), sortedDifferenceRows); + sortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ValueDifferences-" + fileNameSortedAll + ".csv"); + sortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ValueDifferences-" + fileNameSortedAll + ".csv"); + plot("values.gp", kind + "-" + run + "-ValueDifferences-" + fileNameSortedAll + ".csv", kind + "-" + run + "-ValueDifferences-" + fileNameSortedAll + ".eps"); + + List unsortedRowsPercentExecute = Data.filterBy(unsortedRowsExecute, percentIndexes); + int[] sortFilterPercent = Data.getSortFilter(unsortedRowsPercentExecute, 0); + List sortedDifferenceRowsPercent = Data.filterBy(unsortedDifferenceRowsPercent, sortFilterPercent); + Result sortedPercentResult = new Result(generateHeader(null, databases, systems, false, 0), sortedDifferenceRowsPercent); + sortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ValueDifferences-" + fileNameSortedPercent + ".csv"); + sortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ValueDifferences-" + fileNameSortedPercent + ".csv"); + plot("values.gp", kind + "-" + run + "-ValueDifferences-" + fileNameSortedPercent + ".csv", kind + "-" + run + "-ValueDifferences-" + fileNameSortedPercent + ".eps"); + } + } catch (IOException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (InterruptedException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (ExecutionException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } + } + + private static void outputExecuteTime(int[] percentIndexes, Log[][] logsForKind, String kind) { + try { + for (int i = 0; i < logsForKind.length; i++) { + String run = "run" + i; + if (i == 0) run = "average"; + Log[] logsRun = logsForKind[i]; + + List unsortedRows = Log.rowsExecuteTimes(logsRun, true); + Result unsortedResult = new Result(generateHeader(null, databases, systems, true, 0), unsortedRows); + unsortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedAll +".csv"); + unsortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedAll +".csv"); + plot("execute.gp", kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedAll +".csv", kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedAll +".eps"); + + List unsortedRowsPercent = Data.filterBy(unsortedRows, percentIndexes); + Result unsortedPercentResult = new Result(generateHeader(null, databases, systems, true, 0), unsortedRowsPercent); + unsortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedPercent + ".csv"); + unsortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedPercent + ".csv"); + plot("execute.gp", kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedPercent + ".csv", kind + "-" + run + "-ExecuteTimes-" + fileNameUnsortedPercent + ".eps"); + + List unsortedRowsNoSte = Log.rowsExecuteTimes(logsRun, false); + List unsortedRelativeRowsDirect = Data.relativeNumbers(unsortedRowsNoSte, 0); + Result unsortedRelativeResultDirect = new Result(generateHeader(null, databases, systems, false, 0), unsortedRelativeRowsDirect); + unsortedRelativeResultDirect.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-direct-relative-unsorted.csv"); + unsortedRelativeResultDirect.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ExecuteTimes-direct-relative-unsorted.csv"); + plot("relative.gp", kind + "-" + run + "-ExecuteTimes-direct-relative-unsorted.csv", kind + "-" + run + "-ExecuteTimes-direct-relative-unsorted.eps"); + List unsortedCdfRowsDirect = Data.cdf(unsortedRelativeRowsDirect, 0.0, 1000.0, 100000, true, 1000); + Result unsortedCdfResultDirect = new Result(generateRelativeHeader(new String[] { "cdf" }, databases, systems, 0 + 1, 1), unsortedCdfRowsDirect); + unsortedCdfResultDirect.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-direct-cdf.csv"); + plot("executeRelative.gp", kind + "-" + run + "-ExecuteTimes-direct-cdf.csv", kind + "-" + run + "-ExecuteTimes-direct-cdf.eps"); + + List unsortedRelativeRowsPINQ = Data.relativeNumbers(unsortedRowsNoSte, 1); + Result unsortedRelativeResultPINQ = new Result(generateHeader(null, databases, systems, false, 0), unsortedRelativeRowsPINQ); + unsortedRelativeResultPINQ.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-pinq-relative-unsorted.csv"); + unsortedRelativeResultPINQ.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ExecuteTimes-pinq-relative-unsorted.csv"); + plot("relative.gp", kind + "-" + run + "-ExecuteTimes-pinq-relative-unsorted.csv", kind + "-" + run + "-ExecuteTimes-pinq-relative-unsorted.eps"); + List unsortedCdfRowsPINQ = Data.cdf(unsortedRelativeRowsPINQ, 0.0, 1000.0, 100000, true, 1000); + Result unsortedCdfResultPINQ = new Result(generateRelativeHeader(new String[] { "cdf" }, databases, systems, 1 + 1, 1), unsortedCdfRowsPINQ); + unsortedCdfResultPINQ.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-pinq-cdf.csv"); + plot("executeRelative.gp", kind + "-" + run + "-ExecuteTimes-pinq-cdf.csv", kind + "-" + run + "-ExecuteTimes-pinq-cdf.eps"); + + int[] sortFilter = Data.getSortFilter(unsortedRows, 0); + List sortedRows = Data.filterBy(unsortedRows, sortFilter); + Result sortedResult = new Result(generateHeader(null, databases, systems, true, 0), sortedRows); + sortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-" + fileNameSortedAll +".csv"); + sortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ExecuteTimes-" + fileNameSortedAll + ".csv"); + plot("execute.gp", kind + "-" + run + "-ExecuteTimes-" + fileNameSortedAll + ".csv", kind + "-" + run + "-ExecuteTimes-" + fileNameSortedAll + ".eps"); + + int[] sortFilterPercent = Data.getSortFilter(unsortedRowsPercent, 0); + List sortedRowsPercent = Data.filterBy(unsortedRowsPercent, sortFilterPercent); + Result sortedPercentResult = new Result(generateHeader(null, databases, systems, true, 0), sortedRowsPercent); + sortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-ExecuteTimes-" + fileNameSortedPercent + ".csv"); + sortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-ExecuteTimes-" + fileNameSortedPercent + ".csv"); + plot("execute.gp", kind + "-" + run + "-ExecuteTimes-" + fileNameSortedPercent + ".csv", kind + "-" + run + "-ExecuteTimes-" + fileNameSortedPercent + ".eps"); + } + } catch (IOException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (InterruptedException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (ExecutionException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } + } + + private static void outputTotalTime(int[] percentIndexes, Log[][] logsForKind, String kind) { + try { + for (int i = 0; i < logsForKind.length; i++) { + String run = "run" + i; + if (i == 0) run = "average"; + Log[] logsRun = logsForKind[i]; + + List unsortedRows = Log.rowsTotalTimes(logsRun, true); + Result unsortedResult = new Result(generateHeader(null, databases, systems, true, 0), unsortedRows); + unsortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-" + fileNameUnsortedAll + ".csv"); + unsortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TotalTimes-" + fileNameUnsortedAll + ".csv"); + plot("execute.gp", kind + "-" + run + "-TotalTimes-" + fileNameUnsortedAll + ".csv", kind + "-" + run + "-TotalTimes-" + fileNameUnsortedAll + ".eps"); + + List unsortedRowsPercent = Data.filterBy(unsortedRows, percentIndexes); + Result unsortedPercentResult = new Result(generateHeader(null, databases, systems, true, 0), unsortedRowsPercent); + unsortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-" + fileNameUnsortedPercent + ".csv"); + unsortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TotalTimes-" + fileNameUnsortedPercent + ".csv"); + plot("execute.gp", kind + "-" + run + "-TotalTimes-" + fileNameUnsortedPercent + ".csv", kind + "-" + run + "-TotalTimes-" + fileNameUnsortedPercent + ".eps"); + + List unsortedRowsNoSte = Log.rowsTotalTimes(logsRun, false); + List unsortedRelativeRowsDirect = Data.relativeNumbers(unsortedRowsNoSte, 0); + Result unsortedRelativeResultDirect = new Result(generateHeader(null, databases, systems, false, 0), unsortedRelativeRowsDirect); + unsortedRelativeResultDirect.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-direct-relative-unsorted.csv"); + unsortedRelativeResultDirect.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TotalTimes-direct-relative-unsorted.csv"); + plot("relative.gp", kind + "-" + run + "-TotalTimes-direct-relative-unsorted.csv", kind + "-" + run + "-TotalTimes-direct-relative-unsorted.eps"); + List unsortedCdfRowsDirect = Data.cdf(unsortedRelativeRowsDirect, 0.0, 1000.0, 100000, true, 1000); + Result unsortedCdfResultDirect = new Result(generateRelativeHeader(new String[] { "cdf" }, databases, systems, 0 + 1, 1), unsortedCdfRowsDirect); + unsortedCdfResultDirect.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-direct-cdf.csv"); + plot("executeRelative.gp", kind + "-" + run + "-TotalTimes-direct-cdf.csv", kind + "-" + run + "-TotalTimes-direct-cdf.eps"); + + List unsortedRelativeRowsPINQ = Data.relativeNumbers(unsortedRowsNoSte, 1); + Result unsortedRelativeResultPINQ = new Result(generateHeader(null, databases, systems, false, 0), unsortedRelativeRowsPINQ); + unsortedRelativeResultPINQ.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-pinq-relative-unsorted.csv"); + unsortedRelativeResultPINQ.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TotalTimes-pinq-relative-unsorted.csv"); + plot("relative.gp", kind + "-" + run + "-TotalTimes-pinq-relative-unsorted.csv", kind + "-" + run + "-TotalTimes-pinq-relative-unsorted.eps"); + List unsortedCdfRowsPINQ = Data.cdf(unsortedRelativeRowsPINQ, 0.0, 1000.0, 100000, true, 1000); + Result unsortedCdfResultPINQ = new Result(generateRelativeHeader(new String[] { "cdf" }, databases, systems, 1 + 1, 1), unsortedCdfRowsPINQ); + unsortedCdfResultPINQ.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-pinq-cdf.csv"); + plot("executeRelative.gp", kind + "-" + run + "-TotalTimes-pinq-cdf.csv", kind + "-" + run + "-TotalTimes-pinq-cdf.eps"); + + int[] sortFilter = Data.getSortFilter(unsortedRows, 0); + List sortedRows = Data.filterBy(unsortedRows, sortFilter); + Result sortedResult = new Result(generateHeader(null, databases, systems, true, 0), sortedRows); + sortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-" + fileNameSortedAll + ".csv"); + sortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TotalTimes-" + fileNameSortedAll + ".csv"); + plot("execute.gp", kind + "-" + run + "-TotalTimes-" + fileNameSortedAll + ".csv", kind + "-" + run + "-TotalTimes-" + fileNameSortedAll + ".eps"); + + int[] sortFilterPercent = Data.getSortFilter(unsortedRowsPercent, 0); + List sortedRowsPercent = Data.filterBy(unsortedRowsPercent, sortFilterPercent); + Result sortedPercentResult = new Result(generateHeader(null, databases, systems, true, 0), sortedRowsPercent); + sortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TotalTimes-" + fileNameSortedPercent + ".csv"); + sortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TotalTimes-" + fileNameSortedPercent + ".csv"); + plot("execute.gp", kind + "-" + run + "-TotalTimes-" + fileNameSortedPercent + ".csv", kind + "-" + run + "-TotalTimes-" + fileNameSortedPercent + ".eps"); + } + } catch (IOException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (InterruptedException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (ExecutionException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } + } + + private static void outputPartitions(int[] percentIndexes, Log[][] logsForKind, String kind) { + try { + for (int i = 0; i < logsForKind.length; i++) { + String run = "run" + i; + if (i == 0) run = "average"; + Log[] logsRun = logsForKind[i]; + + List unsortedRows = Log.rowsTeardownPartitions(logsRun); + Result unsortedResult = new Result(generateHeader(null, databases, systems, false, 0), unsortedRows); + unsortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedAll + ".csv"); + unsortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedAll + ".csv"); + plot("partitions.gp", kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedAll + ".csv", kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedAll + ".eps"); + + List unsortedRowsPercent = Data.filterBy(unsortedRows, percentIndexes); + Result unsortedPercentResult = new Result(generateHeader(null, databases, systems, false, 0), unsortedRowsPercent); + unsortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedPercent + ".csv"); + unsortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedPercent + ".csv"); + plot("partitions.gp", kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedPercent + ".csv", kind + "-" + run + "-TeardownPartitions-" + fileNameUnsortedPercent + ".eps"); + + List unsortedRowsExecute = Log.rowsExecuteTimes(logsRun, true); + int[] sortFilter = Data.getSortFilter(unsortedRowsExecute, 0); + List sortedRows = Data.filterBy(unsortedRows, sortFilter); + Result sortedResult = new Result(generateHeader(null, databases, systems, false, 0), sortedRows); + sortedResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TeardownPartitions-" + fileNameSortedAll + ".csv"); + sortedResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TeardownPartitions-" + fileNameSortedAll + ".csv"); + plot("partitions.gp", kind + "-" + run + "-TeardownPartitions-" + fileNameSortedAll + ".csv", kind + "-" + run + "-TeardownPartitions-" + fileNameSortedAll + ".eps"); + + List unsortedRowsPercentExecute = Data.filterBy(unsortedRowsExecute, percentIndexes); + int[] sortFilterPercent = Data.getSortFilter(unsortedRowsPercentExecute, 0); + List sortedRowsPercent = Data.filterBy(unsortedRowsPercent, sortFilterPercent); + Result sortedPercentResult = new Result(generateHeader(null, databases, systems, false, 0), sortedRowsPercent); + sortedPercentResult.writeTo(outputPath + File.separator + kind + "-" + run + "-TeardownPartitions-" + fileNameSortedPercent + ".csv"); + sortedPercentResult.writeStatisticsTo(outputPath + File.separator + "statistics-" + kind + "-" + run + "-TeardownPartitions-" + fileNameSortedPercent + ".csv"); + plot("partitions.gp", kind + "-" + run + "-TeardownPartitions-" + fileNameSortedPercent + ".csv", kind + "-" + run + "-TeardownPartitions-" + fileNameSortedPercent + ".eps"); + } + } catch (IOException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (InterruptedException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (ExecutionException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } + } + + public static void main(String[] args) { + try { + System.out.println("Output STARTED"); + (new File(outputPath)).mkdirs(); + (new File(plotPath)).mkdirs(); + int[] percentIndexes = Data.generatePercent(5000, percentRate); + for (String kind : kinds) { + if (kind == null) continue; + Log[][] logs = Parser.generateLogs(databases, systems, kind, instancePath, runs, filterDummies, skip); + + Thread budgetUsedThread = new Thread(() -> outputBudgetUsedCDF(kind)); + Thread executeTimeThread = new Thread(() -> outputExecuteTime(percentIndexes, logs, kind)); + Thread totalTimeThread = new Thread(() -> outputTotalTime(percentIndexes, logs, kind)); + Thread partitionsThread = new Thread(() -> outputPartitions(percentIndexes, logs, kind)); + Thread valueDifferencesThread = new Thread(() -> outputValueDifferences(percentIndexes, logs, kind)); + + budgetUsedThread.start(); + executeTimeThread.start(); + totalTimeThread.start(); + partitionsThread.start(); + valueDifferencesThread.start(); + + budgetUsedThread.join(); + executeTimeThread.join(); + totalTimeThread.join(); + partitionsThread.join(); + valueDifferencesThread.join(); + } + System.out.println("Output FINISHED"); + } catch (NumberFormatException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (IOException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } catch (InterruptedException e) { + System.out.println("Something went wrong: " + e.getMessage()); + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/Parser.java b/source/java/logConverter/src/logConverter/Parser.java new file mode 100644 index 0000000000000000000000000000000000000000..2c255ebb81c1648bd69df7ebc9f05d357f4e0a39 --- /dev/null +++ b/source/java/logConverter/src/logConverter/Parser.java @@ -0,0 +1,152 @@ +package logConverter; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +public class Parser { + + public static Log[][] generateLogs(String[] databases, String[] systems, String kind, String instancePath, int runs, boolean filterDummies, int skip) throws NumberFormatException, IOException { + String[] systemPaths = systemPaths(databases, systems, kind, instancePath); + String[][] logFileNames = logFileNames(systemPaths, runs); + + int numLogs = runs; + if (numLogs == 1) numLogs = 2; + Log[][] logs = new Log[numLogs][logFileNames.length]; + for (int i = 0; i < logFileNames.length; i++) { + List list = new ArrayList<>(numLogs - 1); + for (int j = 1; j < numLogs; j++) { + if (logFileNames[i] == null) logs[j][i] = null; + else { + logs[j][i] = Parser.parseLogFile(logFileNames[i][j - 1], filterDummies); + list.add(logs[j][i]); + } + } + if (list.size() == 0) logs[0][i] = null; + else if (list.size() == 1) logs[0][i] = list.get(0); + else if (list.size() < 1 + 2 * skip) logs[0][i] = Log.average(list); + else logs[0][i] = Log.averageSkipExtremes(list, skip); + } + return logs; + } + + private static String[] systemPaths(String[] databases, String[] systems, String kind, String instancePath) { + String[] ret = new String[databases.length * systems.length]; + for (int i = 0; i < databases.length; i++) for (int j = 0; j < systems.length; j++) { + if (databases[i] != null && systems[j] != null) ret[i * systems.length + j] = instancePath + File.separator + databases[i] + "-" + systems[j] + "-" + kind; + else ret[i * systems.length + j] = null; + } + return ret; + } + + private static String[][] logFileNames(String[] systemPaths, int runs) { + String[][] ret = new String[systemPaths.length][]; + for (int i = 0; i < systemPaths.length; i++) { + if (systemPaths[i] == null) ret[i] = null; + else if (runs == 1) ret[i] = new String[] { systemPaths[i] + File.separator + "run0_results.csv" }; + else { + ret[i] = new String[runs - 1]; + for (int j = 1; j < runs; j++) ret[i][j - 1] = systemPaths[i] + File.separator + "run" + j + "_results.csv"; + } + } + return ret; + } + + private static Log parseLogFile(String fileName, boolean filterDummies) throws IOException, NumberFormatException { + if (fileName == null) throw new NullPointerException("fileName is null"); + FileReader fileReader = new FileReader(fileName); + BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + boolean aggregateHappened = false; + List lastAnalytic = new LinkedList<>(); + List lastAggregate = new LinkedList<>(); + List lastRange = new LinkedList<>(); + LogEntryNumberPartitions lastNumberPartitions = null; + LogEntryNumberPartitions currentLastNumberPartitions = null; + while ((line = bufferedReader.readLine()) != null) { + String[] words = line.split(";"); + switch (words[0]) { + case "Range": + if (aggregateHappened) { + lastRange = new LinkedList<>(); + aggregateHappened = false; + } + lastRange.add(new LogEntryRange(words[1], Long.parseLong(words[2]), Long.parseLong(words[3]))); + break; + case "NumberPartitions": + if (lastNumberPartitions != null) throw new IllegalStateException("cannot have new number of partitions without the old one being used by a primitive query"); + lastNumberPartitions = new LogEntryNumberPartitions(Integer.parseInt(words[1]), Integer.parseInt(words[2]), Integer.parseInt(words[3])); + break; + case "Aggregate": + currentLastNumberPartitions = new LogEntryNumberPartitions(1, 1, 1); + if (lastNumberPartitions != null) currentLastNumberPartitions = lastNumberPartitions; + double epsilon = (new Long(Long.parseLong(words[3]))).doubleValue(); + double value = Double.parseDouble(words[4]); + double setupTime = Double.parseDouble(words[5]); + double executeTime = Double.parseDouble(words[6]); + double bkExecuteTime = Double.parseDouble(words[7]); + double teardownTime = Double.parseDouble(words[8]); + LogEntryAggregate entry = new LogEntryAggregate(words[1], words[2], new Value(epsilon, 0.0), new Value(value, 0.0), new Value(setupTime, 0.0), new Value(executeTime, 0.0), + new Value(bkExecuteTime, 0.0), new Value(teardownTime, 0.0), new Value(setupTime + executeTime + teardownTime, 0.0), lastRange, currentLastNumberPartitions); + if (!(filterDummies && words[1].equals("Dummy"))) lastAggregate.add(entry); + lastNumberPartitions = null; + aggregateHappened = true; + break; + case "Analytic": + if (lastNumberPartitions != null) throw new IllegalStateException("cannot have number of partitions for analytic"); + lastAnalytic.add(new LogEntryAnalytic(Long.parseLong(words[1]), new Value(Double.parseDouble(words[2]), 0.0), words[3], lastAggregate)); + lastAggregate = new LinkedList<>(); + lastRange = new LinkedList<>(); + aggregateHappened = false; + break; + default: + throw new IllegalStateException("Unrecognized line: " + line); + } + } + fileReader.close(); + if (lastNumberPartitions != null) throw new IllegalStateException("cannot have number of partitions at end"); + if (lastRange.size() > 0) throw new IllegalStateException("cannot have ranges at end"); + if (lastAggregate.size() > 0) throw new IllegalStateException("cannot have aggregates at end"); + return new Log(fileName, lastAnalytic); + } + + public static List> generateBudgetUsedLists(String[] databases, String[] systems, String kind, String instancePath) throws NumberFormatException, IOException { + String[] systemPaths = systemPaths(databases, systems, kind, instancePath); + String[] budgetUsedFileNames = budgetUsedFileNames(systemPaths); + List> data = new ArrayList>(budgetUsedFileNames.length); + for (int i = 0; i < budgetUsedFileNames.length; i++) { + if (budgetUsedFileNames[i] == null) data.add(null); + else data.add(parseBudgetUsedFile(budgetUsedFileNames[i])); + } + return data; + } + + private static String[] budgetUsedFileNames(String[] systemPaths) { + String[] ret = new String[systemPaths.length]; + for (int i = 0; i < systemPaths.length; i++) { + if (systemPaths[i] == null) ret[i] = null; + else ret[i] = systemPaths[i] + File.separator + "run0_budgetUsed.csv"; + } + return ret; + } + + private static List parseBudgetUsedFile(String fileName) throws IOException, NumberFormatException { + if (fileName == null) throw new NullPointerException("fileName is null"); + FileReader fileReader = new FileReader(fileName); + BufferedReader bufferedReader = new BufferedReader(fileReader); + String line; + List counts = new LinkedList<>(); + while ((line = bufferedReader.readLine()) != null) { + String[] words = line.split(","); + double budget = Double.parseDouble(words[0]); + double number = new Integer(Integer.parseInt(words[1])).doubleValue(); + counts.add(new double[] { budget, number }); + } + fileReader.close(); + return counts; + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/Result.java b/source/java/logConverter/src/logConverter/Result.java new file mode 100644 index 0000000000000000000000000000000000000000..ffcb028ba4f350157e0c56f79437dcbd74fe5616 --- /dev/null +++ b/source/java/logConverter/src/logConverter/Result.java @@ -0,0 +1,105 @@ +package logConverter; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.List; + +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; + +public class Result { + + private String[] headerRow; + private List rows; + private DescriptiveStatistics[] statistics; + private int[] NPos; + private int[] NNeg; + private int[] NNaN; + + public Result(String[] headerRow, List rows) { + if (headerRow == null) throw new NullPointerException("headerRow is null"); + if (rows == null) throw new NullPointerException("rows is null"); + if (rows.isEmpty()) throw new IllegalArgumentException("rows is empty"); + for (double[] row : rows) { + if (row == null) throw new NullPointerException("row is null"); + if (row.length != headerRow.length) { + throw new IllegalArgumentException("all rows must have the same number of columns"); + } + } + for (String header : headerRow) if (header == null) throw new NullPointerException("header is null"); + this.headerRow = headerRow; + this.rows = rows; + GenerateStatistics(); + } + + public String[] getHeaderRow() { + return headerRow; + } + + public List getRows() { + return rows; + } + + public void GenerateStatistics() { + statistics = new DescriptiveStatistics[headerRow.length]; + NPos = new int[headerRow.length]; + NNeg = new int[headerRow.length]; + NNaN = new int[headerRow.length]; + for (int i = 0; i < headerRow.length; i++) statistics[i] = new DescriptiveStatistics(); + for (double[] row : rows) { + for (int i = 0; i < headerRow.length; i++) { + if (Double.isNaN(row[i])) { + NNaN[i]++; + continue; + } + statistics[i].addValue(row[i]); + if (row[i] < 0.0) NNeg[i]++; + else NPos[i]++; + } + } + } + + public void writeTo(String fileName) throws IOException { + if (fileName == null) throw new NullPointerException("fileName is null"); + PrintWriter pw = new PrintWriter(new FileWriter(fileName)); + for (int i = 0; i < headerRow.length; i++) pw.print(headerRow[i] + (i < headerRow.length - 1 ? "," : "")); + pw.println(); + for (double[] row : rows) { + for (int i = 0; i < headerRow.length; i++) pw.print(row[i] + (i < headerRow.length - 1 ? "," : "")); + pw.println(); + } + pw.close(); + } + + public void writeStatisticsTo(String fileName) throws IOException { + if (fileName == null) throw new NullPointerException("fileName is null"); + PrintWriter pw = new PrintWriter(new FileWriter(fileName)); + for (int i = 0; i < headerRow.length; i++) pw.print(headerRow[i] + ","); + pw.println("statistic"); + for (int i = 0; i < headerRow.length; i++) pw.print(NNeg[i] + ","); + pw.println("countNeg"); + for (int i = 0; i < headerRow.length; i++) pw.print(NPos[i] + ","); + pw.println("countPos"); + for (int i = 0; i < headerRow.length; i++) pw.print(NNaN[i] + ","); + pw.println("countNaN"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getN() + ","); + pw.println("count"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getMin() + ","); + pw.println("minimum"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getMax() + ","); + pw.println("maximum"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getMean() + ","); + pw.println("mean"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getPercentile(50.0) + ","); + pw.println("median"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getPercentile(90.0) + ","); + pw.println("90th"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getPercentile(95.0) + ","); + pw.println("95th"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getPercentile(99.0) + ","); + pw.println("99th"); + for (int i = 0; i < headerRow.length; i++) pw.print(statistics[i].getStandardDeviation() + ","); + pw.println("std"); + pw.close(); + } +} \ No newline at end of file diff --git a/source/java/logConverter/src/logConverter/Value.java b/source/java/logConverter/src/logConverter/Value.java new file mode 100644 index 0000000000000000000000000000000000000000..40f559485a336463eae3a28c69d0d1493fb0d1d6 --- /dev/null +++ b/source/java/logConverter/src/logConverter/Value.java @@ -0,0 +1,34 @@ +package logConverter; + +public class Value { + + private double value; + private double StE; + + public Value(double value, double StE) { + this.value = value; + this.StE = StE; + } + + public double getValue() { + return value; + } + + public double getStE() { + return StE; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof Value)) return false; + Value other = (Value) object; + if (value != other.value) return false; + if (StE != other.StE) return false; + return true; + } + + @Override + public String toString() { + return "value:" + value + ",StE:" + StE; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/pom.xml b/source/java/recordTransformer/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..a7f458d6ad272dd4cad0eb93cad0cebc1991a445 --- /dev/null +++ b/source/java/recordTransformer/pom.xml @@ -0,0 +1,19 @@ + + 4.0.0 + unitrax + recordTransformer + 0.0.1-SNAPSHOT + + src + + + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + + \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/DictionaryJoinerDefault.java b/source/java/recordTransformer/src/recordTransformer/core/DictionaryJoinerDefault.java new file mode 100644 index 0000000000000000000000000000000000000000..770526c92510419939cd7c301715bc1d59ac9856 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/DictionaryJoinerDefault.java @@ -0,0 +1,23 @@ +package recordTransformer.core; + +public class DictionaryJoinerDefault implements DictionaryJoinerI { + + private int[] includeIndexesA; + private int[] includeIndexesB; + + public DictionaryJoinerDefault(int[] includeIndexesA, int[] includeIndexesB) { + if (includeIndexesA == null) throw new NullPointerException("includeIndexesA is null"); + if (includeIndexesB == null) throw new NullPointerException("includeIndexesB is null"); + this.includeIndexesA = includeIndexesA; + this.includeIndexesB = includeIndexesB; + } + + @Override + public void joinDictionaries(Table tableA, Table tableB, TableBuilder tableBuilder) { + if (tableA == null) throw new NullPointerException("tableA is null"); + if (tableB == null) throw new NullPointerException("tableB is null"); + if (tableBuilder == null) throw new NullPointerException("tableBuilder is null"); + for (int i = 0; i < includeIndexesA.length; i++) tableBuilder.addDictionary(tableA.dictionaryAt(includeIndexesA[i])); + for (int i = 0; i < includeIndexesB.length; i++) tableBuilder.addDictionary(tableB.dictionaryAt(includeIndexesB[i])); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/DictionaryJoinerI.java b/source/java/recordTransformer/src/recordTransformer/core/DictionaryJoinerI.java new file mode 100644 index 0000000000000000000000000000000000000000..37f4fa3860d5cbdb953d4cc7074edca11daa53e4 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/DictionaryJoinerI.java @@ -0,0 +1,6 @@ +package recordTransformer.core; + +public interface DictionaryJoinerI { + + public void joinDictionaries(Table tableA, Table tableB, TableBuilder tableBuilder); +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/DummyLinePreparer.java b/source/java/recordTransformer/src/recordTransformer/core/DummyLinePreparer.java new file mode 100644 index 0000000000000000000000000000000000000000..3a8b1bdc54bb024f611eb4f598c4a45f97b8dc19 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/DummyLinePreparer.java @@ -0,0 +1,9 @@ +package recordTransformer.core; + +public class DummyLinePreparer implements LinePreparerI { + + @Override + public String prepare(String line) { + return line; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/LinePreparerI.java b/source/java/recordTransformer/src/recordTransformer/core/LinePreparerI.java new file mode 100644 index 0000000000000000000000000000000000000000..84c417440ae03dc21d313af6bbfec41fa2f47133 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/LinePreparerI.java @@ -0,0 +1,6 @@ +package recordTransformer.core; + +public interface LinePreparerI { + + public String prepare(String line); +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/Record.java b/source/java/recordTransformer/src/recordTransformer/core/Record.java new file mode 100644 index 0000000000000000000000000000000000000000..bbba7f41a9ab88ad21db6ab8bd660a61650e5cfd --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/Record.java @@ -0,0 +1,60 @@ +package recordTransformer.core; + +import java.text.ParseException; + +public class Record { + + public static Record parseRecord(String s, String delimiter, RecordFieldParserI[] fieldParsers, RecordFinalizerI recordFinalizer) { + if (s == null) throw new NullPointerException("s is null"); + if (delimiter == null) throw new NullPointerException("delimiter is null"); + if (fieldParsers == null) throw new NullPointerException("fieldParsers is null"); + String[] words = s.split(delimiter + "(?=([^\"]*\"[^\"]*\")*[^\"]*$)", -1); + if (words.length != fieldParsers.length) throw new IllegalArgumentException("not the correct number of field parsers, required " + words.length + ", was " + fieldParsers.length + " for input " + s); + RecordBuilder recordBuilder = new RecordBuilder(); + for (int i = 0; i < words.length; i++) { + try { + fieldParsers[i].parse(words[i], recordBuilder); + } catch (ParseException e) { + System.out.println("FAILED to parse! FieldParser #" + i + " failed to parse word " + words[i] + " of record " + s); + return null; + } + } + if (recordFinalizer != null) recordFinalizer.finalize(recordBuilder); + return recordBuilder.toRecord(); + } + + private double[] doubles; + private String[] strings; + + public Record(double[] doubles, String[] strings) { + if (doubles == null) throw new NullPointerException("doubles is null"); + if (strings == null) throw new NullPointerException("strings is null"); + this.doubles = doubles; + this.strings = strings; + }; + + public double doubleAt(int index) { + return doubles[index]; + } + + public String stringAt(int index) { + return strings[index]; + } + + public int numDoubles() { + return doubles.length; + } + + public int numStrings() { + return strings.length; + } + + @Override + public String toString() { + StringBuffer b = new StringBuffer(); + b.append("Record: "); + for (int i = 0; i < doubles.length; i++) b.append(doubleAt(i) + ","); + for (int i = 0; i < strings.length; i++) b.append(stringAt(i) + ","); + return b.toString(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordBuilder.java b/source/java/recordTransformer/src/recordTransformer/core/RecordBuilder.java new file mode 100644 index 0000000000000000000000000000000000000000..4c1fb072aeba9d88606cc6dbad3f6a82bbffec4b --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordBuilder.java @@ -0,0 +1,34 @@ +package recordTransformer.core; + +import java.util.LinkedList; + +public class RecordBuilder { + + private LinkedList doubleFields; + private LinkedList stringFields; + + public RecordBuilder() { + doubleFields = new LinkedList<>(); + stringFields = new LinkedList<>(); + } + + public void add(double value) { + doubleFields.add(value); + } + + public void add(String value) { + stringFields.add(value); + } + + public Record toRecord() { + double[] doubleFields = new double[this.doubleFields.size()]; + for (int i = 0; i < doubleFields.length; i++) { + doubleFields[i] = this.doubleFields.get(i); + } + String[] stringFields = new String[this.stringFields.size()]; + for (int i = 0; i < stringFields.length; i++) { + stringFields[i] = this.stringFields.get(i); + } + return new Record(doubleFields, stringFields); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDate.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDate.java new file mode 100644 index 0000000000000000000000000000000000000000..8000ede19a9fee67f44c84bf6c8000b6170ce90a --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDate.java @@ -0,0 +1,49 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.Locale; +import java.util.TimeZone; + +public class RecordFieldParserDate implements RecordFieldParserI { + + private String pattern; + private Date startDate; + + public RecordFieldParserDate(String pattern, Date startDate) { + if (pattern == null) throw new NullPointerException("pattern is null"); + this.pattern = pattern; + this.startDate = startDate; + } + + public RecordFieldParserDate(String pattern) { + this(pattern, null); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + SimpleDateFormat sdf = new SimpleDateFormat(pattern, Locale.ENGLISH); + if (startDate != null) sdf.set2DigitYearStart(startDate); + GregorianCalendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ENGLISH); + cal.clear(); + cal.setTime(sdf.parse(field)); + double value = new Long(cal.getTimeInMillis() / 1000l).doubleValue(); + recordBuilder.add(value); + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + return new HashMap[] { null }; + } + + @Override + public String report() { + return "RecordFieldParserDate: no report data available"; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDouble.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDouble.java new file mode 100644 index 0000000000000000000000000000000000000000..4c45a3c66923088221677716d2ee55784de61adc --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDouble.java @@ -0,0 +1,34 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.util.HashMap; + +public class RecordFieldParserDouble implements RecordFieldParserI { + + private int numberFormatExceptions = 0; + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + Double value = null; + try { + value = Double.parseDouble(field); + } catch(NumberFormatException e) { + numberFormatExceptions++; + throw new ParseException("Caught NumberFormatException", 0); + } + recordBuilder.add(value); + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + return new HashMap[] { null }; + } + + @Override + public String report() { + return "RecordFieldParserDouble: had " + numberFormatExceptions + " NumberFormatExceptions"; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDummy.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDummy.java new file mode 100644 index 0000000000000000000000000000000000000000..d4b6b1591e0f4a730ee21f734a6717a26d28eeb5 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserDummy.java @@ -0,0 +1,23 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.util.HashMap; + +public class RecordFieldParserDummy implements RecordFieldParserI { + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + recordBuilder.add(field); + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + return new HashMap[] {}; + } + + @Override + public String report() { + return "RecordFieldParserDummy: no report data available"; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerable.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerable.java new file mode 100644 index 0000000000000000000000000000000000000000..902c3c5e68107f5f41f086ab4de3a61cf0310a40 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerable.java @@ -0,0 +1,73 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; + +public class RecordFieldParserEnumerable implements RecordFieldParserI { + + private HashMap dictionary; + private int addedWords; + + public RecordFieldParserEnumerable() { + dictionary = new HashMap<>(); + dictionary.put("", 0.0); + addedWords = 0; + } + + public RecordFieldParserEnumerable(HashMap dictionary) { + this(); + if (dictionary == null) throw new NullPointerException("dictionary is null"); + for (HashMap.Entry entry : dictionary.entrySet()) for (String value : entry.getValue()) this.dictionary.put(value, entry.getKey()); + } + + public Double get(String key) { + if (key == null) throw new NullPointerException("key is null"); + return dictionary.get(key); + } + + public void put(String key, Double value) { + if (key == null) throw new NullPointerException("key is null"); + if (value == null) throw new NullPointerException("value is null"); + dictionary.put(key, value); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + Double value = get(field); + if (value == null) { + put(field, Collections.max(dictionary.values()) + 1.0); + addedWords++; + value = get(field); + } + recordBuilder.add(value); + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + HashMap> tmpDictionary = new HashMap<>(); + for(HashMap.Entry entry : dictionary.entrySet()){ + LinkedList list = tmpDictionary.get(entry.getValue()); + if (list == null) { + tmpDictionary.put(entry.getValue(), new LinkedList<>()); + list = tmpDictionary.get(entry.getValue()); + } + list.add(entry.getKey()); + } + HashMap retDictionary = new HashMap<>(); + for (HashMap.Entry> entry : tmpDictionary.entrySet()) { + String[] arr = new String[entry.getValue().size()]; + retDictionary.put(entry.getKey(), entry.getValue().toArray(arr)); + } + return new HashMap[] { retDictionary }; + } + + @Override + public String report() { + return "RecordFieldParserEnumerable: added " + addedWords + " to the dictionary"; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerableDate.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerableDate.java new file mode 100644 index 0000000000000000000000000000000000000000..d730e315c3b9e4c73e2c3e07d4374d7a488fa725 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerableDate.java @@ -0,0 +1,47 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.Locale; +import java.util.TimeZone; + +public class RecordFieldParserEnumerableDate extends RecordFieldParserEnumerable { + + private String pattern; + private double low; + private double high; + private Date startDate; + + public RecordFieldParserEnumerableDate(String pattern, Date startDate, double low, double high, HashMap dictionary) { + super(dictionary); + if (pattern == null) throw new NullPointerException("pattern is null"); + this.pattern = pattern; + this.startDate = startDate; + this.low = low; + this.high = high; + } + + public RecordFieldParserEnumerableDate(String pattern, double low, double high, HashMap dictionary) { + this(pattern, null, low, high, dictionary); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + Double value = get(field); + if (value == null) { + SimpleDateFormat sdf = new SimpleDateFormat(pattern, Locale.ENGLISH); + if (startDate != null) sdf.set2DigitYearStart(startDate); + GregorianCalendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ENGLISH); + cal.clear(); + cal.setTime(sdf.parse(field)); + value = new Long(cal.getTimeInMillis() / 1000l).doubleValue(); + if (value < low || high < value) throw new ParseException("value " + value + " outside allowed bounds (" + low + ", " + high + ")", 0); + } + recordBuilder.add(value); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerableDouble.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerableDouble.java new file mode 100644 index 0000000000000000000000000000000000000000..589abf9082a1ffc73ac12d814f15b2586720fe46 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserEnumerableDouble.java @@ -0,0 +1,46 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.util.HashMap; + +public class RecordFieldParserEnumerableDouble extends RecordFieldParserEnumerable { + + private double low; + private double high; + + private int count = 0; + private int numberFormatException = 0; + private int outOfBoundsException = 0; + + public RecordFieldParserEnumerableDouble(double low, double high, HashMap dictionary) { + super(dictionary); + this.low = low; + this.high = high; + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + count++; + Double value = get(field); + if (value == null) { + try { + value = Double.parseDouble(field); + } catch (NumberFormatException e) { + numberFormatException++; + throw new ParseException("Caught NumberFormatException: " + e.getMessage(), 0); + } + if (value < low || high < value) { + outOfBoundsException++; + throw new ParseException("value " + value + " outside allowed bounds (" + low + ", " + high + ")", 0); + } + } + recordBuilder.add(value); + } + + @Override + public String report() { + return "RecordFieldParserEnumerableDouble: " + (numberFormatException + outOfBoundsException) + " fails out of " + count + " (nfe " + numberFormatException + ", oobe " + outOfBoundsException + ") -- " + super.report(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserI.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserI.java new file mode 100644 index 0000000000000000000000000000000000000000..55d0ee03f84df30ad4b43a57d3b7e66e113ffea0 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserI.java @@ -0,0 +1,10 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.util.HashMap; + +public interface RecordFieldParserI { + public void parse(String field, RecordBuilder recordBuilder) throws ParseException; + public HashMap[] getDictionaries(); + public String report(); +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserPlusMinus.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserPlusMinus.java new file mode 100644 index 0000000000000000000000000000000000000000..26f132111de0beaff4bf3d5273afda77a940f2e3 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFieldParserPlusMinus.java @@ -0,0 +1,33 @@ +package recordTransformer.core; + +import java.text.ParseException; +import java.util.HashMap; + +public class RecordFieldParserPlusMinus extends RecordFieldParserEnumerable { + + private static HashMap plusMinusMap() { + HashMap plusMinusMap = new HashMap<>(); + plusMinusMap.put(0.0, new String[] { "" }); + plusMinusMap.put(1.0, new String[] { "-" }); + plusMinusMap.put(2.0, new String[] { "+-" }); + plusMinusMap.put(3.0, new String[] { "+" }); + plusMinusMap.put(4.0, new String[] { "2+" }); + return plusMinusMap; + } + + public RecordFieldParserPlusMinus() { + super(plusMinusMap()); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + Double value = get(field); + if (value == null) { + value = get(field.substring(0, 1)); + if (value == null) throw new ParseException("Cannot parse plus/minus from " + field, 0); + } + recordBuilder.add(value); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFinalizerBudget.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFinalizerBudget.java new file mode 100644 index 0000000000000000000000000000000000000000..41c34c78800065525e791286bb6ee1a2b1d756ad --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFinalizerBudget.java @@ -0,0 +1,23 @@ +package recordTransformer.core; + +import java.util.HashMap; + +public class RecordFinalizerBudget implements RecordFinalizerI { + + private double budget; + + public RecordFinalizerBudget(double budget) { + this.budget = budget; + } + + @Override + public void finalize(RecordBuilder recordBuilder) { + recordBuilder.add(budget); + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + return new HashMap[] { null }; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordFinalizerI.java b/source/java/recordTransformer/src/recordTransformer/core/RecordFinalizerI.java new file mode 100644 index 0000000000000000000000000000000000000000..a88fa8764343cfc1a0a2b5b9f4a284c97bb75bca --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordFinalizerI.java @@ -0,0 +1,8 @@ +package recordTransformer.core; + +import java.util.HashMap; + +public interface RecordFinalizerI { + public void finalize(RecordBuilder recordBuilder); + public HashMap[] getDictionaries(); +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordJoinerDefault.java b/source/java/recordTransformer/src/recordTransformer/core/RecordJoinerDefault.java new file mode 100644 index 0000000000000000000000000000000000000000..b8f9b9b60c81704fa4724889c9ef3dd3efefecbd --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordJoinerDefault.java @@ -0,0 +1,38 @@ +package recordTransformer.core; + +public class RecordJoinerDefault implements RecordJoinerI { + + private int[] includeDoublesA; + private int[] includeStringsA; + private int[] includeDoublesB; + private int[] includeStringsB; + + public RecordJoinerDefault(int[] includeDoublesA, int[] includeStringsA, int[] includeDoublesB, int[] includeStringsB) { + if (includeDoublesA == null) throw new NullPointerException("includeDoublesA is null"); + if (includeStringsA == null) throw new NullPointerException("includeStringsA is null"); + if (includeDoublesB == null) throw new NullPointerException("includeDoublesB is null"); + if (includeStringsB == null) throw new NullPointerException("includeStringsB is null"); + this.includeDoublesA = includeDoublesA; + this.includeStringsA = includeStringsA; + this.includeDoublesB = includeDoublesB; + this.includeStringsB = includeStringsB; + } + + @Override + public Record join(Record recordA, Record[] recordsB) { + if (recordA == null) throw new NullPointerException("recordA is null"); + if (recordsB != null && recordsB.length > 1) throw new IllegalArgumentException("recordsB cannot have more than one record"); + RecordBuilder recordBuilder = new RecordBuilder(); + for (int i = 0; i < includeDoublesA.length; i++) recordBuilder.add(recordA.doubleAt(includeDoublesA[i])); + for (int i = 0; i < includeStringsA.length; i++) recordBuilder.add(recordA.stringAt(includeStringsA[i])); + for (int i = 0; i < includeDoublesB.length; i++) { + if (recordsB == null) recordBuilder.add(0.0); + else recordBuilder.add(recordsB[0].doubleAt(includeDoublesB[i])); + } + for (int i = 0; i < includeStringsB.length; i++) { + if (recordsB == null) recordBuilder.add("\"\""); + else recordBuilder.add(recordsB[0].stringAt(includeStringsB[i])); + } + return recordBuilder.toRecord(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/RecordJoinerI.java b/source/java/recordTransformer/src/recordTransformer/core/RecordJoinerI.java new file mode 100644 index 0000000000000000000000000000000000000000..f3f1e5dff6c104f09eb3d9b95a939a1de6d475f7 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/RecordJoinerI.java @@ -0,0 +1,6 @@ +package recordTransformer.core; + +public interface RecordJoinerI { + + public Record join(Record recordA, Record[] recordsB); +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/Table.java b/source/java/recordTransformer/src/recordTransformer/core/Table.java new file mode 100644 index 0000000000000000000000000000000000000000..298aa01624ba5a9bc052fb642c9477c460efff75 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/Table.java @@ -0,0 +1,216 @@ +package recordTransformer.core; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.text.DecimalFormat; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; + +public class Table { + + public static Table join(String label, String[] columnLabels, int idIndex, int numDoubleColumns, Table tableA, Table tableB, + int matchColumnIndexA, int matchColumnIndexB, RecordJoinerI recordJoiner, DictionaryJoinerI dictionaryJoiner) { + if (label == null) throw new NullPointerException("label is null"); + if (tableA == null) throw new NullPointerException("tableA is null"); + if (tableB == null) throw new NullPointerException("tableB is null"); + if (columnLabels == null) throw new NullPointerException("columnLabels is null"); + if (recordJoiner == null) throw new NullPointerException("recordJoiner is null"); + if (dictionaryJoiner == null) throw new NullPointerException("dictionaryJoiner is null"); + TableBuilder tableBuilder = new TableBuilder(label, columnLabels, idIndex, numDoubleColumns); + HashSet matchedRecordIndexesB = new HashSet<>(); + for (int i = 0; i < tableA.numRecords(); i++) { + LinkedList matchRecordIndexesB = new LinkedList<>(); + for (int j = 0; j < tableB.numRecords(); j++) { + if (tableA.doubleAt(i, matchColumnIndexA) == tableB.doubleAt(j, matchColumnIndexB)) { + matchRecordIndexesB.add(j); + matchedRecordIndexesB.add(j); + } + } + if (matchRecordIndexesB.isEmpty()) tableBuilder.addRecord(recordJoiner.join(tableA.recordAt(i), null)); + else { + Record[] matches = new Record[matchRecordIndexesB.size()]; + for (int j = 0; j < matchRecordIndexesB.size(); j++) matches[j] = tableB.recordAt(matchRecordIndexesB.get(j)); + tableBuilder.addRecord(recordJoiner.join(tableA.recordAt(i), matches)); + } + } + for (int i = 0; i < tableB.numRecords(); i++) { + if (matchedRecordIndexesB.contains(i)) continue; + tableBuilder.addRecord(recordJoiner.join(null, new Record[] { tableB.recordAt(i) })); + } + dictionaryJoiner.joinDictionaries(tableA, tableB, tableBuilder); + return tableBuilder.toTable(); + } + + public static Table readFrom(BufferedReader reader, String delimiter, String label, String[] columnLabels, RecordFieldParserI[] fieldParsers, RecordFinalizerI recordFinalizer, int idIndex, int numDoubleColumns, LinePreparerI linePreparer) throws IOException { + if (reader == null) throw new NullPointerException("reader is null"); + if (delimiter == null) throw new NullPointerException("delimiter is null"); + if (label == null) throw new NullPointerException("label is null"); + if (columnLabels == null) throw new NullPointerException("columnLabels is null"); + if (fieldParsers == null) throw new NullPointerException("fieldParsers is null"); + if (linePreparer == null) throw new NullPointerException("linePreparer is null"); + TableBuilder tableBuilder = new TableBuilder(label, columnLabels, idIndex, numDoubleColumns); + LinkedList unparseableLines = new LinkedList<>(); + int linesParsed = 0; + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + line = linePreparer.prepare(line); + Record record = Record.parseRecord(line, delimiter, fieldParsers, recordFinalizer); + if (record != null) { + tableBuilder.addRecord(record); + linesParsed++; + } + else unparseableLines.add(line); + } + for (RecordFieldParserI fieldParser : fieldParsers) { + if (fieldParser.getDictionaries() == null) throw new NullPointerException("got null array"); + else for (HashMap dictionary : fieldParser.getDictionaries()) tableBuilder.addDictionary(dictionary); + } + if (recordFinalizer != null) { + if (recordFinalizer.getDictionaries() == null) throw new NullPointerException("got null array"); + else for (HashMap dictionary : recordFinalizer.getDictionaries()) tableBuilder.addDictionary(dictionary); + } + System.out.flush(); + System.out.print("Table: Successfully parsed " + linesParsed + " of " + (linesParsed + unparseableLines.size()) + " lines."); + if (unparseableLines.size() > 0) System.out.println(" The following " + unparseableLines.size() + " lines failed to parse:"); + else System.out.println(); + for (String line : unparseableLines) System.out.println(line); + int index = 0; + for (RecordFieldParserI fieldParser : fieldParsers) { + System.out.println(columnLabels[index++]); + System.out.println(fieldParser.report()); + } + return tableBuilder.toTable(); + } + + public static Table readFrom(String filename, String delimiter, String label, String[] columnLabels, RecordFieldParserI[] fieldParsers, RecordFinalizerI recordFinalizer, int idIndex, int numDoubleColumns, LinePreparerI linePreparer) { + if (filename == null) throw new NullPointerException("filename is null"); + if (delimiter == null) throw new NullPointerException("delimiter is null"); + if (label == null) throw new NullPointerException("label is null"); + if (columnLabels == null) throw new NullPointerException("columnLabels is null"); + if (fieldParsers == null) throw new NullPointerException("fieldParsers is null"); + if (linePreparer == null) throw new NullPointerException("linePreparer is null"); + try { + BufferedReader br = new BufferedReader(new FileReader(filename)); + br.readLine(); + return Table.readFrom(br, delimiter, label, columnLabels, fieldParsers, recordFinalizer, idIndex, numDoubleColumns, linePreparer); + } catch (FileNotFoundException e) { + e.printStackTrace(); + throw new RuntimeException("files should be in the provided path"); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("files should have content"); + } + } + + private String label; + private String[] columnLabels; + private Record[] records; + private HashMap[] dictionaries; + + public Table(String label, String[] columnLabels, Record[] records, HashMap[] dictionaries) { + if (label == null) throw new NullPointerException("label is null"); + if (columnLabels == null) throw new NullPointerException("columnLabels is null"); + if (records == null) throw new NullPointerException("records is null"); + if (dictionaries == null) throw new NullPointerException("dictionaries is null"); + if (records.length <= 0) throw new IllegalArgumentException("records is empty"); + if (columnLabels.length != records[0].numDoubles() + records[0].numStrings()) throw new IllegalArgumentException("not the required amount of columnlabels"); + if (dictionaries.length != records[0].numDoubles()) throw new IllegalArgumentException("not the required amount of dictionaries"); + for (Record record : records) if (record.numDoubles() != records[0].numDoubles() || record.numStrings() != records[0].numStrings()) throw new IllegalArgumentException("records have different number of fields"); + this.label = label; + this.columnLabels = columnLabels; + this.records = records; + this.dictionaries = dictionaries; + } + + public String getLabel() { + return label; + } + + public int numColumns() { + return columnLabels.length; + } + + public int numRecords() { + return records.length; + } + + public String columnLabelAt(int index) { + if (index < 0 || index > columnLabels.length - 1) throw new IndexOutOfBoundsException(); + return columnLabels[index]; + } + + public double doubleAt(int recordIndex, int doubleColumnIndex) { + if (recordIndex < 0 || recordIndex > records.length - 1) throw new IndexOutOfBoundsException(); + if (doubleColumnIndex < 0 || doubleColumnIndex > columnLabels.length - 1 || doubleColumnIndex > records[0].numDoubles() - 1) throw new IndexOutOfBoundsException(); + return records[recordIndex].doubleAt(doubleColumnIndex); + } + + public String stringAt(int recordIndex, int stringColumnIndex) { + if (recordIndex < 0 || recordIndex > records.length - 1) throw new IndexOutOfBoundsException(); + if (stringColumnIndex < 0 || stringColumnIndex > columnLabels.length - 1 || stringColumnIndex > records[0].numStrings() - 1) throw new IndexOutOfBoundsException(); + return records[recordIndex].stringAt(stringColumnIndex); + } + + public Record recordAt(int index) { + if (index < 0 || index > records.length - 1) throw new IndexOutOfBoundsException(); + return records[index]; + } + + public HashMap dictionaryAt(int index) { + if (index < 0 || index > columnLabels.length - 1) throw new IndexOutOfBoundsException(); + if (dictionaries[index] == null) return null; + return new HashMap(dictionaries[index]); + } + + public void writeRecordsTo(String filename, String delimiter) throws IOException { + if (filename == null) throw new NullPointerException("filename is null"); + if (delimiter == null) throw new NullPointerException("delimiter is null"); + PrintWriter pw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(filename), "UTF-8")); + for (int i = 0; i < columnLabels.length; i++) { + pw.print(columnLabels[i]); + if (i < columnLabels.length - 1) pw.print(delimiter); + } + pw.println(); + DecimalFormat format = new DecimalFormat("0.0"); + for (Record record : records) { + for (int i = 0; i < record.numDoubles(); i++) { + pw.print(format.format(record.doubleAt(i))); + if (i < record.numDoubles() + record.numStrings() - 1) pw.print(delimiter); + } + for (int i = 0; i < record.numStrings(); i++) { + pw.print("\"" + record.stringAt(i) + "\""); + if (i < record.numStrings() - 1) pw.print(delimiter); + } + pw.println(); + } + pw.close(); + } + + public void writeDictionariesTo(String filename, String delimiter) throws IOException { + if (filename == null) throw new NullPointerException("filename is null"); + if (delimiter == null) throw new NullPointerException("delimiter is null"); + PrintWriter pw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(filename), "UTF-8")); + for (int i = 0; i < records[0].numDoubles(); i++) { + pw.println(columnLabels[i]); + if (dictionaries[i] == null) continue; + LinkedList labels = new LinkedList<>(dictionaries[i].keySet()); + Collections.sort(labels); + for (Double label : labels) pw.println(label + delimiter + Arrays.toString(dictionaries[i].get(label))); + } + pw.close(); + } + + public void writeTo(String path) throws IOException { + if (path == null) throw new NullPointerException("path is null"); + writeRecordsTo(path + File.separator + getLabel() + "_records.csv", ";"); + writeDictionariesTo(path + File.separator + getLabel() + "_dictionaries.csv", ";"); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/core/TableBuilder.java b/source/java/recordTransformer/src/recordTransformer/core/TableBuilder.java new file mode 100644 index 0000000000000000000000000000000000000000..7d9ac281ee3864350b4140a27ef6f348c3a5db09 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/core/TableBuilder.java @@ -0,0 +1,67 @@ +package recordTransformer.core; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; + +public class TableBuilder { + + private String label; + private String[] columnLabels; + private LinkedList records; + private LinkedList skippedRecords; + private LinkedList> dictionaries; + private int idIndex; + private HashSet ids; + private int numDoubleColumns; + + public TableBuilder(String label, String[] columnLabels, int idIndex, int numDoubleColumns) { + if (label == null) throw new NullPointerException("label is null"); + if (columnLabels == null) throw new NullPointerException("columnLabels is null"); + this.label = label; + this.columnLabels = columnLabels; + records = new LinkedList<>(); + skippedRecords = new LinkedList<>(); + dictionaries = new LinkedList<>(); + this.idIndex = idIndex; + ids = new HashSet<>(); + this.numDoubleColumns = numDoubleColumns; + } + + public void addRecord(Record record) { + if (record == null) throw new NullPointerException("record is null"); + if (record.numDoubles() + record.numStrings() != columnLabels.length) throw new UnsupportedOperationException("record (" + record.numDoubles() + " + " + record.numStrings() + ") and columnLabels (" + columnLabels.length + ") have different length; " + record.toString() + " labels: " + Arrays.toString(columnLabels)); + if (ids.contains(record.doubleAt(idIndex))) { + skippedRecords.add(record); + return; + } + ids.add(record.doubleAt(idIndex)); + records.add(record); + } + + public void addDictionary(HashMap dictionary) { + dictionaries.add(dictionary); + } + + public Table toTable() { + if (records.isEmpty()) throw new UnsupportedOperationException("no records added"); + if (dictionaries.size() != numDoubleColumns) throw new UnsupportedOperationException("number of dictionaries (" + dictionaries.size() + ") do not match numDoubleColumns (" + numDoubleColumns + ")"); + Record[] r = new Record[records.size()]; + Iterator recordIterator = records.iterator(); + int index = 0; + while (recordIterator.hasNext()) r[index++] = recordIterator.next(); + @SuppressWarnings("unchecked") + HashMap[] d = new HashMap[dictionaries.size()]; + Iterator> dictionaryIterator = dictionaries.iterator(); + index = 0; + while (dictionaryIterator.hasNext()) d[index++] = dictionaryIterator.next(); + Table table = new Table(label, columnLabels, r, d); + System.out.print("TableBuilder: Successfully built table."); + if (skippedRecords.size() > 0) System.out.println(" The following records were skipped due to already existing IDs:"); + else System.out.println(); + for (Record record : skippedRecords) System.out.println(record.toString()); + return table; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialDictionaryJoinerAccountDisp.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialDictionaryJoinerAccountDisp.java new file mode 100644 index 0000000000000000000000000000000000000000..644b469b324045c8e48365a7da27dd0bace9ad11 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialDictionaryJoinerAccountDisp.java @@ -0,0 +1,25 @@ +package recordTransformer.datasets.financial; + +import recordTransformer.core.DictionaryJoinerI; +import recordTransformer.core.Table; +import recordTransformer.core.TableBuilder; + +public class FinancialDictionaryJoinerAccountDisp implements DictionaryJoinerI { + + private int[] includeIndexesA; + private int[] includeIndexesB; + + public FinancialDictionaryJoinerAccountDisp() { + includeIndexesA = new int[] {0, 1, 2, 3, 4}; + includeIndexesB = new int[] {2, 3, 4, 5, 6}; + } + @Override + public void joinDictionaries(Table tableA, Table tableB, TableBuilder tableBuilder) { + if (tableA == null) throw new NullPointerException("tableA is null"); + if (tableB == null) throw new NullPointerException("tableB is null"); + if (tableBuilder == null) throw new NullPointerException("tableBuilder is null"); + for (int i = 0; i < includeIndexesA.length; i++) tableBuilder.addDictionary(tableA.dictionaryAt(includeIndexesA[i])); + for (int i = 0; i < includeIndexesB.length; i++) tableBuilder.addDictionary(tableB.dictionaryAt(includeIndexesB[i])); + for (int i = 0; i < includeIndexesB.length; i++) tableBuilder.addDictionary(tableB.dictionaryAt(includeIndexesB[i])); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialDictionaryJoinerAccountTrans.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialDictionaryJoinerAccountTrans.java new file mode 100644 index 0000000000000000000000000000000000000000..0db568a20296da154c893b52ae5f28ffbd1fa43a --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialDictionaryJoinerAccountTrans.java @@ -0,0 +1,22 @@ +package recordTransformer.datasets.financial; + +import recordTransformer.core.DictionaryJoinerI; +import recordTransformer.core.Table; +import recordTransformer.core.TableBuilder; + +public class FinancialDictionaryJoinerAccountTrans implements DictionaryJoinerI { + + private int[] includeIndexesA; + + public FinancialDictionaryJoinerAccountTrans() { + includeIndexesA = new int[] {0, 1, 2, 3}; + } + @Override + public void joinDictionaries(Table tableA, Table tableB, TableBuilder tableBuilder) { + if (tableA == null) throw new NullPointerException("tableA is null"); + if (tableB == null) throw new NullPointerException("tableB is null"); + if (tableBuilder == null) throw new NullPointerException("tableBuilder is null"); + for (int i = 0; i < includeIndexesA.length; i++) tableBuilder.addDictionary(tableA.dictionaryAt(includeIndexesA[i])); + tableBuilder.addDictionary(null); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserBirthNumber.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserBirthNumber.java new file mode 100644 index 0000000000000000000000000000000000000000..6db4c3786ef0ddb31a0d9912702f3489ce5a2473 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserBirthNumber.java @@ -0,0 +1,74 @@ +package recordTransformer.datasets.financial; + +import java.text.ParseException; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Locale; +import java.util.TimeZone; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserI; + +public class FinancialRecordFieldParserBirthNumber implements RecordFieldParserI { + + private HashMap dictionary; + private int femaleCount = 0; + private int maleCount = 0; + + public FinancialRecordFieldParserBirthNumber() { + dictionary = new HashMap<>(); + dictionary.put("", 0.0); + dictionary.put("male", 1.0); + dictionary.put("female", 2.0); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + field = field.replaceAll("^\"|\"$", ""); + String sex = "male"; + int year = Integer.parseInt(field.substring(0, 2)); + int month = Integer.parseInt(field.substring(2, 4)); + int date = Integer.parseInt(field.substring(4, 6)); + maleCount++; + if (month > 50) { + sex = "female"; + month -= 50; + maleCount--; + femaleCount++; + } + GregorianCalendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ENGLISH); + cal.set(1900 + year, month - 1, date, 0, 0, 0); + long time = cal.getTimeInMillis() / 1000l; + double birthdate = new Long(time).doubleValue(); + recordBuilder.add(birthdate); + recordBuilder.add(dictionary.get(sex)); + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + HashMap> tmpDictionary = new HashMap<>(); + for(HashMap.Entry entry : dictionary.entrySet()){ + LinkedList list = tmpDictionary.get(entry.getValue()); + if (list == null) { + tmpDictionary.put(entry.getValue(), new LinkedList<>()); + list = tmpDictionary.get(entry.getValue()); + } + list.add(entry.getKey()); + } + HashMap retDictionary = new HashMap<>(); + for (HashMap.Entry> entry : tmpDictionary.entrySet()) { + String[] arr = new String[entry.getValue().size()]; + retDictionary.put(entry.getKey(), entry.getValue().toArray(arr)); + } + return new HashMap[] { null, retDictionary }; + } + + @Override + public String report() { + return "FinancialRecordFieldParserBirthNumber: females " + femaleCount + " males " + maleCount; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDate.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDate.java new file mode 100644 index 0000000000000000000000000000000000000000..8708e1a5711adeab627313d7e2b956b78a797fce --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDate.java @@ -0,0 +1,20 @@ +package recordTransformer.datasets.financial; + +import java.text.ParseException; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserDate; + +public class FinancialRecordFieldParserDate extends RecordFieldParserDate { + + public FinancialRecordFieldParserDate() { + super("yyMMdd"); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + super.parse(field.replaceAll("^\"|\"$", ""), recordBuilder); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDateTime.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDateTime.java new file mode 100644 index 0000000000000000000000000000000000000000..d361a36e6ea66047be803a9811ff7e09bead4010 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDateTime.java @@ -0,0 +1,20 @@ +package recordTransformer.datasets.financial; + +import java.text.ParseException; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserDate; + +public class FinancialRecordFieldParserDateTime extends RecordFieldParserDate { + + public FinancialRecordFieldParserDateTime() { + super("yyMMdd hh:mm:ss"); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + super.parse(field.replaceAll("^\"|\"$", ""), recordBuilder); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDouble.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDouble.java new file mode 100644 index 0000000000000000000000000000000000000000..1b1df5f1f230c098132490f06eb2b1e690770b50 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDouble.java @@ -0,0 +1,14 @@ +package recordTransformer.datasets.financial; + +import java.text.ParseException; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserDouble; + +public class FinancialRecordFieldParserDouble extends RecordFieldParserDouble { + + @Override + public void parse(String field, RecordBuilder record) throws ParseException { + super.parse(field.replaceAll("^\"|\"$", ""), record); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDoubleWithNull.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDoubleWithNull.java new file mode 100644 index 0000000000000000000000000000000000000000..2391cb3b4db86bbc27e9099ea49dd8e634df063b --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDoubleWithNull.java @@ -0,0 +1,16 @@ +package recordTransformer.datasets.financial; + +import java.text.ParseException; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserDouble; + +public class FinancialRecordFieldParserDoubleWithNull extends RecordFieldParserDouble { + + @Override + public void parse(String field, RecordBuilder record) throws ParseException { + String f = field.replaceAll("^\"|\"$", ""); + if (f.equals("")) f = "0.0"; + super.parse(f, record); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDoubleWithUnknown.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDoubleWithUnknown.java new file mode 100644 index 0000000000000000000000000000000000000000..b3579ae39066a5fdbb1742daeca58b6062508f59 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserDoubleWithUnknown.java @@ -0,0 +1,16 @@ +package recordTransformer.datasets.financial; + +import java.text.ParseException; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserDouble; + +public class FinancialRecordFieldParserDoubleWithUnknown extends RecordFieldParserDouble { + + @Override + public void parse(String field, RecordBuilder record) throws ParseException { + String f = field.replaceAll("^\"|\"$", ""); + if (f.equals("?")) f = "-1.0"; + super.parse(f, record); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserEnumerate.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserEnumerate.java new file mode 100644 index 0000000000000000000000000000000000000000..b5a157f7877c846da738ce9c6064bf818000a315 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordFieldParserEnumerate.java @@ -0,0 +1,23 @@ +package recordTransformer.datasets.financial; + +import java.text.ParseException; +import java.util.HashMap; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserEnumerable; + +public class FinancialRecordFieldParserEnumerate extends RecordFieldParserEnumerable { + + public FinancialRecordFieldParserEnumerate() { + super(); + } + + public FinancialRecordFieldParserEnumerate(HashMap dictionary) { + super(dictionary); + } + + @Override + public void parse(String field, RecordBuilder record) throws ParseException { + super.parse(field.replaceAll("^\"|\"$", ""), record); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordJoinerAccountDisp.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordJoinerAccountDisp.java new file mode 100644 index 0000000000000000000000000000000000000000..8ad28d40d9545da226d8ad78de7f184e8deb9905 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordJoinerAccountDisp.java @@ -0,0 +1,45 @@ +package recordTransformer.datasets.financial; + +import java.util.HashMap; + +import recordTransformer.core.Record; +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordJoinerI; + +public class FinancialRecordJoinerAccountDisp implements RecordJoinerI { + + private static final int[] INCLUDEINDEXESA = new int[] {0, 1, 2, 3, 4}; + private static final int[] INCLUDEINDEXESB = new int[] {2, 3, 4, 5, 6}; + + private HashMap dictionary; + + public FinancialRecordJoinerAccountDisp(HashMap dictionary) { + if (dictionary == null) throw new NullPointerException("dictionary is null"); + this.dictionary = dictionary; + } + + @Override + public Record join(Record recordA, Record[] recordsB) { + if (recordA == null) throw new NullPointerException("recordA is null"); + if (recordsB == null) throw new NullPointerException("recordsB is null"); + if (recordsB.length <= 0 || recordsB.length > 2) throw new IllegalArgumentException("recordsB must have 1-2 records"); + Record owner = recordsB[0]; + Record user = null; + if (recordsB.length > 1) { + user = recordsB[1]; + if (dictionary.get(user.doubleAt(1))[0].equals("OWNER")) { + Record tmp = user; + user = owner; + owner = tmp; + } + } + RecordBuilder recordBuilder = new RecordBuilder(); + for (int i = 0; i < INCLUDEINDEXESA.length; i++) recordBuilder.add(recordA.doubleAt(INCLUDEINDEXESA[i])); + for (int i = 0; i < INCLUDEINDEXESB.length; i++) recordBuilder.add(recordsB[0].doubleAt(INCLUDEINDEXESB[i])); + for (int i = 0; i < INCLUDEINDEXESB.length; i++) { + if (recordsB.length > 1) recordBuilder.add(recordsB[1].doubleAt(INCLUDEINDEXESB[i])); + else recordBuilder.add(0.0); + } + return recordBuilder.toRecord(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordJoinerAccountTrans.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordJoinerAccountTrans.java new file mode 100644 index 0000000000000000000000000000000000000000..b4c0eab2fcfad75576a17f6aad93f651ac419055 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordJoinerAccountTrans.java @@ -0,0 +1,32 @@ +package recordTransformer.datasets.financial; + +import recordTransformer.core.Record; +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordJoinerI; + +public class FinancialRecordJoinerAccountTrans implements RecordJoinerI { + + private int[] includeIndexesA; + + public FinancialRecordJoinerAccountTrans() { + includeIndexesA = new int[] {0, 1, 2, 3}; + } + + @Override + public Record join(Record recordA, Record[] recordsB) { + if (recordA == null) throw new NullPointerException("recordA is null"); + if (recordsB == null) throw new NullPointerException("recordsB is null"); + Record last = null; + if (recordsB.length > 0) last = recordsB[0]; + if (recordsB.length > 1) { + for (Record record : recordsB) { + if (record.doubleAt(2) > last.doubleAt(2)) last = record; + } + } + RecordBuilder recordBuilder = new RecordBuilder(); + for (int i = 0; i < includeIndexesA.length; i++) recordBuilder.add(recordA.doubleAt(includeIndexesA[i])); + if (recordsB.length > 0) recordBuilder.add(last.doubleAt(6)); + else recordBuilder.add(0.0); + return recordBuilder.toRecord(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordTransformer.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordTransformer.java new file mode 100644 index 0000000000000000000000000000000000000000..72f8751002dceef6077f49008f1e072ab906eb62 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialRecordTransformer.java @@ -0,0 +1,47 @@ +package recordTransformer.datasets.financial; + +import java.io.IOException; + +import recordTransformer.core.DictionaryJoinerDefault; +import recordTransformer.core.RecordJoinerDefault; +import recordTransformer.core.Table; + +public class FinancialRecordTransformer { + + public static void main(String[] args) { + FinancialTables tables = new FinancialTables("/home/user/unitrax/data/financial/original"); + Table dispClient = Table.join("dispClient", new String[] {"disp_id", "account_id", "type", "birthdate", "sex", "district_id"}, 0, 6, + tables.disp(), tables.client(), 1, 0, + new RecordJoinerDefault(new int[] {0, 2, 3}, new int[] {}, new int[] {1, 2, 3}, new int[] {}), + new DictionaryJoinerDefault(new int[] {0, 2, 3}, new int[] {1, 2, 3})); + Table dispClientCard = Table.join("dispClientCard", new String[] {"account_id", "type", "birthdate", "sex", "district_id", "card_type", "card_issued"}, 0, 7, + dispClient, tables.card(), 0, 1, + new RecordJoinerDefault(new int[] {1, 2, 3, 4, 5}, new int[] {}, new int[] {2, 3}, new int[] {}), + new DictionaryJoinerDefault(new int[] {1, 2, 3, 4, 5}, new int[] {2, 3})); + Table accountTrans = Table.join("account", new String[] {"account_id", "account_district_id", "account_frequency", "account_date", "account_balance"}, 0, 5, + tables.account(), tables.trans(), 0, 1, + new FinancialRecordJoinerAccountTrans(), + new FinancialDictionaryJoinerAccountTrans()); + Table accountOwnerUser = Table.join("accountOwnerUser", new String[] {"account_id", "account_district_id", "account_frequency", "account_date", "account_balance", + "owner_birthdate", "owner_sex", "owner_district_id", "owner_card_type", "owner_card_issued", + "user_birthdate", "user_sex", "user_district_id", "user_card_type", "user_card_issued"}, 0, 15, + accountTrans, dispClientCard, 0, 0, + new FinancialRecordJoinerAccountDisp(tables.disp().dictionaryAt(3)), + new FinancialDictionaryJoinerAccountDisp()); + Table account = Table.join("account", new String[] {"account_id", "account_district_id", "account_frequency", "account_date", "account_balance", + "owner_birthdate", "owner_sex", "owner_district_id", "owner_card_type", "owner_card_issued", + "user_birthdate", "user_sex", "user_district_id", "user_card_type", "user_card_issued", + "loan_date", "loan_amount", "loan_duration", "loan_payments", "loan_status"}, 0, 20, + accountOwnerUser, tables.loan(), 0, 1, + new RecordJoinerDefault(new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, new int[] {}, new int[] {2, 3, 4, 5, 6}, new int[] {}), + new DictionaryJoinerDefault(new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, new int[] {2, 3, 4, 5, 6})); + try { + account.writeTo("/home/user/unitrax/data/financial/transformed"); + tables.district().writeTo("/home/user/unitrax/data/financial/transformed"); + tables.order().writeTo("/home/user/unitrax/data/financial/transformed"); + tables.trans().writeTo("/home/user/unitrax/data/financial/transformed"); + } catch (IOException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialTables.java b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialTables.java new file mode 100644 index 0000000000000000000000000000000000000000..97793ac5d84c55de891c7d6eaff15fbc260213b8 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/financial/FinancialTables.java @@ -0,0 +1,147 @@ +package recordTransformer.datasets.financial; + +import java.io.File; +import java.io.IOException; + +import recordTransformer.core.DummyLinePreparer; +import recordTransformer.core.RecordFieldParserI; +import recordTransformer.core.Table; + +public class FinancialTables { + + private Table loan = null; + private Table order = null; + private Table trans = null; + private Table district = null; + private Table account = null; + private Table card = null; + private Table disp = null; + private Table client = null; + + public FinancialTables(String path) { + String[] loanColumnLabels = new String[] {"loan_id", "account_id", "date", "amount", "duration", "payments", "status"}; + loan = Table.readFrom(path + File.separator + "loan.asc", "loan", ";", loanColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDate(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserEnumerate()}, + null, 0, 7, new DummyLinePreparer()); + String[] orderColumnLabels = new String[] {"order_id", "account_id", "bank_to", "account_to", "amount", "k_symbol"}; + FinancialRecordFieldParserEnumerate parser_bank_to = new FinancialRecordFieldParserEnumerate(); + FinancialRecordFieldParserEnumerate parser_k_symbol = new FinancialRecordFieldParserEnumerate(); + order = Table.readFrom(path + File.separator + "order.asc", "order", ";", orderColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + parser_bank_to, + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + parser_k_symbol}, + null, 0, 6, new DummyLinePreparer()); + String[] transColumnLabels = new String[] {"trans_id", "account_id", "date", "type", "operation", "amount", "balance", "k_symbol", "bank", "account"}; + trans = Table.readFrom(path + File.separator + "trans.asc", "trans", ";", transColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDate(), + new FinancialRecordFieldParserEnumerate(), + new FinancialRecordFieldParserEnumerate(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserEnumerate(parser_k_symbol.getDictionaries()[0]), + new FinancialRecordFieldParserEnumerate(parser_bank_to.getDictionaries()[0]), + new FinancialRecordFieldParserDoubleWithNull()}, + null, 0, 10, new DummyLinePreparer()); + String[] districtColumnLabels = new String[] {"district_id", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "A9", "A10", "A11", "A12", "A13", "A14", "A15", "A16"}; + district = Table.readFrom(path + File.separator + "district.asc", "district", ";", districtColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserEnumerate(), + new FinancialRecordFieldParserEnumerate(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDoubleWithUnknown(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDoubleWithUnknown(), + new FinancialRecordFieldParserDouble()}, + null, 0, 16, new DummyLinePreparer()); + String[] accountColumnLabels = new String[] {"account_id", "district_id", "frequency", "date"}; + account = Table.readFrom(path + File.separator + "account.asc", "account", ";", accountColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserEnumerate(), + new FinancialRecordFieldParserDate()}, + null, 0, 4, new DummyLinePreparer()); + String[] cardColumnLabels = new String[] {"card_id", "disp_id", "type", "issued"}; + card = Table.readFrom(path + File.separator + "card.asc", "card", ";", cardColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserEnumerate(), + new FinancialRecordFieldParserDateTime()}, + null, 0, 4, new DummyLinePreparer()); + String[] dispColumnLabels = new String[] {"disp_id", "client_id", "account_id", "type"}; + disp = Table.readFrom(path + File.separator + "disp.asc", "disp", ";", dispColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserEnumerate()}, + null, 0, 4, new DummyLinePreparer()); + String[] clientColumnLabels = new String[] {"client_id", "birthdate", "sex", "district_id"}; + client = Table.readFrom(path + File.separator + "client.asc", "client", ";", clientColumnLabels, new RecordFieldParserI[] { + new FinancialRecordFieldParserDouble(), + new FinancialRecordFieldParserBirthNumber(), + new FinancialRecordFieldParserDouble()}, + null, 0, 4, new DummyLinePreparer()); + } + + public Table loan() { + return loan; + } + + public Table order() { + return order; + } + + public Table trans() { + return trans; + } + + public Table district() { + return district; + } + + public Table account() { + return account; + } + + public Table card() { + return card; + } + + public Table disp() { + return disp; + } + + public Table client() { + return client; + } + + public void writeAllTablesTo(String path) throws IOException { + if (path == null) throw new NullPointerException("path is null"); + loan.writeTo(path); + order.writeTo(path); + trans.writeTo(path); + district.writeTo(path); + account.writeTo(path); + card.writeTo(path); + disp.writeTo(path); + client.writeTo(path); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalDictionaryJoinerPatientSpecialLab.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalDictionaryJoinerPatientSpecialLab.java new file mode 100644 index 0000000000000000000000000000000000000000..13d3ae5d5f25a451a9582c73798220cb61c839d7 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalDictionaryJoinerPatientSpecialLab.java @@ -0,0 +1,33 @@ +package recordTransformer.datasets.medical; + +import recordTransformer.core.DictionaryJoinerI; +import recordTransformer.core.Table; +import recordTransformer.core.TableBuilder; + +public class MedicalDictionaryJoinerPatientSpecialLab implements DictionaryJoinerI { + + private static final int[] DIAGNOSISINDEXESA = new int[] {6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22}; + + @Override + public void joinDictionaries(Table tableA, Table tableB, TableBuilder tableBuilder) { + if (tableA == null) throw new NullPointerException("tableA is null"); + if (tableB == null) throw new NullPointerException("tableB is null"); + if (tableBuilder == null) throw new NullPointerException("tableBuilder is null"); + + // id, sex, date_of_birth, date_of_initial_data_collection, date_of_first_visit + for (int i : new int[] { 0, 1, 2, 3, 4 }) tableBuilder.addDictionary(tableA.dictionaryAt(i)); + // date_of_examination + tableBuilder.addDictionary(tableB.dictionaryAt(1)); + // admission + tableBuilder.addDictionary(tableA.dictionaryAt(5)); + // diagnosis_sjs, diagnosis_sle, diagnosis_behcet, diagnosis_pss, diagnosis_ra, diagnosis_mctd, diagnosis_aps, diagnosis_pm, diagnosis_dm, diagnosis_pn, diagnosis_fuo, diagnosis_aortitis, diagnosis_ip, diagnosis_mra, diagnosis_collagen, diagnosis_raynaud, diagnosis_other + for (int i = 0; i < DIAGNOSISINDEXESA.length; i++) { + int indexA = DIAGNOSISINDEXESA[i]; + tableBuilder.addDictionary(tableA.dictionaryAt(indexA)); + } + // acl_iga, acl_igg, acl_igm, ana, ana_pattern_d, ana_pattern_n, ana_pattern_p, ana_pattern_s, kct, rvvt, lac, thrombosis + for (int i : new int[] { 9, 2, 3, 4, 5, 6, 7, 8, 27, 28, 29, 30 }) tableBuilder.addDictionary(tableB.dictionaryAt(i)); + // budget + tableBuilder.addDictionary(tableA.dictionaryAt(23)); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalLinePreparerLaboratory.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalLinePreparerLaboratory.java new file mode 100644 index 0000000000000000000000000000000000000000..3c6071073b6cdd403eb408cf216d9050cde754fa --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalLinePreparerLaboratory.java @@ -0,0 +1,30 @@ +package recordTransformer.datasets.medical; + +import recordTransformer.core.LinePreparerI; + +public class MedicalLinePreparerLaboratory implements LinePreparerI { + + private String delimiter; + private int numFields; + private double index; + + public MedicalLinePreparerLaboratory(String delimiter, int numFields) { + if (delimiter == null) throw new NullPointerException("delimiter is null"); + this.delimiter = delimiter; + this.numFields = numFields; + index = 0.0; + } + + @Override + public String prepare(String line) { + String[] words = line.split("," + "(?=([^\"]*\"[^\"]*\")*[^\"]*$)", -1); + StringBuffer b = new StringBuffer(); + b.append(index++); + b.append(delimiter); + for (int i = 0; i < numFields - 1; i++) { + if (words.length > i) b.append(words[i]); + if (i < numFields - 2) b.append(delimiter); + } + return b.toString(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserAnaPattern.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserAnaPattern.java new file mode 100644 index 0000000000000000000000000000000000000000..09f9aa12386efe5be752f519c61c5a75f2ede1c9 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserAnaPattern.java @@ -0,0 +1,40 @@ +package recordTransformer.datasets.medical; + +import java.text.ParseException; +import java.util.Arrays; +import java.util.HashMap; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserI; + +public class MedicalRecordFieldParserAnaPattern implements RecordFieldParserI { + + public static final String[] ANAPATTERNSTRINGS = new String[] {"D", "N", "P", "S"}; + public static final String[] ANACOLUMNNAMES = new String[] {"ana_pattern_d", "ana_pattern_n", "ana_pattern_p", "ana_pattern_s"}; + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + field = field.replaceAll("^\"|\"$", ""); + Double[] anaPatternValues = new Double[ANAPATTERNSTRINGS.length]; + Arrays.fill(anaPatternValues, 0.0); + for (int i = 0; i < ANAPATTERNSTRINGS.length; i++) { + if (field.contains(ANAPATTERNSTRINGS[i])) anaPatternValues[i] = 1.0; + } + for (int i = 0; i < anaPatternValues.length; i++) recordBuilder.add(anaPatternValues[i]); + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + HashMap[] dictionaries = new HashMap[ANAPATTERNSTRINGS.length]; + Arrays.fill(dictionaries, null); + return dictionaries; + } + + @Override + public String report() { + return "MedicalRecordFieldParserAnaPattern: no report data available"; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDateYearFirst.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDateYearFirst.java new file mode 100644 index 0000000000000000000000000000000000000000..48e723085afd238192196afbc7469a306be4fe2d --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDateYearFirst.java @@ -0,0 +1,70 @@ +package recordTransformer.datasets.medical; + +import java.text.ParseException; +import java.util.Date; +import java.util.HashMap; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserEnumerableDate; +import recordTransformer.core.RecordFieldParserI; + +public class MedicalRecordFieldParserDateYearFirst implements RecordFieldParserI { + + private double low; + private double high; + private double nullValue; + private HashMap dictionary; + + private int count = 0; + private int e1Count = 0; + private int e2Count = 0; + private int e3Count = 0; + + public MedicalRecordFieldParserDateYearFirst(double nullValue, double low, double high) { + if (low <= nullValue && nullValue <= high) throw new IllegalArgumentException("nullValue " + nullValue + " between low " + low + " and high " + high); + this.low = low; + this.high = high; + this.nullValue = nullValue; + dictionary = new HashMap<>(); + dictionary.put(nullValue, new String[] { "" }); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + count++; + Date startDate = new Date(-2208988800000l); + RecordFieldParserEnumerableDate dateParser; + try { + dateParser = new RecordFieldParserEnumerableDate("yyyy/M/d", startDate, low, high, dictionary); + dateParser.parse(field, recordBuilder); + } catch (ParseException pe1) { + e1Count++; + try { + dateParser = new RecordFieldParserEnumerableDate("yy.M.d", startDate, low, high, dictionary); + dateParser.parse(field, recordBuilder); + } catch (ParseException pe2) { + e2Count++; + try { + dateParser = new RecordFieldParserEnumerableDate("yyMMdd", startDate, low, high, dictionary); + dateParser.parse(field, recordBuilder); + } catch (ParseException pe3) { + e3Count++; + recordBuilder.add(nullValue); + } + } + } + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + return new HashMap[] { dictionary }; + } + + @Override + public String report() { + return "MedicalRecordFieldParserDateYearFirst: " + e3Count + " fails out of " + count + " (e1 " + e1Count + " e2 " + e2Count + " e3 " + e3Count + ")"; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDiagnosis.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDiagnosis.java new file mode 100644 index 0000000000000000000000000000000000000000..f9825fced02c6f7391be2570a531182b46e6ea36 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDiagnosis.java @@ -0,0 +1,71 @@ +package recordTransformer.datasets.medical; + +import java.text.ParseException; +import java.util.Arrays; +import java.util.HashMap; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserI; + +public class MedicalRecordFieldParserDiagnosis implements RecordFieldParserI { + + // SJS 359, SLE 310, BEHCET 103, PSS 94, RA 79, MCTD 63, APS 53, PM 51, DM 39, PN 23, FUO 22, AORTITIS 16, (IP 13), MRA 12, COLLAGEN 12, RAYNAUD 10 + public static final String[] DIAGNOSISSTRINGS = new String[] {"sjs", "sle", "behcet", "pss", "ra", "mctd", "aps", "pm", "dm", "pn", "fuo", "aortitis", "ip", "mra", "collagen", "raynaud", "other"}; + public static final String[] DIAGNOSISCOLUMNNAMES = new String[] {"diagnosis_sjs", "diagnosis_sle", "diagnosis_behcet", "diagnosis_pss", "diagnosis_ra", "diagnosis_mctd", "diagnosis_aps", "diagnosis_pm", "diagnosis_dm", "diagnosis_pn", "diagnosis_fuo", "diagnosis_aortitis", "diagnosis_ip", "diagnosis_mra", "diagnosis_collagen", "diagnosis_raynaud", "diagnosis_other"}; + + private static final String GROUPSEPARATOR = Character.toString((char) 29); + + private int count = 0; + private int otherCount = 0; + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + count++; + field = field.replaceAll("^\"|\"$", ""); + Double[] diagnosisValues = new Double[DIAGNOSISSTRINGS.length]; + Arrays.fill(diagnosisValues, 0.0); + String[] words = field.split("[(), \\+\\-" + GROUPSEPARATOR + "]+"); + for (int i = 0; i < words.length; i++) { + String word = words[i]; + while (word != null) word = detectAndRemove(word, diagnosisValues); + } + for (int i = 0; i < diagnosisValues.length; i++) recordBuilder.add(diagnosisValues[i]); + if (diagnosisValues[diagnosisValues.length - 1] > 0.0) otherCount++; + recordBuilder.add(field); + } + + private String detectAndRemove(String word, Double[] diagnosisValues) { + if (word == null) throw new NullPointerException("word is null"); + if (diagnosisValues == null) throw new NullPointerException("diagnosisValues is null"); + if (word.isEmpty()) return null; + int index = -1; + int length = -1; + for (int i = 0; i < DIAGNOSISSTRINGS.length; i++) { + if (word.toLowerCase().startsWith(DIAGNOSISSTRINGS[i]) && DIAGNOSISSTRINGS[i].length() > length) { + index = i; + length = DIAGNOSISSTRINGS[i].length(); + } + } + if (index > -1) { + diagnosisValues[index] = 1.0; + return word.toLowerCase().replaceAll("^" + DIAGNOSISSTRINGS[index], ""); + } + if (!word.isEmpty()) diagnosisValues[diagnosisValues.length - 1] = 1.0; + return null; + } + + @SuppressWarnings("unchecked") + @Override + public HashMap[] getDictionaries() { + HashMap[] dictionaries = new HashMap[DIAGNOSISSTRINGS.length]; + Arrays.fill(dictionaries, null); + return dictionaries; + } + + @Override + public String report() { + return "MedicalRecordFieldParserDiagnosis: " + otherCount + " out of " + count + " fields had other diagnosis"; + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDummy.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDummy.java new file mode 100644 index 0000000000000000000000000000000000000000..2606382e2e7e5cdf912c19e5a7a78545246a6ea8 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserDummy.java @@ -0,0 +1,14 @@ +package recordTransformer.datasets.medical; + +import java.text.ParseException; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserDummy; + +public class MedicalRecordFieldParserDummy extends RecordFieldParserDummy{ + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + super.parse(field.replaceAll("^\"|\"$", ""), recordBuilder); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserEnumerableDouble.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserEnumerableDouble.java new file mode 100644 index 0000000000000000000000000000000000000000..f2ce6f08a1be72bb9ac5883174261fb4a1486be8 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserEnumerableDouble.java @@ -0,0 +1,52 @@ +package recordTransformer.datasets.medical; + +import java.text.ParseException; +import java.util.HashMap; +import java.util.LinkedList; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserEnumerableDouble; + +public class MedicalRecordFieldParserEnumerableDouble extends RecordFieldParserEnumerableDouble{ + + private static HashMap doubleDictionary(double nullValue) { + HashMap doubleDictionary = new HashMap<>(); + doubleDictionary.put(nullValue, new String[] { "" }); + return doubleDictionary; + } + + private double nullValue; + private int count = 0; + private int exceptionCount = 0; + LinkedList exceptionMessages = new LinkedList<>(); + + public MedicalRecordFieldParserEnumerableDouble(double low, double high, double nullValue) { + this(low, high, nullValue, doubleDictionary(nullValue)); + } + + public MedicalRecordFieldParserEnumerableDouble(double low, double high, double nullValue, HashMap dictionary) { + super(low, high, dictionary); + this.nullValue = nullValue; + } + + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + count++; + try { + super.parse(field, recordBuilder); + } catch (ParseException e) { + exceptionMessages.add(e.getMessage()); + exceptionCount++; + recordBuilder.add(nullValue); + } + } + + @Override + public String report() { + StringBuffer b = new StringBuffer(); + b.append("MedicalRecordFieldParserEnumerableDouble: parse exceptions " + exceptionCount + " out of " + count + " -- " + super.report()); + for (String e : exceptionMessages) b.append("\n" + e); + return b.toString(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserPlusMinus.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserPlusMinus.java new file mode 100644 index 0000000000000000000000000000000000000000..d7f4ffc0dbaa4a189a73e23572bcd5247f46be76 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserPlusMinus.java @@ -0,0 +1,28 @@ +package recordTransformer.datasets.medical; + +import java.text.ParseException; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserPlusMinus; + +public class MedicalRecordFieldParserPlusMinus extends RecordFieldParserPlusMinus { + + private int count = 0; + private int exceptionCount = 0; + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + count++; + try { + super.parse(field, recordBuilder); + } catch (ParseException e) { + exceptionCount++; + recordBuilder.add(0.0); + } + } + + @Override + public String report() { + return "MedicalRecordFieldParserPlusMinus: parse exceptions " + exceptionCount + " out of " + count + " -- " + super.report(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserSex.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserSex.java new file mode 100644 index 0000000000000000000000000000000000000000..32dd6d80a2c35b1ebd5e1dcb53afbc79a978a881 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordFieldParserSex.java @@ -0,0 +1,43 @@ +package recordTransformer.datasets.medical; + +import java.text.ParseException; +import java.util.HashMap; + +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordFieldParserEnumerable; + +public class MedicalRecordFieldParserSex extends RecordFieldParserEnumerable { + + private static HashMap sexMap() { + HashMap sexMap = new HashMap<>(); + sexMap.put(0.0, new String[] { "" }); + sexMap.put(1.0, new String[] { "F" }); + sexMap.put(2.0, new String[] { "M" }); + return sexMap; + } + + private int count = 0; + private int exceptionCount = 0; + + public MedicalRecordFieldParserSex() { + super(sexMap()); + } + + @Override + public void parse(String field, RecordBuilder recordBuilder) throws ParseException { + if (field == null) throw new NullPointerException("field is null"); + if (recordBuilder == null) throw new NullPointerException("recordBuilder is null"); + count++; + Double value = get(field); + if (value == null) { + exceptionCount++; + throw new ParseException("Cannot parse sex from " + field, 0); + } + recordBuilder.add(value); + } + + @Override + public String report() { + return "MedicalRecordFieldParserSex: parse exceptions " + exceptionCount + " out of " + count + " -- " + super.report(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordJoinerPatientSpecialLab.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordJoinerPatientSpecialLab.java new file mode 100644 index 0000000000000000000000000000000000000000..f11c40b51261427712728091ac9373ed307d3832 --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordJoinerPatientSpecialLab.java @@ -0,0 +1,118 @@ +package recordTransformer.datasets.medical; + +import recordTransformer.core.Record; +import recordTransformer.core.RecordBuilder; +import recordTransformer.core.RecordJoinerI; + +public class MedicalRecordJoinerPatientSpecialLab implements RecordJoinerI { + + private static final int[] DIAGNOSISINDEXESA = new int[] {6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22}; + private static final int[] DIAGNOSISINDEXESB = new int[] {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26}; + + private double sexNull; + private double dateOfBirthNull; + private double dateOfInitialDataCollectionNull; + private double dateOfFirstVisitNull; + private double dateOfExaminationNull; + private double admissionNull; + private double aclIgaNull; + private double aclIggNull; + private double aclIgmNull; + private double anaNull; + private double anaPatternDNull; + private double anaPatternNNull; + private double anaPatternPNull; + private double anaPatternSNull; + private double kctNull; + private double rvvtNull; + private double lacNull; + private double thrombosisNull; + private double budgetNull; + private String stringNull; + + public MedicalRecordJoinerPatientSpecialLab(double sexNull, double dateOfBirthNull, double dateOfInitialDataCollectionNull, double dateOfFirstVisitNull, double dateOfExaminationNull, double admissionNull, + double aclIgaNull, double aclIggNull, double aclIgmNull, double anaNull, double anaPatternDNull, double anaPatternNNull, double anaPatternPNull, double anaPatternSNull, double kctNull, double rvvtNull, double lacNull, double thrombosisNull, double budgetNull, String stringNull) { + if (stringNull == null) throw new NullPointerException("stringNull is null"); + this.sexNull = sexNull; + this.dateOfBirthNull = dateOfBirthNull; + this.dateOfInitialDataCollectionNull = dateOfInitialDataCollectionNull; + this.dateOfFirstVisitNull = dateOfFirstVisitNull; + this.dateOfExaminationNull = dateOfExaminationNull; + this.admissionNull = admissionNull; + this.aclIgaNull = aclIgaNull; + this.aclIggNull = aclIggNull; + this.aclIgmNull = aclIgmNull; + this.anaNull = anaNull; + this.anaPatternDNull = anaPatternDNull; + this.anaPatternNNull = anaPatternNNull; + this.anaPatternPNull = anaPatternPNull; + this.anaPatternSNull = anaPatternSNull; + this.kctNull = kctNull; + this.rvvtNull = rvvtNull; + this.lacNull = lacNull; + this.thrombosisNull = thrombosisNull; + this.budgetNull = budgetNull; + this.stringNull = stringNull; + } + + @Override + public Record join(Record recordA, Record[] recordsB) { + if (recordA == null && recordsB == null) throw new NullPointerException("both inputs are null"); + if (recordA == null && (recordsB.length <= 0 || recordsB.length > 1)) throw new IllegalArgumentException("recordsB must have 1 record"); + if (recordA != null && recordsB != null && (recordsB.length <= 0 || recordsB.length > 1)) throw new IllegalArgumentException("recordsB must have 1 record"); + RecordBuilder recordBuilder = new RecordBuilder(); + + // id, sex, date_of_birth, date_of_initial_data_collection, date_of_first_visit + if (recordA != null) for (int i : new int[] { 0, 1, 2, 3, 4 }) recordBuilder.add(recordA.doubleAt(i)); + else { + recordBuilder.add(recordsB[0].doubleAt(0)); + recordBuilder.add(sexNull); + recordBuilder.add(dateOfBirthNull); + recordBuilder.add(dateOfInitialDataCollectionNull); + recordBuilder.add(dateOfFirstVisitNull); + } + // date_of_examination + if (recordsB != null) recordBuilder.add(recordsB[0].doubleAt(1)); + else recordBuilder.add(dateOfExaminationNull); + // admission + if (recordA != null) recordBuilder.add(recordA.doubleAt(5)); + else recordBuilder.add(admissionNull); + // diagnosis_sjs, diagnosis_sle, diagnosis_behcet, diagnosis_pss, diagnosis_ra, diagnosis_mctd, diagnosis_aps, diagnosis_pm, diagnosis_dm, diagnosis_pn, diagnosis_fuo, diagnosis_aortitis, diagnosis_ip, diagnosis_mra, diagnosis_collagen, diagnosis_raynaud, diagnosis_other + for (int i = 0; i < DIAGNOSISINDEXESA.length; i++) { + int indexA = DIAGNOSISINDEXESA[i]; + int indexB = DIAGNOSISINDEXESB[i]; + if (recordA == null) recordBuilder.add(recordsB[0].doubleAt(indexB)); + else if (recordsB == null) recordBuilder.add(recordA.doubleAt(indexA)); + else recordBuilder.add(recordA.doubleAt(indexA) > 0.0 || recordsB[0].doubleAt(indexB) > 0.0 ? 1.0 : 0.0); + } + // acl_iga, acl_igg, acl_igm, ana, ana_pattern_d, ana_pattern_n, ana_pattern_p, ana_pattern_s, kct, rvvt, lac, thrombosis + if (recordsB != null) for (int i : new int[] { 9, 2, 3, 4, 5, 6, 7, 8, 27, 28, 29, 30 }) recordBuilder.add(recordsB[0].doubleAt(i)); + else { + recordBuilder.add(aclIgaNull); + recordBuilder.add(aclIggNull); + recordBuilder.add(aclIgmNull); + recordBuilder.add(anaNull); + recordBuilder.add(anaPatternDNull); + recordBuilder.add(anaPatternNNull); + recordBuilder.add(anaPatternPNull); + recordBuilder.add(anaPatternSNull); + recordBuilder.add(kctNull); + recordBuilder.add(rvvtNull); + recordBuilder.add(lacNull); + recordBuilder.add(thrombosisNull); + } + // budget + if (recordA != null) recordBuilder.add(recordA.doubleAt(23)); + else recordBuilder.add(budgetNull); + // diagnosis_general + if (recordA != null) recordBuilder.add(recordA.stringAt(0)); + else recordBuilder.add(stringNull); + // diagnosis_special, symptoms + if (recordsB != null) for (int i : new int[] { 0, 1 }) recordBuilder.add(recordsB[0].stringAt(i)); + else { + recordBuilder.add(stringNull); + recordBuilder.add(stringNull); + } + return recordBuilder.toRecord(); + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordTransformer.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordTransformer.java new file mode 100644 index 0000000000000000000000000000000000000000..b91ec0efcedd5ecd9d50ac61a345707f7d39d1ee --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalRecordTransformer.java @@ -0,0 +1,34 @@ +package recordTransformer.datasets.medical; + +import java.io.IOException; + +import recordTransformer.core.Table; + +public class MedicalRecordTransformer { + + public static void main(String[] args) { + MedicalTables tables = new MedicalTables("/home/user/unitrax/data/medical/original"); + Table patient = Table.join("patient", new String[] { + "id", "sex", "date_of_birth", "date_of_initial_data_collection", "date_of_first_visit", + "date_of_examination", + "admission", + "diagnosis_sjs", "diagnosis_sle", "diagnosis_behcet", "diagnosis_pss", "diagnosis_ra", "diagnosis_mctd", "diagnosis_aps", + "diagnosis_pm", "diagnosis_dm", "diagnosis_pn", "diagnosis_fuo", "diagnosis_aortitis", "diagnosis_ip", "diagnosis_mra", + "diagnosis_collagen", "diagnosis_raynaud", "diagnosis_other", + "acl_iga", "acl_igg", "acl_igm", "ana", "ana_pattern_d", "ana_pattern_n", "ana_pattern_p", "ana_pattern_s", "kct", "rvvt", "lac", "thrombosis", + "budget", + "diagnosis_general", + "diagnosis_special", "symptoms"}, + 0, 37, tables.patient(), tables.specialLab(), 0, 0, new MedicalRecordJoinerPatientSpecialLab( + 0.0, -2208988800.0, 631152000.0, 0.0, 0.0, 0.0, + -1.0, -1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, -1.0, 1.0, ""), + new MedicalDictionaryJoinerPatientSpecialLab()); + try { + patient.writeTo("/home/user/unitrax/data/medical/transformed"); + tables.laboratory().writeTo("/home/user/unitrax/data/medical/transformed"); + } catch (IOException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalTables.java b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalTables.java new file mode 100644 index 0000000000000000000000000000000000000000..1687aaae8ec9aa0a28d6212d5cb01c74415585fd --- /dev/null +++ b/source/java/recordTransformer/src/recordTransformer/datasets/medical/MedicalTables.java @@ -0,0 +1,356 @@ +package recordTransformer.datasets.medical; + +import java.io.File; +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; + +import recordTransformer.core.DummyLinePreparer; +import recordTransformer.core.RecordFieldParserDouble; +import recordTransformer.core.RecordFieldParserEnumerableDate; +import recordTransformer.core.RecordFieldParserEnumerableDouble; +import recordTransformer.core.RecordFieldParserI; +import recordTransformer.core.RecordFinalizerBudget; +import recordTransformer.core.Table; + +public class MedicalTables { + + private Table patient = null; + private Table specialLab = null; + private Table laboratory = null; + + public MedicalTables(String path) { + String[] patientPreLabels = new String[] {"id", "sex", "date_of_birth", "date_of_initial_data_collection", "date_of_first_visit", "admission"}; + String[] patientPostLabels = new String[] {"budget", "diagnosis"}; + String[] patientColumnLabels = new String[patientPreLabels.length + MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length + patientPostLabels.length]; + for (int i = 0; i < patientPreLabels.length; i++) patientColumnLabels[i] = patientPreLabels[i]; + for (int i = 0; i < MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length; i++) patientColumnLabels[patientPreLabels.length + i] = MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES[i]; + for (int i = 0; i < patientPostLabels.length; i++) patientColumnLabels[patientPreLabels.length + MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length + i] = patientPostLabels[i]; + patient = Table.readFrom(path + File.separator + "TSUM_A.CSV", ",", "patient", patientColumnLabels, new RecordFieldParserI[] { + new RecordFieldParserDouble(), + new MedicalRecordFieldParserSex(), + // -2208988800.0 = GMT: Monday, January 1, 1900 12:00:00 AM + // -2208988799.0 = GMT: Monday, January 1, 1900 12:00:01 AM + // 631152000.0 = GMT: Monday, January 1, 1990 12:00:00 AM + new MedicalRecordFieldParserDateYearFirst(-2208988800.0, -2208988799.0, 631152000.0), + // 631152000.0 = GMT: Monday, January 1, 1990 12:00:00 AM + // 631152001.0 = GMT: Monday, January 1, 1990 12:00:01 AM + // 915148800.0 = GMT: Friday, January 1, 1999 12:00:00 AM + new MedicalRecordFieldParserDateYearFirst(631152000.0, 631152001.0, 915148800.0), + // 0.0 = GMT: Thursday, January 1, 1970 12:00:00 AM + // 1.0 = GMT: Thursday, January 1, 1970 12:00:01 AM + // 915148800.0 = GMT: Friday, January 1, 1999 12:00:00 AM + new MedicalRecordFieldParserDateYearFirst(0.0, 1.0, 915148800.0), + new MedicalRecordFieldParserPlusMinus(), + new MedicalRecordFieldParserDiagnosis()}, + new RecordFinalizerBudget(1.0), 0, patientPreLabels.length + MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length + 1, new DummyLinePreparer()); + + String[] specialLabPreLabels = new String[] {"id", "date_of_examination", "acl_igg", "acl_igm", "ana"}; + String[] specialLabMidLabels = new String[] {"acl_iga"}; + String[] specialLabPostLabels = new String[] {"kct", "rvvt", "lac", "thrombosis", "diagnosis", "symptoms"}; + String[] specialLabColumnLabels = new String[specialLabPreLabels.length + MedicalRecordFieldParserAnaPattern.ANACOLUMNNAMES.length + specialLabMidLabels.length + MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length + specialLabPostLabels.length]; + for (int i = 0; i < specialLabPreLabels.length; i++) specialLabColumnLabels[i] = specialLabPreLabels[i]; + for (int i = 0; i < MedicalRecordFieldParserAnaPattern.ANACOLUMNNAMES.length; i++) specialLabColumnLabels[specialLabPreLabels.length + i] = MedicalRecordFieldParserAnaPattern.ANACOLUMNNAMES[i]; + for (int i = 0; i < specialLabMidLabels.length; i++) specialLabColumnLabels[specialLabPreLabels.length + MedicalRecordFieldParserAnaPattern.ANACOLUMNNAMES.length + i] = specialLabMidLabels[i]; + for (int i = 0; i < MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length; i++) specialLabColumnLabels[specialLabPreLabels.length + MedicalRecordFieldParserAnaPattern.ANACOLUMNNAMES.length + specialLabMidLabels.length + i] = MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES[i]; + for (int i = 0; i < specialLabPostLabels.length; i++) specialLabColumnLabels[specialLabPreLabels.length + MedicalRecordFieldParserAnaPattern.ANACOLUMNNAMES.length + specialLabMidLabels.length + MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length + i] = specialLabPostLabels[i]; + HashMap anaMap = new HashMap<>(); + anaMap.put(4097.0, new String[] { ">4096" }); + anaMap.put(-1.0, new String[] { "" }); + HashMap doubleNullMap = new HashMap<>(); + doubleNullMap.put(-1.0, new String[] { "" }); + specialLab = Table.readFrom(path + File.separator + "TSUM_B.CSV", ",", "specialLab", specialLabColumnLabels, new RecordFieldParserI[] { + new RecordFieldParserDouble(), + // 0.0 = GMT: Thursday, January 1, 1970 12:00:00 AM + // 1.0 = GMT: Thursday, January 1, 1970 12:00:01 AM + // 915148800.0 = GMT: Friday, January 1, 1999 12:00:00 AM + new MedicalRecordFieldParserDateYearFirst(0.0, 1.0, 915148800.0), + new RecordFieldParserEnumerableDouble(0.0, 1000.0, doubleNullMap), + new RecordFieldParserEnumerableDouble(0.0, 200000.0, doubleNullMap), + new RecordFieldParserEnumerableDouble(0.0, 4096.0, anaMap), + new MedicalRecordFieldParserAnaPattern(), + new RecordFieldParserEnumerableDouble(0.0, 50000.0, doubleNullMap), + new MedicalRecordFieldParserDiagnosis(), + new MedicalRecordFieldParserPlusMinus(), + new MedicalRecordFieldParserPlusMinus(), + new MedicalRecordFieldParserPlusMinus(), + new MedicalRecordFieldParserDummy(), + new RecordFieldParserEnumerableDouble(0.0, 3.0, doubleNullMap)}, + null, 0, specialLabPreLabels.length + MedicalRecordFieldParserAnaPattern.ANACOLUMNNAMES.length + specialLabMidLabels.length + MedicalRecordFieldParserDiagnosis.DIAGNOSISCOLUMNNAMES.length + 4, new DummyLinePreparer()); + + HashMap dateMap = new HashMap<>(); + dateMap.put(0.0, new String[] { "" }); + HashMap ldhMap = new HashMap<>(); + ldhMap.put(-1.0, new String[] { "" }); + ldhMap.put(10000.0, new String[] { ">5000" }); + HashMap cpkMap = new HashMap<>(); + cpkMap.put(-1.0, new String[] { "" }); + cpkMap.put(10.0, new String[] { "<37", "<33", "<34" }); + cpkMap.put(10000.0, new String[] { "5000<" }); + HashMap wbcMap = new HashMap<>(); + wbcMap.put(-1.0, new String[] { "" }); + wbcMap.put(1.0, new String[] { "<13.9" }); + HashMap pltMap = new HashMap<>(); + pltMap.put(-1.0, new String[] { "" }); + pltMap.put(15.0, new String[] { ">12" }); + pltMap.put(25.0, new String[] { ">22PT" }); + pltMap.put(60.0, new String[] { ">50" }); + pltMap.put(65.0, new String[] { "60-70" }); + pltMap.put(80.0, new String[] { ">77" }); + pltMap.put(90.0, new String[] { ">86" }); + pltMap.put(110.0, new String[] { ">109", ">108", ">107", ">100" }); + pltMap.put(120.0, new String[] { ">116" }); + pltMap.put(140.0, new String[] { ">133" }); + pltMap.put(160.0, new String[] { ">155", ">150" }); + pltMap.put(170.0, new String[] { ">166", ">165", ">163PT", ">163" }); + pltMap.put(180.0, new String[] { ">174PT", ">171", ">170" }); + pltMap.put(200.0, new String[] { ">195" }); + pltMap.put(220.0, new String[] { ">217", ">216" }); + pltMap.put(230.0, new String[] { ">221" }); + pltMap.put(240.0, new String[] { ">230" }); + pltMap.put(260.0, new String[] { ">253", ">250" }); + pltMap.put(270.0, new String[] { ">267", ">264", ">260" }); + pltMap.put(280.0, new String[] { ">271PT" }); + pltMap.put(290.0, new String[] { ">280PT" }); + pltMap.put(300.0, new String[] { ">297" }); + pltMap.put(310.0, new String[] { ">305" }); + pltMap.put(320.0, new String[] { ">313" }); + pltMap.put(340.0, new String[] { ">330" }); + pltMap.put(380.0, new String[] { "370<" }); + pltMap.put(390.0, new String[] { ">382" }); + pltMap.put(400.0, new String[] { ">393" }); + HashMap apttMap = new HashMap<>(); + apttMap.put(-1.0, new String[] { "" }); + apttMap.put(181.0, new String[] { ">180" }); + HashMap at3Map = new HashMap<>(); + at3Map.put(202.0, new String[] { "202H" }); + at3Map.put(188.0, new String[] { "188H" }); + at3Map.put(172.0, new String[] { "172H" }); + at3Map.put(170.0, new String[] { "170H" }); + at3Map.put(168.0, new String[] { "168H" }); + at3Map.put(166.0, new String[] { "166H" }); + at3Map.put(160.0, new String[] { "160H" }); + at3Map.put(159.0, new String[] { "159H" }); + at3Map.put(158.0, new String[] { "158H" }); + at3Map.put(157.0, new String[] { "157H" }); + at3Map.put(154.0, new String[] { "154H" }); + at3Map.put(153.0, new String[] { "153H" }); + at3Map.put(152.0, new String[] { "152H", "152.0H" }); + at3Map.put(150.0, new String[] { "150H", "150.0H" }); + at3Map.put(149.0, new String[] { "149H" }); + at3Map.put(148.0, new String[] { "148H" }); + at3Map.put(147.0, new String[] { "147H" }); + at3Map.put(146.0, new String[] { "146H" }); + at3Map.put(145.0, new String[] { "145H" }); + at3Map.put(144.0, new String[] { "144H" }); + at3Map.put(143.0, new String[] { "143H" }); + at3Map.put(142.0, new String[] { "142H" }); + at3Map.put(141.0, new String[] { "141H", "141.0H" }); + at3Map.put(140.0, new String[] { "140H", "140.0H" }); + at3Map.put(139.0, new String[] { "139H" }); + at3Map.put(138.0, new String[] { "138H", "138.0H" }); + at3Map.put(137.0, new String[] { "137H" }); + at3Map.put(136.0, new String[] { "136H", "136.0H" }); + at3Map.put(135.0, new String[] { "135H" }); + at3Map.put(134.0, new String[] { "134H", "134.0H" }); + at3Map.put(133.0, new String[] { "133H", "133.0H" }); + at3Map.put(-1.0, new String[] { "" }); + HashMap a2piMap = new HashMap<>(); + a2piMap.put(172.0, new String[] { "172H" }); + a2piMap.put(166.0, new String[] { "166H" }); + a2piMap.put(154.0, new String[] { "154H" }); + a2piMap.put(150.0, new String[] { "150H" }); + a2piMap.put(149.0, new String[] { "149H" }); + a2piMap.put(148.0, new String[] { "148H" }); + a2piMap.put(147.0, new String[] { "147H" }); + a2piMap.put(146.0, new String[] { "146H" }); + a2piMap.put(144.0, new String[] { "144H" }); + a2piMap.put(143.0, new String[] { "143H" }); + a2piMap.put(142.0, new String[] { "142H" }); + a2piMap.put(141.0, new String[] { "141H" }); + a2piMap.put(140.0, new String[] { "140H", "140.0H" }); + a2piMap.put(139.0, new String[] { "139H" }); + a2piMap.put(138.0, new String[] { "138H" }); + a2piMap.put(137.0, new String[] { "137.0H" }); + a2piMap.put(136.0, new String[] { "136H", "136.0H" }); + a2piMap.put(135.0, new String[] { "135H", "135.0H" }); + a2piMap.put(134.0, new String[] { "134H", "134.0H" }); + a2piMap.put(133.0, new String[] { "133H", "132H", "131H", "131.0H" }); + a2piMap.put(-1.0, new String[] { "" }); + HashMap uproMap = new HashMap<>(); + uproMap.put(-16.0, new String[] { "" }); + uproMap.put(-17.0, new String[] { "TR" }); + uproMap.put(-18.0, new String[] { "ERROR" }); + uproMap.put(-19.0, new String[] { "%%" }); + uproMap.put(-20.0, new String[] { "-" }); + uproMap.put(30.0, new String[] { "+1(30)" }); + uproMap.put(100.0, new String[] { "+2(100)" }); + uproMap.put(500.0, new String[] { "+3(500)" }); + uproMap.put(350.0, new String[] { ">=300" }); + uproMap.put(1100.0, new String[] { ">=1000" }); + HashMap iggMap = new HashMap<>(); + iggMap.put(-1.0, new String[] { "" }); + iggMap.put(1.0, new String[] { "<2.00" }); + HashMap igaMap = new HashMap<>(); + igaMap.put(-1.0, new String[] { "" }); + igaMap.put(16.0, new String[] { "<33" }); + igaMap.put(15.0, new String[] { "<31", "<30" }); + igaMap.put(1.0, new String[] { "<29" }); + igaMap.put(0.2, new String[] { "<0.4" }); + igaMap.put(0.15, new String[] { "<0.30" }); + igaMap.put(0.0, new String[] { "<0" }); + HashMap igmMap = new HashMap<>(); + igmMap.put(-1.0, new String[] { "" }); + igmMap.put(15.0, new String[] { "<30" }); + igmMap.put(14.0, new String[] { "<29", "<28" }); + igmMap.put(13.0, new String[] { "<27" }); + igmMap.put(12.0, new String[] { "<25" }); + igmMap.put(10.0, new String[] { "<20" }); + igmMap.put(0.8, new String[] { "<1.6" }); + igmMap.put(0.5, new String[] { "<1" }); + igmMap.put(0.4, new String[] { "<0.9" }); + igmMap.put(0.3, new String[] { "<0.7", "<0.6" }); + igmMap.put(0.1, new String[] { "<0.30", "<0.3" }); + HashMap crpMap = new HashMap<>(); + crpMap.put(-9.0, new String[] { "" }); + crpMap.put(-8.0, new String[] { "-" }); + crpMap.put(-7.0, new String[] { "6+<" }); + crpMap.put(-6.0, new String[] { "6+" }); + crpMap.put(-5.0, new String[] { "5+" }); + crpMap.put(-4.0, new String[] { "4+" }); + crpMap.put(-3.0, new String[] { "3+" }); + crpMap.put(-2.0, new String[] { "2+" }); + crpMap.put(-1.0, new String[] { "+" }); + crpMap.put(80.0, new String[] { ">79.5" }); + crpMap.put(14.0, new String[] { ">7.0" }); + crpMap.put(12.0, new String[] { ">6" }); + crpMap.put(0.15, new String[] { "<0.3" }); + crpMap.put(0.1, new String[] { "<0.2" }); + crpMap.put(0.05, new String[] { "<0.1" }); + crpMap.put(0.001, new String[] { "<0.002" }); + HashMap rfMap = new HashMap<>(); + rfMap.put(-1.0, new String[] { "" }); + rfMap.put(2561.0, new String[] { ">2560" }); + rfMap.put(39.0, new String[] { "40>", "<40" }); + rfMap.put(24.9, new String[] { "<25.0" }); + rfMap.put(23.5, new String[] { "<23.6" }); + rfMap.put(23.2, new String[] { "<23.3" }); + rfMap.put(22.9, new String[] { "<23.0" }); + rfMap.put(22.2, new String[] { "<22.3" }); + rfMap.put(21.9, new String[] { "<22.0" }); + rfMap.put(21.2, new String[] { "<21.3" }); + rfMap.put(20.9, new String[] { "<21.0", "<21" }); + rfMap.put(20.7, new String[] { "<20.8" }); + rfMap.put(20.4, new String[] { "<20.5" }); + rfMap.put(19.9, new String[] { "<20.0" }); + rfMap.put(19.7, new String[] { "<19.8" }); + rfMap.put(19.4, new String[] { "<19.5" }); + rfMap.put(19.2, new String[] { "<19.3" }); + rfMap.put(18.9, new String[] { "<19.0", "<19" }); + rfMap.put(18.7, new String[] { "<18.8" }); + rfMap.put(18.4, new String[] { "<18.5" }); + rfMap.put(18.2, new String[] { "<18.3" }); + rfMap.put(17.7, new String[] { "<17.8" }); + rfMap.put(17.4, new String[] { "<17.5" }); + rfMap.put(16.9, new String[] { "<17.0" }); + rfMap.put(16.4, new String[] { "<16.5" }); + rfMap.put(16.2, new String[] { "<16.3" }); + rfMap.put(15.9, new String[] { "<16" }); + rfMap.put(15.7, new String[] { "<15.8" }); + rfMap.put(15.2, new String[] { "<15.3" }); + rfMap.put(11.9, new String[] { "<12" }); + rfMap.put(11.0, new String[] { "<11.1" }); + rfMap.put(10.9, new String[] { "<11" }); + rfMap.put(10.0, new String[] { "<10.6" }); + rfMap.put(9.0, new String[] { "<10" }); + HashMap c3Map = new HashMap<>(); + c3Map.put(-1.0, new String[] { "" }); + c3Map.put(2.0, new String[] { "<2.7" }); + c3Map.put(1.0, new String[] { "<2" }); + HashMap c4Map = new HashMap<>(); + c4Map.put(-1.0, new String[] { "" }); + c4Map.put(11.0, new String[] { "<12" }); + c4Map.put(6.0, new String[] { "<7", "<6.6", "<6.5" }); + c4Map.put(5.0, new String[] { "<6" }); + c4Map.put(4.0, new String[] { "<5" }); + c4Map.put(3.0, new String[] { "<4" }); + c4Map.put(2.0, new String[] { "<3" }); + c4Map.put(1.0, new String[] { "<2", "<1.4" }); + c4Map.put(0.5, new String[] { "<1" }); + HashMap rnpSmSc170SsaSsbCentromeaMap = new HashMap<>(); + rnpSmSc170SsaSsbCentromeaMap.put(-2.0, new String[] { "" }); + rnpSmSc170SsaSsbCentromeaMap.put(-1.0, new String[] { "negative" }); + laboratory = Table.readFrom(path + File.separator + "TSUM_C.CSV", ",", "laboratory", new String[] { + "id", "patient_id", "date", "got", "gpt", "ldh", "alp", "tp", "alb", "ua", "un", "cre", "t-bil", "t-cho", "tg", "cpk", "glu", "wbc", "rbc", "hgb", "hct", "plt", "pt", "pt_note", "aptt", + "fg", "at3", "a2pi", "u-pro", "igg", "iga", "igm", "crp", "ra", "rf", "c3", "c4", "rnp", "sm", "sc170", "ssa", "ssb", "centromea", "dna"}, + new RecordFieldParserI[] { + new RecordFieldParserDouble(), // id + new RecordFieldParserDouble(), // patient_id + // 0.0 = GMT: Thursday, January 1, 1970 12:00:00 AM + // 1.0 = GMT: Thursday, January 1, 1970 12:00:01 AM + // 946684800.0 = GMT: Saturday, January 1, 2000 12:00:00 AM + new RecordFieldParserEnumerableDate("yyMMdd", new Date(-2208988800000l), 1.0, 946684800.0, dateMap), // date + new MedicalRecordFieldParserEnumerableDouble(0.0, 25000.0, -1.0), // got + new MedicalRecordFieldParserEnumerableDouble(0.0, 5000.0, -1.0), // gpt + new MedicalRecordFieldParserEnumerableDouble(0.0, 70000.0, -1.0, ldhMap), // ldh + new MedicalRecordFieldParserEnumerableDouble(0.0, 3200.0, -1.0), // alp + new MedicalRecordFieldParserEnumerableDouble(0.0, 12.0, -1.0), // tp + new MedicalRecordFieldParserEnumerableDouble(0.0, 6.0, -1.0), // alb + new MedicalRecordFieldParserEnumerableDouble(0.0, 33.0, -1.0), // ua + new MedicalRecordFieldParserEnumerableDouble(0.0, 160.0, -1.0), // un + new MedicalRecordFieldParserEnumerableDouble(0.0, 18.0, -1.0), // cre + new MedicalRecordFieldParserEnumerableDouble(0.0, 27.0, -1.0), // t-bil + new MedicalRecordFieldParserEnumerableDouble(0.0, 1000.0, -1.0), // t-cho + new MedicalRecordFieldParserEnumerableDouble(0.0, 1500.0, -1.0), // tg + new MedicalRecordFieldParserEnumerableDouble(0.0, 25000.0, -1.0, cpkMap), // cpk + new MedicalRecordFieldParserEnumerableDouble(0.0, 1000.0, -1.0), // glu + new MedicalRecordFieldParserEnumerableDouble(0.0, 120.0, -1.0, wbcMap), // wbc + new MedicalRecordFieldParserEnumerableDouble(0.0, 10.0, -1.0), // rbc + new MedicalRecordFieldParserEnumerableDouble(0.0, 20.0, -1.0), // hgb + new MedicalRecordFieldParserEnumerableDouble(0.0, 60.0, -1.0), // hct + new MedicalRecordFieldParserEnumerableDouble(0.0, 9000.0, -1.0, pltMap), // plt + new MedicalRecordFieldParserEnumerableDouble(0.0, 75.0, -1.0), // pt + new MedicalRecordFieldParserEnumerableDouble(0.0, 185.0, -1.0), // pt_note + new MedicalRecordFieldParserEnumerableDouble(0.0, 180.0, -1.0, apttMap), // aptt + new MedicalRecordFieldParserEnumerableDouble(0.0, 1000.0, -1.0), // fg + new MedicalRecordFieldParserEnumerableDouble(0.0, 200.0, -1.0, at3Map), // at3 + new MedicalRecordFieldParserEnumerableDouble(0.0, 200.0, -1.0, a2piMap), // a2pi + new MedicalRecordFieldParserEnumerableDouble(-15.0, 1500.0, -16.0, uproMap), // u-pro + new MedicalRecordFieldParserEnumerableDouble(0.0, 8000.0, -1.0, iggMap), // igg + new MedicalRecordFieldParserEnumerableDouble(0.0, 2000.0, -1.0, igaMap), // iga + new MedicalRecordFieldParserEnumerableDouble(0.0, 3000.0, -1.0, igmMap), // igm + new MedicalRecordFieldParserEnumerableDouble(0.0, 50.0, -9.0, crpMap), // crp + new MedicalRecordFieldParserPlusMinus(), // ra + new MedicalRecordFieldParserEnumerableDouble(0.0, 7000.0, -1.0, rfMap), // rf + new MedicalRecordFieldParserEnumerableDouble(0.0, 230.0, -1.0, c3Map), // c3 + new MedicalRecordFieldParserEnumerableDouble(0.0, 1000.0, -1.0, c4Map), // c4 + new MedicalRecordFieldParserEnumerableDouble(0.0, 256.0, -2.0, rnpSmSc170SsaSsbCentromeaMap), // rnp + new MedicalRecordFieldParserEnumerableDouble(0.0, 32.0, -2.0, rnpSmSc170SsaSsbCentromeaMap), // sm + new MedicalRecordFieldParserEnumerableDouble(0.0, 16.0, -2.0, rnpSmSc170SsaSsbCentromeaMap), // sc170 + new MedicalRecordFieldParserEnumerableDouble(0.0, 256.0, -2.0, rnpSmSc170SsaSsbCentromeaMap), // ssa + new MedicalRecordFieldParserEnumerableDouble(0.0, 128.0, -2.0, rnpSmSc170SsaSsbCentromeaMap), // ssb + new MedicalRecordFieldParserEnumerableDouble(0.0, 5500.0, -2.0, rnpSmSc170SsaSsbCentromeaMap), // hgb + new MedicalRecordFieldParserEnumerableDouble(0.0, 200.0, -1.0), // dna + }, + null, 0, 44, new MedicalLinePreparerLaboratory(",", 44)); + } + + public Table patient() { + return patient; + } + + public Table specialLab() { + return specialLab; + } + + public Table laboratory() { + return laboratory; + } + + public void writeAllTablesTo(String path) throws IOException { + if (path == null) throw new NullPointerException("path is null"); + patient.writeTo(path); + specialLab.writeTo(path); + laboratory.writeTo(path); + } +} \ No newline at end of file