Skip to content

Commit

Permalink
Merge branch 'pgsql'
Browse files Browse the repository at this point in the history 
# Conflicts:
#	Signum.Engine/Engine/SchemaGenerator.cs
#	Signum.Engine/Engine/SchemaSynchronizer.cs
#	Signum.Engine/Engine/SqlBuilder.cs
#	Signum.Engine/Linq/ExpressionVisitor/QueryFormatter.cs
#	Signum.React/Signum.React.csproj
  • Loading branch information
@olmobrutall
olmobrutall committed Jan 24, 2020
2 parents 9cd36ef + f78f5f7 commit 2895528
Show file tree
Hide file tree
Showing 93 changed files with 5,799 additions and 1,823 deletions.
115 changes: 67 additions & 48 deletions Signum.Engine/Administrator.cs
View file

Large diffs are not rendered by default.

51 changes: 25 additions & 26 deletions Signum.Engine/Basics/TypeLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,37 +44,35 @@ public static void Start(SchemaBuilder sb)
{
if (sb.NotDefined(MethodInfo.GetCurrentMethod()))
{
Schema current = Schema.Current;
Schema schema = Schema.Current;

current.SchemaCompleted += () =>
sb.Include<TypeEntity>()
.WithQuery(() => t => new
{
Entity = t,
t.Id,
t.TableName,
t.CleanName,
t.ClassName,
t.Namespace,
});

schema.SchemaCompleted += () =>
{
var attributes = current.Tables.Keys.Select(t => KeyValuePair.Create(t, t.GetCustomAttribute<EntityKindAttribute>(true))).ToList();
var attributes = schema.Tables.Keys.Select(t => KeyValuePair.Create(t, t.GetCustomAttribute<EntityKindAttribute>(true))).ToList();

var errors = attributes.Where(a => a.Value == null).ToString(a => "Type {0} does not have an EntityTypeAttribute".FormatWith(a.Key.Name), "\r\n");

if (errors.HasText())
throw new InvalidOperationException(errors);
};

current.Initializing += () =>
schema.Initializing += () =>
{
current.typeCachesLazy.Load();
schema.typeCachesLazy.Load();
};

current.typeCachesLazy = sb.GlobalLazy(() => new TypeCaches(current),
new InvalidateWith(typeof(TypeEntity)),
Schema.Current.InvalidateMetadata);

sb.Include<TypeEntity>()
.WithQuery(() => t => new
{
Entity = t,
t.Id,
t.TableName,
t.CleanName,
t.ClassName,
t.Namespace,
});
schema.typeCachesLazy = sb.GlobalLazy(() => new TypeCaches(schema), new InvalidateWith(typeof(TypeEntity)), Schema.Current.InvalidateMetadata);

TypeEntity.SetTypeEntityCallbacks(
t => TypeToEntity.GetOrThrow(t),
Expand All @@ -92,15 +90,16 @@ join t in Schema.Current.Tables.Keys on dn.FullClassName equals (EnumEntity.Extr
public static SqlPreCommand? Schema_Synchronizing(Replacements replacements)
{
var schema = Schema.Current;
var isPostgres = schema.Settings.IsPostgres;

Dictionary<string, TypeEntity> should = GenerateSchemaTypes().ToDictionaryEx(s => s.TableName, "tableName in memory");

var currentList = Administrator.TryRetrieveAll<TypeEntity>(replacements);

{ //External entities are nt asked in SchemaSynchronizer
replacements.AskForReplacements(
currentList.Where(t => schema.IsExternalDatabase(ObjectName.Parse(t.TableName).Schema.Database)).Select(a => a.TableName).ToHashSet(),
should.Values.Where(t => schema.IsExternalDatabase(ObjectName.Parse(t.TableName).Schema.Database)).Select(a => a.TableName).ToHashSet(),
currentList.Where(t => schema.IsExternalDatabase(ObjectName.Parse(t.TableName, isPostgres).Schema.Database)).Select(a => a.TableName).ToHashSet(),
should.Values.Where(t => schema.IsExternalDatabase(ObjectName.Parse(t.TableName, isPostgres).Schema.Database)).Select(a => a.TableName).ToHashSet(),
Replacements.KeyTables);
}

Expand All @@ -109,12 +108,12 @@ join t in Schema.Current.Tables.Keys on dn.FullClassName equals (EnumEntity.Extr

{ //Temporal solution until applications are updated
var repeated =
should.Keys.Select(k => ObjectName.Parse(k)).GroupBy(a => a.Name).Where(a => a.Count() > 1).Select(a => a.Key).Concat(
current.Keys.Select(k => ObjectName.Parse(k)).GroupBy(a => a.Name).Where(a => a.Count() > 1).Select(a => a.Key)).ToList();
should.Keys.Select(k => ObjectName.Parse(k, isPostgres)).GroupBy(a => a.Name).Where(a => a.Count() > 1).Select(a => a.Key).Concat(
current.Keys.Select(k => ObjectName.Parse(k, isPostgres)).GroupBy(a => a.Name).Where(a => a.Count() > 1).Select(a => a.Key)).ToList();

Func<string, string> simplify = tn =>
{
ObjectName name = ObjectName.Parse(tn);
ObjectName name = ObjectName.Parse(tn, isPostgres);
return repeated.Contains(name.Name) ? name.ToString() : name.Name;
};

Expand All @@ -138,8 +137,8 @@ join t in Schema.Current.Tables.Keys on dn.FullClassName equals (EnumEntity.Extr

if (c.TableName != s.TableName)
{
var pc = ObjectName.Parse(c.TableName);
var ps = ObjectName.Parse(s.TableName);
var pc = ObjectName.Parse(c.TableName, isPostgres);
var ps = ObjectName.Parse(s.TableName, isPostgres);

if (!EqualsIgnoringDatabasePrefix(pc, ps))
{
Expand Down
33 changes: 17 additions & 16 deletions Signum.Engine/BulkInserter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ public static int BulkInsert<T>(this IEnumerable<T> entities,
using (HeavyProfiler.Log(nameof(BulkInsert), () => typeof(T).TypeName()))
using (Transaction tr = new Transaction())
{

var table = Schema.Current.Table(typeof(T));

if (!disableIdentity && table.IdentityBehaviour && table.TablesMList().Any())
Expand Down Expand Up @@ -155,27 +154,28 @@ public static int BulkInsertTable<T>(IEnumerable<T> entities,

var t = Schema.Current.Table<T>();
bool disableIdentityBehaviour = copyOptions.HasFlag(SqlBulkCopyOptions.KeepIdentity);
bool oldIdentityBehaviour = t.IdentityBehaviour;

DataTable dt = new DataTable();
foreach (var c in t.Columns.Values.Where(c => !(c is SystemVersionedInfo.Column) && (disableIdentityBehaviour || !c.IdentityBehaviour)))
var columns = t.Columns.Values.Where(c => !(c is SystemVersionedInfo.SqlServerPeriodColumn) && (disableIdentityBehaviour || !c.IdentityBehaviour)).ToList();
foreach (var c in columns)
dt.Columns.Add(new DataColumn(c.Name, c.Type.UnNullify()));

if (disableIdentityBehaviour) t.IdentityBehaviour = false;
foreach (var e in list)
using (disableIdentityBehaviour ? Administrator.DisableIdentity(t, behaviourOnly: true) : null)
{
if (!e.IsNew)
throw new InvalidOperationException("Entites should be new");
t.SetToStrField(e);
dt.Rows.Add(t.BulkInsertDataRow(e));
foreach (var e in list)
{
if (!e.IsNew)
throw new InvalidOperationException("Entites should be new");
t.SetToStrField(e);
dt.Rows.Add(t.BulkInsertDataRow(e));
}
}
if (disableIdentityBehaviour) t.IdentityBehaviour = oldIdentityBehaviour;

using (Transaction tr = new Transaction())
{
Schema.Current.OnPreBulkInsert(typeof(T), inMListTable: false);

Executor.BulkCopy(dt, t.Name, copyOptions, timeout);
Executor.BulkCopy(dt, columns, t.Name, copyOptions, timeout);

foreach (var item in list)
item.SetNotModified();
Expand Down Expand Up @@ -272,8 +272,8 @@ public static int BulkInsertMListTable<E, V>(
var maxRowId = updateParentTicks.Value ? Database.MListQuery(mListProperty).Max(a => (PrimaryKey?)a.RowId) : null;

DataTable dt = new DataTable();

foreach (var c in mlistTable.Columns.Values.Where(c => !(c is SystemVersionedInfo.Column) && !c.IdentityBehaviour))
var columns = mlistTable.Columns.Values.Where(c => !(c is SystemVersionedInfo.SqlServerPeriodColumn) && !c.IdentityBehaviour).ToList();
foreach (var c in columns)
dt.Columns.Add(new DataColumn(c.Name, c.Type.UnNullify()));

var list = mlistElements.ToList();
Expand All @@ -287,7 +287,7 @@ public static int BulkInsertMListTable<E, V>(
{
Schema.Current.OnPreBulkInsert(typeof(E), inMListTable: true);

Executor.BulkCopy(dt, mlistTable.Name, copyOptions, timeout);
Executor.BulkCopy(dt, columns, mlistTable.Name, copyOptions, timeout);

var result = list.Count;

Expand Down Expand Up @@ -327,8 +327,9 @@ public static int BulkInsertView<T>(this IEnumerable<T> entities,

bool disableIdentityBehaviour = copyOptions.HasFlag(SqlBulkCopyOptions.KeepIdentity);

var columns = t.Columns.Values.ToList();
DataTable dt = new DataTable();
foreach (var c in t.Columns.Values)
foreach (var c in columns)
dt.Columns.Add(new DataColumn(c.Name, c.Type.UnNullify()));

foreach (var e in entities)
Expand All @@ -340,7 +341,7 @@ public static int BulkInsertView<T>(this IEnumerable<T> entities,
{
Schema.Current.OnPreBulkInsert(typeof(T), inMListTable: false);

Executor.BulkCopy(dt, t.Name, copyOptions, timeout);
Executor.BulkCopy(dt, columns, t.Name, copyOptions, timeout);

return tr.Commit(list.Count);
}
Expand Down
29 changes: 16 additions & 13 deletions Signum.Engine/CodeGeneration/EntityCodeGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Signum.Engine.Engine;
using Signum.Engine.Maps;
using Signum.Entities;
using Signum.Entities.Reflection;
Expand All @@ -26,7 +27,7 @@ public class EntityCodeGenerator

public virtual void GenerateEntitiesFromDatabaseTables()
{
CurrentSchema = Schema.Current;
CurrentSchema = Schema.Current;

var tables = GetTables();

Expand Down Expand Up @@ -70,7 +71,9 @@ protected virtual string GetProjectFolder()

protected virtual List<DiffTable> GetTables()
{
return SchemaSynchronizer.DefaultGetDatabaseDescription(Schema.Current.DatabaseNames()).Values.ToList();
return Schema.Current.Settings.IsPostgres ?
PostgresCatalogSchema.GetDatabaseDescription(Schema.Current.DatabaseNames()).Values.ToList() :
SysTablesSchema.GetDatabaseDescription(Schema.Current.DatabaseNames()).Values.ToList();
}

protected virtual void GetSolutionInfo(out string solutionFolder, out string solutionName)
Expand Down Expand Up @@ -538,7 +541,7 @@ protected virtual string GetTicksColumnAttribute(DiffTable table)
{
StringBuilder sb = new StringBuilder();
sb.Append("TableName(\"" + objectName.Name + "\"");
if (objectName.Schema != SchemaName.Default)
if (objectName.Schema != SchemaName.Default(CurrentSchema.Settings.IsPostgres))
sb.Append(", SchemaName = \"" + objectName.Schema.Name + "\"");

if (objectName.Schema.Database != null)
Expand Down Expand Up @@ -636,7 +639,7 @@ protected virtual IEnumerable<string> GetPropertyAttributes(DiffTable table, Dif
parts.Add("Min = " + min);

if (col.Length != -1)
parts.Add("Max = " + col.Length / DiffColumn.BytesPerChar(col.SqlDbType));
parts.Add("Max = " + col.Length);

return "StringLengthValidator(" + parts.ToString(", ") + ")";
}
Expand Down Expand Up @@ -706,7 +709,7 @@ protected virtual bool HasUniqueIndex(DiffTable table, DiffColumn col)
ix.FilterDefinition == null &&
ix.Columns.Only()?.Let(ic => ic.ColumnName == col.Name && ic.IsIncluded == false) == true &&
ix.IsUnique &&
ix.Type == DiffIndexType.NonClustered);
ix.IsPrimary);
}

protected virtual string DefaultColumnName(DiffTable table, DiffColumn col)
Expand Down Expand Up @@ -743,19 +746,19 @@ protected virtual List<string> GetSqlDbTypeParts(DiffColumn col, Type type)
{
List<string> parts = new List<string>();
var pair = CurrentSchema.Settings.GetSqlDbTypePair(type);
if (pair.SqlDbType != col.SqlDbType)
parts.Add("SqlDbType = SqlDbType." + col.SqlDbType);
if (pair.DbType.SqlServer != col.DbType.SqlServer)
parts.Add("SqlDbType = SqlDbType." + col.DbType.SqlServer);

var defaultSize = CurrentSchema.Settings.GetSqlSize(null, null, pair.SqlDbType);
var defaultSize = CurrentSchema.Settings.GetSqlSize(null, null, pair.DbType);
if (defaultSize != null)
{
if (!(defaultSize == col.Precision || defaultSize == col.Length / DiffColumn.BytesPerChar(col.SqlDbType) || defaultSize == int.MaxValue && col.Length == -1))
if (!(defaultSize == col.Precision || defaultSize == col.Length || defaultSize == int.MaxValue && col.Length == -1))
parts.Add("Size = " + (col.Length == -1 ? "int.MaxValue" :
col.Length != 0 ? (col.Length / DiffColumn.BytesPerChar(col.SqlDbType)).ToString() :
col.Length != 0 ? col.Length.ToString() :
col.Precision != 0 ? col.Precision.ToString() : "0"));
}

var defaultScale = CurrentSchema.Settings.GetSqlScale(null, null, col.SqlDbType);
var defaultScale = CurrentSchema.Settings.GetSqlScale(null, null, col.DbType);
if (defaultScale != null)
{
if (!(col.Scale == defaultScale))
Expand Down Expand Up @@ -801,7 +804,7 @@ protected virtual bool IsLite(DiffTable table, DiffColumn col)

protected internal virtual Type GetValueType(DiffColumn col)
{
switch (col.SqlDbType)
switch (col.DbType.SqlServer)
{
case SqlDbType.BigInt: return typeof(long);
case SqlDbType.Binary: return typeof(byte[]);
Expand Down Expand Up @@ -834,7 +837,7 @@ protected internal virtual Type GetValueType(DiffColumn col)
case SqlDbType.Udt: return Schema.Current.Settings.UdtSqlName
.SingleOrDefaultEx(kvp => StringComparer.InvariantCultureIgnoreCase.Equals(kvp.Value, col.UserTypeName))
.Key;
default: throw new NotImplementedException("Unknown translation for " + col.SqlDbType);
default: throw new NotImplementedException("Unknown translation for " + col.DbType.SqlServer);
}
}

Expand Down
2 changes: 1 addition & 1 deletion Signum.Engine/CodeGeneration/LogicCodeGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -426,7 +426,7 @@ protected virtual bool IsSimpleValueType(Type type)
{
var t = CurrentSchema.Settings.TryGetSqlDbTypePair(type.UnNullify());

return t != null && t.UserDefinedTypeName == null && t.SqlDbType != SqlDbType.Image && t.SqlDbType != SqlDbType.VarBinary;
return t != null && t.UserDefinedTypeName == null && (t.DbType.IsNumber() || t.DbType.IsString() || t.DbType.IsDate());
}

protected virtual string WriteOperations(Type type)
Expand Down
Loading

6 comments on commit 2895528

@olmobrutall
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PostgreSQL support is here! 🎉 🎈 ⭐️

After 10 years of Front-end changes (WPF -> ASP.Net MVC -> React and Typescript) and Back-end changes (C# 3.0 and .Net Framework 3.0 -> C# 8.0, .NEt Core 3.1) it's time to change the only think that has stayed stable: Microsoft SQL Server.

No worries! ...more than a change is a new option. Microsoft SQL Server continues being fully supported and there is no plans to keep it in maintenance mode, but it will need to make place to his new little brother, the open-source and completely free PostgreSQL!

Why PostgreSQL?

Signum Framework was born in a world where .Net was not open-source, there was no cloud, and Microsoft SQL Server was the default option for .Net developers. During all this years there have been a small but stable flow of request from users coming from the 'free world' asking for cheaper alternatives (@avifatal, @aengin, @KonstantinLukaschenko), like the very popular LAMP stack (Linux, PHP, Apache and MySQL).

Now Microsoft is the biggest open-source contributor in GitGub and .Net Core works on Docker over Linux, so the two worlds are converging... but SQL Server is expensive, closed source and has bad performance in Linux.

Hopefully by adding PostgreSQL to the mix, Signum Framework applications will be installed in environments that where before not considered a target market. Also... did I mention is completely gratis?

Moreover, PostgresSQL is quite powerful. It has an alternative to many of the exotic features of SQL Server, either out of-the-box or using some extension:

Some features are more solid in Sql Server:

  • Multi-database queries

While others, look more powerful in PostgreeSQL

One Framework, two DBMS

For now, no extensibility model has been built in Signum.Engine to allow other DBMSs (Oracle, MySql, MariaDB...) to be developed (i.e. Referencing a DBMS-specific Project), instead Signum.Engine now depends to both System.Data.SqlClient (.Net driver for Microsoft SQL Server) and Npgsql (.Net driver for PostgreSQL) and reuses most of the code (Save, LINQ Provider, etc) with conditional logic when needed. You can use Schema.Current.Settings.IsPostgrees or Connector.Current is PostgreSqlConnector to check if the application is using a PostgresSQL.

The reason for this, is that building an abstraction always has a cost, and there are just to many places in Signum.Engine where the conditional behavior is need. Some important differences from PostgreSQL:

  • In PostgreSQL the names of tables, columns, etc... are case sensitive when quoted ('MyTable'), and are automatically lower-cased when not quoted. In SQL Server names are always case insensitive, and only special charactes or keywords need to be quoted ([MyTable]).
  • Column types in PostgreSQL (NpgsqlDbType) are different to the ones from SqlServer (SqlDbType) but there is a simple mapping from .Net types.
  • The system tables used for introspection in PostgreSQL (like pg_catalog.class) are completely different in SQL Server (sys.table).
  • In Postgres using ; between statements is mandatory, in SQL not.
  • Variables can not be declared in a SQL script, an anonymous DO block needs to be used instead.
  • ...as well as other minor changes like INTO being mandatory after INSERT.

But all this small differences are hidden inside of Signum.Engine, very few changes (if any) will surface to your code.

Another reason to skip the extesibility model is to avoid the Lowest Common Denominator problem that will occur if less capable DBMS will be supported. Fortunately the intersection of features between SQL Server and PostgreSQL is quite big.

What works right now?

All the LINQ provider tests work on both SQL Server and PostgreSQL, including:

  • Complex queries with joins, group by, aggregates etc...
  • Specific functions for strings, date times, etc...
  • UnsafeUpadate/UnsafeInsert/UnsafeDelete
  • BulkInsert

Also the Schema Synchronizer/Sql Migrations has been tested for the common cases, some rough edges are to be expected for more complex schema changes.

Only some functionality depending on SqlDependency, like Processes, still needs to be checked, and a in order to use TreeEntity some conditional logic using ltree will be needed.

Other than that, no PostgreSQL-specific bugs are known by me at this time, create database, run c# migration, run application, looks like everything works OK.

How to start using it?

Again, there are very few changes needed in your code to start using Postgres:

  • Replace SqlDbTypeAttribute by DbTypeAttribute (now with SqlDbType and NpgsqlDbType properties), if you have any. This needs to be done whether you want to use PostgreSQL or not.
  • Install the latest version of PostgreSQL. It's a simple process with no marketing bullshit where you will have to set your master password and select some components. You don't need the .Net Framework driver, but you probably want to install pgAdmin (the equialent of Microsoft Sql Management Studio but as web app).
  • Replace your connection string from something lik:
    Data Source=.\\SQLEXPRESS;Initial Catalog=Southwind;Integrated Security=true
    to something like:
    host=localhost;Username=postgres;Password=yourPassword;Database=Southwind
    To avoid committing the password you can use .Net Core UserSecrets feature.
  • In your starter, replace SqlConnector by PostgreSqlConnector and SqlServerVersionDetector by PostgresVersionDetector. Or make it dependent on the connection string like Southwind does.

That's it! Now you can create database, synchronize, generate environment, etc.. like we have been doing for 10 years!

image

@MehdyKarimpour
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Outstanding!

@KonstantinLukaschenko
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Awesome work, Olmo!

@MehdyKarimpour
Copy link
Contributor

@MehdyKarimpour MehdyKarimpour commented on 2895528 Jan 25, 2020 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rezanos
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Congratulations Olmo!
This is a tremendous major improvement on Signum, that could boost its ecosystem.
Thanks to your great and valuable efforts.
Congratulations everyone! :)

@olmobrutall
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

some things when migrating:

  1. SqlConnector is renamed to SqlServerConnector
  2. ObjectName / SchemaName / DatabaseName constructors require a isPostgres parameter, you can take it from Schema.Current.Settings.IsPostgres
  3. All the Indexes have been renamed to be globally unique, so an index in dbo.Alert that was named IX_Name not is named IX_Alert_Name, expect a big Sync script.

Please sign in to comment.