Skip to content

Commit 11ff59c

Browse files
authored
Merge pull request #404 from DataObjects-NET/7.1-pgsql-schema-extraction-imp
Improves schema extraction for PostgreSQL
2 parents b26b68a + 3639ffc commit 11ff59c

File tree

6 files changed

+117
-68
lines changed

6 files changed

+117
-68
lines changed

ChangeLog/7.1.3_dev.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
[main] Addressed race condition issue with TranslatorState.NonVisitableExpressions
1+
[main] Addressed race condition issue with TranslatorState.NonVisitableExpressions
2+
[postgresql] Improved database structucture extraction

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/DriverFactory.cs

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -111,19 +111,15 @@ private static SqlDriver CreateDriverInstance(
111111

112112
// We support 8.3, 8.4 and any 9.0+
113113

114-
if (version.Major == 8) {
115-
return version.Minor == 3 ? new v8_3.Driver(coreServerInfo) : new v8_4.Driver(coreServerInfo);
116-
}
117-
118-
if (version.Major == 9) {
119-
return version.Minor == 0 ? new v9_0.Driver(coreServerInfo) : new v9_1.Driver(coreServerInfo);
120-
}
121-
122-
if (version.Major < 12) {
123-
return new v10_0.Driver(coreServerInfo);
124-
}
125-
126-
return new v12_0.Driver(coreServerInfo);
114+
return version.Major switch {
115+
8 when version.Minor == 3 => new v8_3.Driver(coreServerInfo),
116+
8 when version.Minor > 3 => new v8_4.Driver(coreServerInfo),
117+
9 when version.Minor == 0 => new v9_0.Driver(coreServerInfo),
118+
9 when version.Minor > 0 => new v9_1.Driver(coreServerInfo),
119+
10 => new v10_0.Driver(coreServerInfo),
120+
11 => new v10_0.Driver(coreServerInfo),
121+
_ => new v12_0.Driver(coreServerInfo)
122+
};
127123
}
128124

129125
/// <inheritdoc/>

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/Resources/Strings.Designer.cs

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/Resources/Strings.resx

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,4 +135,7 @@
135135
<data name="ExSchemaXDoesNotExistOrBelongsToAnotherUser" xml:space="preserve">
136136
<value>Schema '{0}' either does not exist or belongs to another user.</value>
137137
</data>
138+
<data name="ExCantFindSchemaXOwnerWithIdYInTheListOfRoles" xml:space="preserve">
139+
<value>Can't find schema '{0}' owner with oid '{1}' in the list of roles.</value>
140+
</data>
138141
</root>

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_0/Extractor.cs

Lines changed: 88 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,17 @@ protected sealed class ExtractionContext
2626
/// <summary>
2727
/// Specific schemas to extract
2828
/// </summary>
29-
public readonly Dictionary<string, Schema> TargetSchemes = new Dictionary<string, Schema>();
29+
public readonly Dictionary<string, Schema> TargetSchemes = new();
3030

3131
/// <summary>
32-
/// Extracted users.
32+
/// Extracted users (subset of <see cref="RoleLookup"/>).
3333
/// </summary>
34-
public readonly Dictionary<long, string> UserLookup = new Dictionary<long, string>();
34+
public readonly Dictionary<long, string> UserLookup = new();
35+
36+
/// <summary>
37+
/// Extracted roles.
38+
/// </summary>
39+
public readonly Dictionary<long, string> RoleLookup = new();
3540

3641
/// <summary>
3742
/// Catalog to extract information.
@@ -41,46 +46,54 @@ protected sealed class ExtractionContext
4146
/// <summary>
4247
/// Extracted schemas.
4348
/// </summary>
44-
public readonly Dictionary<long, Schema> SchemaMap = new Dictionary<long, Schema>();
49+
public readonly Dictionary<long, Schema> SchemaMap = new();
4550

4651
/// <summary>
4752
/// Extracted schemas identifiers.
4853
/// </summary>
49-
public readonly Dictionary<Schema, long> ReversedSchemaMap = new Dictionary<Schema, long>();
54+
public readonly Dictionary<Schema, long> ReversedSchemaMap = new();
5055

5156
/// <summary>
5257
/// Extracted tables.
5358
/// </summary>
54-
public readonly Dictionary<long, Table> TableMap = new Dictionary<long, Table>();
59+
public readonly Dictionary<long, Table> TableMap = new();
5560

5661
/// <summary>
5762
/// Extracted views.
5863
/// </summary>
59-
public readonly Dictionary<long, View> ViewMap = new Dictionary<long, View>();
64+
public readonly Dictionary<long, View> ViewMap = new();
6065

6166
/// <summary>
6267
/// Extracted sequences.
6368
/// </summary>
64-
public readonly Dictionary<long, Sequence> SequenceMap = new Dictionary<long, Sequence>();
69+
public readonly Dictionary<long, Sequence> SequenceMap = new();
6570

6671
/// <summary>
6772
/// Extracted index expressions.
6873
/// </summary>
69-
public readonly Dictionary<long, ExpressionIndexInfo> ExpressionIndexMap = new Dictionary<long, ExpressionIndexInfo>();
74+
public readonly Dictionary<long, ExpressionIndexInfo> ExpressionIndexMap = new();
7075

7176
/// <summary>
7277
/// Extracted domains.
7378
/// </summary>
74-
public readonly Dictionary<long, Domain> DomainMap = new Dictionary<long, Domain>();
79+
public readonly Dictionary<long, Domain> DomainMap = new();
7580

7681
/// <summary>
7782
/// Extracted columns connected grouped by owner (table or view)
7883
/// </summary>
79-
public readonly Dictionary<long, Dictionary<long, TableColumn>> TableColumnMap = new Dictionary<long, Dictionary<long, TableColumn>>();
84+
public readonly Dictionary<long, Dictionary<long, TableColumn>> TableColumnMap = new();
8085

86+
/// <summary>
87+
/// Roles in which current user is a member, self included.
88+
/// </summary>
89+
public readonly List<long> CurrentUserRoles = new();
90+
91+
public string CurrentUserName { get; set; }
8192
public long CurrentUserSysId { get; set; } = -1;
8293
public long? CurrentUserIdentifier { get; set; }
8394

95+
public bool IsMe(string name) => name == CurrentUserName;
96+
8497
public ExtractionContext(Catalog catalog)
8598
{
8699
Catalog = catalog;
@@ -332,7 +345,7 @@ public override Catalog ExtractSchemes(string catalogName, string[] schemaNames)
332345
{
333346
var (catalog, context) = CreateCatalogAndContext(catalogName, schemaNames);
334347

335-
ExtractUsers(context);
348+
_ = ExtractRoles(context, false);
336349
ExtractSchemas(context);
337350
return catalog;
338351
}
@@ -343,7 +356,7 @@ public override async Task<Catalog> ExtractSchemesAsync(
343356
{
344357
var (catalog, context) = CreateCatalogAndContext(catalogName, schemaNames);
345358

346-
await ExtractUsersAsync(context, token).ConfigureAwait(false);
359+
await ExtractRoles(context, true, token).ConfigureAwait(false);
347360
await ExtractSchemasAsync(context, token).ConfigureAwait(false);
348361
return catalog;
349362
}
@@ -360,48 +373,62 @@ private static (Catalog catalog, ExtractionContext context) CreateCatalogAndCont
360373
return (catalog, context);
361374
}
362375

363-
private void ExtractUsers(ExtractionContext context)
376+
private async ValueTask ExtractRoles(ExtractionContext context, bool isAsync, CancellationToken token = default)
364377
{
365378
context.UserLookup.Clear();
366-
string me;
367-
using (var command = Connection.CreateCommand("SELECT user")) {
368-
me = (string) command.ExecuteScalar();
369-
}
370379

371-
using (var cmd = Connection.CreateCommand("SELECT usename, usesysid FROM pg_user"))
372-
using (var dr = cmd.ExecuteReader()) {
373-
while (dr.Read()) {
374-
ReadUserData(dr, context, me);
380+
var extractCurentUserCommand = Connection.CreateCommand("SELECT user");
381+
// Roles include users.
382+
// Users also can have members for some reason and it doesn't make them groups,
383+
// the only thing that defines user is ability to login :-)
384+
const string ExtractRolesQueryTemplate = "SELECT rolname, oid, rolcanlogin, pg_has_role('{0}', oid,'member') FROM pg_roles";
385+
386+
387+
if (isAsync) {
388+
await using (extractCurentUserCommand.ConfigureAwait(false)) {
389+
context.CurrentUserName = (string) await extractCurentUserCommand.ExecuteScalarAsync(token).ConfigureAwait(false);
375390
}
376-
}
377-
}
378391

379-
private async Task ExtractUsersAsync(ExtractionContext context, CancellationToken token = default)
380-
{
381-
context.UserLookup.Clear();
382-
string me;
383-
var command = Connection.CreateCommand("SELECT user");
384-
await using (command.ConfigureAwait(false)) {
385-
me = (string) await command.ExecuteScalarAsync(token).ConfigureAwait(false);
392+
var getAllUsersCommand = Connection.CreateCommand(string.Format(ExtractRolesQueryTemplate, context.CurrentUserName));
393+
await using (getAllUsersCommand.ConfigureAwait(false)) {
394+
var reader = await getAllUsersCommand.ExecuteReaderAsync(token).ConfigureAwait(false);
395+
await using (reader.ConfigureAwait(false)) {
396+
while (await reader.ReadAsync(token).ConfigureAwait(false)) {
397+
ReadUserData(reader, context);
398+
}
399+
}
400+
}
386401
}
402+
else {
403+
using (extractCurentUserCommand) {
404+
context.CurrentUserName = (string) extractCurentUserCommand.ExecuteScalar();
405+
}
387406

388-
command = Connection.CreateCommand("SELECT usename, usesysid FROM pg_user");
389-
await using (command.ConfigureAwait(false)) {
390-
var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false);
391-
await using (reader.ConfigureAwait(false)) {
392-
while (await reader.ReadAsync(token).ConfigureAwait(false)) {
393-
ReadUserData(reader, context, me);
407+
var getAllUsersCommand = Connection.CreateCommand(string.Format(ExtractRolesQueryTemplate, context.CurrentUserName));
408+
using (getAllUsersCommand)
409+
using (var dr = getAllUsersCommand.ExecuteReader()) {
410+
while (dr.Read()) {
411+
ReadUserData(dr, context);
394412
}
395413
}
396414
}
397415
}
398416

399-
private static void ReadUserData(DbDataReader dr, ExtractionContext context, string me)
417+
private static void ReadUserData(DbDataReader dr, ExtractionContext context)
400418
{
401-
var name = dr[0].ToString();
419+
var name = dr.GetString(0);
420+
// oid, which is basically a number, has its own type - oid! can't be read as int or long (facepalm)
402421
var sysId = Convert.ToInt64(dr[1]);
403-
context.UserLookup.Add(sysId, name);
404-
if (name == me) {
422+
var canLogin = dr.GetBoolean(2);
423+
var containsCurrentUser = dr.GetBoolean(3);
424+
context.RoleLookup.Add(sysId, name);
425+
if(containsCurrentUser) {
426+
context.CurrentUserRoles.Add(sysId);
427+
}
428+
if (canLogin) {
429+
context.UserLookup.Add(sysId, name);
430+
}
431+
if (context.IsMe(name)) {
405432
context.CurrentUserSysId = sysId;
406433
}
407434
}
@@ -499,7 +526,11 @@ protected virtual SqlQueryExpression BuildExtractSchemasQuery(ExtractionContext
499526
selectPublic.Columns.Add(namespaceTable1["nspowner"]);
500527

501528
var selectMine = SqlDml.Select(namespaceTable2);
502-
selectMine.Where = namespaceTable2["nspowner"] == context.CurrentUserIdentifier;
529+
if (context.CurrentUserRoles.Count == 0)
530+
selectMine.Where = namespaceTable2["nspowner"] == context.CurrentUserIdentifier;
531+
else {
532+
selectMine.Where = SqlDml.In(namespaceTable2["nspowner"], SqlDml.Array(context.CurrentUserRoles.ToArray()));
533+
}
503534
selectMine.Columns.Add(namespaceTable2["nspname"]);
504535
selectMine.Columns.Add(namespaceTable2["oid"]);
505536
selectMine.Columns.Add(namespaceTable2["nspowner"]);
@@ -522,7 +553,12 @@ protected virtual void ReadSchemaData(DbDataReader dataReader, ExtractionContext
522553
catalog.DefaultSchema = schema;
523554
}
524555

525-
schema.Owner = context.UserLookup[owner];
556+
if (context.RoleLookup.TryGetValue(owner, out var userName)) {
557+
schema.Owner = userName;
558+
}
559+
else {
560+
throw new InvalidOperationException(string.Format(Resources.Strings.ExCantFindSchemaXOwnerWithIdYInTheListOfRoles, name, owner));
561+
}
526562
context.SchemaMap[oid] = schema;
527563
context.ReversedSchemaMap[schema] = oid;
528564
}
@@ -594,8 +630,8 @@ protected virtual ISqlCompileUnit BuildExtractSchemaContentsQuery(ExtractionCont
594630
select.Columns.Add(relationsTable["relnamespace"]);
595631
select.Columns.Add(tablespacesTable["spcname"]);
596632
select.Columns.Add(new Func<SqlCase>(() => {
597-
var defCase = SqlDml.Case(relationsTable["relkind"]);
598-
defCase.Add('v', SqlDml.FunctionCall("pg_get_viewdef", relationsTable["oid"]));
633+
var defCase = SqlDml.Case(relationsTable["relkind"])
634+
.Add('v', SqlDml.FunctionCall("pg_get_viewdef", relationsTable["oid"]));
599635
return defCase;
600636
})(), "definition");
601637
return select;
@@ -741,7 +777,7 @@ protected virtual void ReadColumnData(DbDataReader dataReader, ExtractionContext
741777
}
742778
else {
743779
var view = viewMap[columnOwnerId];
744-
view.CreateColumn(columnName);
780+
_ = view.CreateColumn(columnName);
745781
}
746782
}
747783

@@ -912,8 +948,9 @@ protected virtual int ReadTableIndexData(DbDataReader dataReader, ExtractionCont
912948
else {
913949
for (int j = 0; j < indexKey.Length; j++) {
914950
int colIndex = indexKey[j];
915-
if (colIndex > 0)
916-
index.CreateIndexColumn(tableColumns[tableIdentifier][colIndex], true);
951+
if (colIndex > 0) {
952+
_ = index.CreateIndexColumn(tableColumns[tableIdentifier][colIndex], true);
953+
}
917954
else {
918955
//column index is 0
919956
//this means that this index column is an expression
@@ -967,12 +1004,9 @@ protected virtual void ReadIndexColumnsData(DbDataReader dataReader, ExtractionC
9671004
var exprIndexInfo = expressionIndexMap[Convert.ToInt64(dataReader[1])];
9681005
for (var j = 0; j < exprIndexInfo.Columns.Length; j++) {
9691006
int colIndex = exprIndexInfo.Columns[j];
970-
if (colIndex > 0) {
971-
exprIndexInfo.Index.CreateIndexColumn(tableColumns[Convert.ToInt64(dataReader[0])][colIndex], true);
972-
}
973-
else {
974-
exprIndexInfo.Index.CreateIndexColumn(SqlDml.Native(dataReader[(j + 1).ToString()].ToString()));
975-
}
1007+
_ = colIndex > 0
1008+
? exprIndexInfo.Index.CreateIndexColumn(tableColumns[Convert.ToInt64(dataReader[0])][colIndex], true)
1009+
: exprIndexInfo.Index.CreateIndexColumn(SqlDml.Native(dataReader[(j + 1).ToString()].ToString()));
9761010
}
9771011
}
9781012

Orm/Xtensive.Orm.Tests.Framework/Orm.config

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@
7272
<domain name="pgsql150"
7373
connectionUrl="postgresql://dotest:dotest@localhost:54150/dotest" />
7474

75+
<domain name="pgsql160"
76+
connectionUrl="postgresql://dotest:dotest@localhost:54160/dotest" />
77+
7578
<domain name="oracle10"
7679
connectionUrl="oracle://dotest:dotest@localhost:5510/ora10" />
7780

@@ -167,6 +170,9 @@
167170
<domain name="pgsql150cs" provider="postgresql"
168171
connectionString="HOST=localhost;PORT=54150;DATABASE=dotest;USER ID=dotest;PASSWORD=dotest" />
169172

173+
<domain name="pgsql160cs" provider="postgresql"
174+
connectionString="HOST=localhost;PORT=54160;DATABASE=dotest;USER ID=dotest;PASSWORD=dotest" />
175+
170176
<domain name="oracle10cs" provider="oracle"
171177
connectionString="DATA SOURCE=&quot;(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=localhost)(PORT=5510))(CONNECT_DATA=(SERVICE_NAME=ora10)))&quot;;USER ID=dotest;PASSWORD=dotest" />
172178

0 commit comments

Comments
 (0)