[open-ils-commits] [GIT] Evergreen ILS branch master updated. 698fbdc4211870d5dd547ec8a0e7302afccb9a50

[Evergreen Git]

This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Evergreen ILS".

The branch, master has been updated
       via  698fbdc4211870d5dd547ec8a0e7302afccb9a50 (commit)
      from  870a7fa4cb080a7e4eac550694b4a656fc278a83 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
commit 698fbdc4211870d5dd547ec8a0e7302afccb9a50
Author: Dan Wells <dbw2 at calvin.edu>
Date:   Fri Mar 2 12:54:46 2018 -0500

    LP#1738488 Optimize Flattener join logic
    
    The current Flattener.pm autogenerates necessary joins for sorting
    and filtering, but in doing so, it gives every intermediate table a
    unique alias, even if the path to that table is exactly the same as
    another member in the map we are flattening.
    
    Instead, let's reuse joins whenever the path is identical, even for
    intermediate tables.  We do so by tracking every path to each core
    type, then reusing as much of that join path as we can.  In cases
    where we have different paths to the same type, we still necessarily
    provide a new unique alias.
    
    This problem was first noticed in the web staff billing history
    interface, where the particular stacking of joins resulted (for one
    specific library) in 17 joins and 44,575,740,147,225,592,344,870,912
    potential rows.
    
    Signed-off-by: Dan Wells <dbw2 at calvin.edu>
    Signed-off-by: Galen Charlton <gmc at equinoxinitiative.org>

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm
index c967247..12918cc 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm
@@ -57,8 +57,8 @@ sub _flattened_search_single_flesh_wad {
 
 # returns a join clause AND a string representing the deepest join alias
 # generated.
-sub _flattened_search_single_join_clause {
-    my ($column_name, $hint, $path)  = @_;
+sub _flattened_search_add_join_clause {
+    my ($column_name, $hint, $path, $core_join, $path_tracker)  = @_;
 
     my $class = OpenSRF::Utils::JSON->lookup_class($hint);
     my $last_ident = $class->Identity;
@@ -67,17 +67,47 @@ sub _flattened_search_single_join_clause {
 
     pop @$path; # last part is just field
 
-    my $core_join = {};
     my $last_join;
     my $piece;
     my $alias;  # yes, we need it out at this scope.
 
+    my @path_key_parts;
+
     while ($piece = shift @$path) {
         my $link = _fm_link_from_class($class, $piece);
         if ($link) {
             $hint = $link->{class};
             $class = OpenSRF::Utils::JSON->lookup_class($hint);
 
+            push (@path_key_parts, ${piece});
+            my $path_key = "__" . join('__', @path_key_parts);
+            my $path_count;
+            if (!$path_tracker->{$hint}) {
+                # first time finding this IDL hint anywhere in the map,
+                # give it #1
+                $path_tracker->{$hint} = {$path_key => 1};
+                $path_count = 1;
+            } elsif ($path_count = $path_tracker->{$hint}{$path_key}) {
+                # we already have this exact path for this hint,
+                # pass
+            } else {
+                # we found a new path to this class, increment and store
+                # the version number
+                $path_count = keys %{$path_tracker->{$hint}}; # count the keys
+                $path_count++;
+                $path_tracker->{$hint}{$path_key} = $path_count;
+            }
+            $alias = "__${hint}_${path_count}";
+
+            # if we have already joined this segment, climb the tree
+            if ($last_join and $last_join->{join}{$alias}) {
+                $last_join = $last_join->{join}{$alias};
+                next;
+            } elsif ($core_join->{$alias}) {
+                $last_join = $core_join->{$alias};
+                next;
+            }
+
             my $reltype = $link->{reltype};
             my $field = $link->{key};
             if ($link->{map}) {
@@ -89,7 +119,6 @@ sub _flattened_search_single_join_clause {
                 );
             }
 
-            $alias = "__${column_name}_${hint}";
             my $new_join;
             if ($reltype eq "has_a") {
                 $new_join = {
@@ -181,12 +210,6 @@ sub _flattened_search_merge_flesh_wad {
     }
 }
 
-sub _flattened_search_merge_join_clause {
-    my ($old, $new) = @_;
-
-    %$old = ( %$old, %$new );
-}
-
 sub _flattened_search_expand_filter_column {
     my ($o, $key, $map) = @_;
 
@@ -264,6 +287,12 @@ sub process_map {
     # redundant joins.
     my $join_coverage = {};
 
+    # we need to be able to reference specific joined tables, but building
+    # aliases directly from the paths can exceed Postgres alias length limits
+    # (generally 63 characters).  Instead, we'll increment for each unique
+    # path to a given IDL class.
+    my $path_tracker = {};
+
     foreach my $k (keys %$map) {
         my $column = $map->{$k} =
             _flattened_search_normalize_map_column($map->{$k});
@@ -290,14 +319,13 @@ sub process_map {
                 ($clause, $last_join_alias) = @{ $join_coverage->{$joinkey} };
             } else {
                 ($clause, $last_join_alias) =
-                    _flattened_search_single_join_clause(
-                        $k, $hint, $column->{path}
+                    _flattened_search_add_join_clause(
+                        $k, $hint, $column->{path}, $jffolo->{join}, $path_tracker
                     );
                 $join_coverage->{$joinkey} = [$clause, $last_join_alias];
             }
 
             $map->{$k}{last_join_alias} = $last_join_alias;
-            _flattened_search_merge_join_clause($jffolo->{join}, $clause);
         }
     }
 

-----------------------------------------------------------------------

Summary of changes:
 .../perlmods/lib/OpenILS/Application/Flattener.pm  |   54 +++++++++++++++-----
 1 files changed, 41 insertions(+), 13 deletions(-)


hooks/post-receive
-- 
Evergreen ILS