diff --git a/edb/edgeql/compiler/expr.py b/edb/edgeql/compiler/expr.py index 2b48f375eb3..2b9a7a850f6 100644 --- a/edb/edgeql/compiler/expr.py +++ b/edb/edgeql/compiler/expr.py @@ -1047,7 +1047,9 @@ def _infer_slice_type( def compile_Indirection( expr: qlast.Indirection, *, ctx: context.ContextLevel ) -> irast.Set: - node: irast.Set | irast.Expr = dispatch.compile(expr.arg, ctx=ctx) + node: irast.Set | irast.IndexIndirection | irast.SliceIndirection = ( + dispatch.compile(expr.arg, ctx=ctx) + ) for indirection_el in expr.indirection: if isinstance(indirection_el, qlast.Index): idx = dispatch.compile(indirection_el.index, ctx=ctx) @@ -1057,7 +1059,11 @@ def compile_Indirection( ) node = irast.IndexIndirection( - expr=node, index=idx, typeref=typeref, span=expr.span + expr=node, + index=idx, + input_typeref=node.typeref, + typeref=typeref, + span=expr.span, ) elif isinstance(indirection_el, qlast.Slice): start: Optional[irast.Base] diff --git a/edb/edgeql/compiler/setgen.py b/edb/edgeql/compiler/setgen.py index 6ac0b238574..19fe170aff1 100644 --- a/edb/edgeql/compiler/setgen.py +++ b/edb/edgeql/compiler/setgen.py @@ -2043,7 +2043,41 @@ def get_global_param( param_name = f'__edb_global_{len(ctx.env.query_globals)}__' target = glob.get_target(ctx.env.schema) - target_typeref = typegen.type_to_typeref(target, env=ctx.env) + + target_typeref: irast.TypeRef + if ( + glob.get_expr(ctx.env.schema) is None + and target.contains_array_of_array(ctx.env.schema) + ): + # User specified global with array> should treat it + # as array>> + padded_collection_type: s_types.Collection + if isinstance(target, s_types.Array): + padded_collection_type = s_types.Array.get_padded_collection( + ctx.env.schema, + element_type=target.get_element_type(ctx.env.schema), + ) + elif isinstance(target, s_types.Tuple): + padded_collection_type = s_types.Tuple.get_padded_collection( + ctx.env.schema, + element_types={ + n: st + for n, st in ( + target + .get_element_types(ctx.env.schema) + .items(ctx.env.schema) + ) + }, + named=target.get_named(ctx.env.schema), + ) + else: + raise NotImplementedError + target_typeref = typegen.type_to_typeref( + padded_collection_type, env=ctx.env + ) + + else: + target_typeref = typegen.type_to_typeref(target, env=ctx.env) ctx.env.query_globals[name] = irast.Global( name=param_name, @@ -2073,6 +2107,7 @@ def get_global_param_sets( typeref=param.ir_type, is_implicit_global=is_implicit_global, ), + type_override=param.schema_type, ctx=ctx, ) if glob.needs_present_arg(ctx.env.schema): diff --git a/edb/ir/ast.py b/edb/ir/ast.py index 7eb216975fa..7d3b0dd2f58 100644 --- a/edb/ir/ast.py +++ b/edb/ir/ast.py @@ -160,6 +160,11 @@ class TypeRef(ImmutableBase): collection: typing.Optional[str] = None # Collection subtypes if this is a collection subtypes: tuple[TypeRef, ...] = () + # If this is array>, the padded type is array>> + padded_array_type: typing.Optional[TypeRef] = None + # If this is array<...> and tuple> is persisted + # in the schema, this is the tuple's id + wrapped_array_id: typing.Optional[uuid.UUID] = None # True, if this describes a scalar type is_scalar: bool = False # True, if this describes a view @@ -1081,8 +1086,9 @@ class OperatorCall(Call): class IndexIndirection(ImmutableExpr): - expr: Base + expr: Set | IndexIndirection | SliceIndirection index: Base + input_typeref: TypeRef typeref: TypeRef diff --git a/edb/ir/typeutils.py b/edb/ir/typeutils.py index cf13915f4bc..59ecb097dfc 100644 --- a/edb/ir/typeutils.py +++ b/edb/ir/typeutils.py @@ -116,6 +116,11 @@ def is_array(typeref: irast.TypeRef) -> bool: return typeref.collection == s_types.Array.get_schema_name() +def is_array_of_array(typeref: irast.TypeRef) -> bool: + """Return True if *typeref* describes an array of array type.""" + return is_array(typeref) and is_array(typeref.subtypes[0]) + + def is_tuple(typeref: irast.TypeRef) -> bool: """Return True if *typeref* describes a tuple type.""" return typeref.collection == s_types.Tuple.get_schema_name() @@ -191,6 +196,18 @@ def is_persistent_tuple(typeref: irast.TypeRef) -> bool: return False +def is_persistent_array_of_array(typeref: irast.TypeRef) -> bool: + if is_array_of_array(typeref): + if typeref.material_type is not None: + material = typeref.material_type + else: + material = typeref + + return material.in_schema + else: + return False + + def is_empty_typeref(typeref: irast.TypeRef) -> bool: return typeref.union is not None and len(typeref.union) == 0 @@ -480,6 +497,31 @@ def _typeref( else: material_typeref = None + padded_array_typeref = None + if ( + isinstance(t, s_types.Array) + and (element_type := t.get_element_type(schema)) + and isinstance(element_type, s_types.Array) + ): + padded_collection_type = s_types.Array.get_padded_collection( + schema, element_type=element_type + ) + padded_array_typeref = _typeref(padded_collection_type) + + wrapped_array_id = None + if isinstance(t, s_types.Array): + wrapped_array_name = s_types.Tuple.generate_name( + element_names={'f1': t.get_name(schema)}, + named=True, + ) + if ( + (wrapped_array_type := schema.get_global( + s_types.Tuple, wrapped_array_name, None + )) + and wrapped_array_type.get_is_persistent(schema) + ): + wrapped_array_id = wrapped_array_type.id + result = irast.TypeRef( id=t.id, name_hint=name_hint, @@ -490,7 +532,9 @@ def _typeref( in_schema=t.get_is_persistent(schema), subtypes=tuple( _typeref(st) for st in t.get_subtypes(schema) - ) + ), + padded_array_type=padded_array_typeref, + wrapped_array_id=wrapped_array_id, ) if cache is not None and typename is None and _name is None: diff --git a/edb/pgsql/compiler/astutils.py b/edb/pgsql/compiler/astutils.py index e3191a04525..3181fa0b77e 100644 --- a/edb/pgsql/compiler/astutils.py +++ b/edb/pgsql/compiler/astutils.py @@ -139,84 +139,113 @@ def array_get_inner_array( an element is accessed, it needs to be unwrapped. Essentially, this function takes tuple> and returns array<...> - - Postgres does not support arbitrarily accessing fields out of unnamed - composites and so we need to do an extra unnest(array[]) to be able to - specify the name and type our resulting array. - - For example, the query: `select [[1]][0];` will produce the following SQL: - - SELECT - "expr-6~2"."array_value~4" AS "array_serialized~1" - FROM - LATERAL - (SELECT - "expr-5~2"."array_value~3" AS "array_value~4" - FROM - LATERAL - (SELECT - (SELECT - "0" - FROM - -- EXTRA unnest(array[]) - unnest(ARRAY[ - -- INDEX INDIRECTION - edgedb_v7_2f26206480._index( - "expr-3~2"."array_value~2", - ($2)::int8, - 'ERROR MESSAGE' - ) - ]) AS ("0" int8[]) - ) AS "array_value~3" - FROM - LATERAL - -- INITAL ARRAY [[1]] - (SELECT - ARRAY[ROW("expr-2~2"."array_value~1")] - AS "array_value~2" - FROM - LATERAL - (SELECT - ARRAY[($1)::int8] - AS "array_value~1" - ) AS "expr-2~2" - ) AS "expr-3~2" - ) AS "expr-5~2" - ) AS "expr-6~2" - WHERE - ("expr-6~2"."array_value~4" IS NOT NULL) - LIMIT - (SELECT - (101)::int8 AS "expr~7_value~1" - ) """ - return pgast.SelectStmt( - target_list=[ - pgast.ResTarget(val=pgast.ColumnRef(name=['0'])), - ], - from_clause=[ - pgast.RangeFunction( - functions=[ - pgast.FuncCall( - name=('unnest',), - args=[ - pgast.ArrayExpr( - elements=[wrapped_array], - ) - ], - coldeflist=[ - pgast.ColumnDef( - name='0', - typename=pgast.TypeName( - name=pg_types.pg_type_from_ir_typeref(array_typeref) + # Nested arrays wrap their inner arrays in a tuple + if array_typeref.wrapped_array_id: + # temporary array of array implemented as array>> + # using named tuple + return pgast.SelectStmt( + target_list=[ + pgast.ResTarget(val=pgast.ColumnRef(name=['f1'])), + ], + from_clause=[ + pgast.RangeFunction( + functions=[ + pgast.FuncCall( + name=('unnest',), + args=[ + pgast.ArrayExpr( + elements=[wrapped_array], ) - ) - ] - ) - ] + ], + ) + ] + ) + ] + ) + else: + # temporary array of array implemented as array>> + # using unnamed tuple + """ + Postgres does not support arbitrarily accessing fields out of unnamed + composites and so we need to do an extra unnest(array[]) to be able to + specify the name and type our resulting array. + + For example, the query: `select [[1]][0];` will produce the following + SQL: + + SELECT + "expr-6~2"."array_value~4" AS "array_serialized~1" + FROM + LATERAL + (SELECT + "expr-5~2"."array_value~3" AS "array_value~4" + FROM + LATERAL + (SELECT + (SELECT + "0" + FROM + -- EXTRA unnest(array[]) + unnest(ARRAY[ + -- INDEX INDIRECTION + edgedb_v7_2f26206480._index( + "expr-3~2"."array_value~2", + ($2)::int8, + 'ERROR MESSAGE' + ) + ]) AS ("0" int8[]) + ) AS "array_value~3" + FROM + LATERAL + -- INITAL ARRAY [[1]] + (SELECT + ARRAY[ROW("expr-2~2"."array_value~1")] + AS "array_value~2" + FROM + LATERAL + (SELECT + ARRAY[($1)::int8] + AS "array_value~1" + ) AS "expr-2~2" + ) AS "expr-3~2" + ) AS "expr-5~2" + ) AS "expr-6~2" + WHERE + ("expr-6~2"."array_value~4" IS NOT NULL) + LIMIT + (SELECT + (101)::int8 AS "expr~7_value~1" ) - ] - ) + """ + + return pgast.SelectStmt( + target_list=[ + pgast.ResTarget(val=pgast.ColumnRef(name=['0'])), + ], + from_clause=[ + pgast.RangeFunction( + functions=[ + pgast.FuncCall( + name=('unnest',), + args=[ + pgast.ArrayExpr( + elements=[wrapped_array], + ) + ], + coldeflist=[ + pgast.ColumnDef( + name='0', + typename=pgast.TypeName( + name=pg_types.pg_type_from_ir_typeref(array_typeref) + ) + ) + ] + ) + ] + ) + ] + ) def is_null_const(expr: pgast.BaseExpr) -> bool: diff --git a/edb/pgsql/compiler/output.py b/edb/pgsql/compiler/output.py index ee5e0fa87d4..7732bde8dde 100644 --- a/edb/pgsql/compiler/output.py +++ b/edb/pgsql/compiler/output.py @@ -248,14 +248,20 @@ def array_as_json_object( # To prevent this, we need to explicitly serialize the inner arrays then # aggregate them. el_name = 'f1' - coldeflist = [ - pgast.ColumnDef( - name=str(el_name), - typename=pgast.TypeName( - name=pgtypes.pg_type_from_ir_typeref(el_type), - ), - ) - ] + if ( + styperef.padded_array_type is not None + and styperef.padded_array_type.in_schema + ): + coldeflist = None + else: + coldeflist = [ + pgast.ColumnDef( + name=str(el_name), + typename=pgast.TypeName( + name=pgtypes.pg_type_from_ir_typeref(el_type), + ), + ) + ] unwrapped_inner_array = pgast.RangeFunction( functions=[ pgast.FuncCall( diff --git a/edb/pgsql/compiler/relgen.py b/edb/pgsql/compiler/relgen.py index 313f854f115..65d6a1b23c4 100644 --- a/edb/pgsql/compiler/relgen.py +++ b/edb/pgsql/compiler/relgen.py @@ -3238,6 +3238,20 @@ def _process_nested_array_set_func_with_ordinality( # Get (ordinal, tuple>) # - unnests the outer array with ordinal # - select the resulting ordinal and tuple> + if inner_rtype.wrapped_array_id: + # tuple in schema + coldeflist = None + else: + coldeflist = [ + pgast.ColumnDef( + name='0', + typename=pgast.TypeName( + name=pg_types.pg_type_from_ir_typeref( + inner_rtype + ) + ) + ) + ] ordinal_tuple_expr = pgast.SelectStmt( target_list=[ pgast.ResTarget( @@ -3253,16 +3267,7 @@ def _process_nested_array_set_func_with_ordinality( pgast.FuncCall( name=('unnest',), args=[args[0]], - coldeflist=[ - pgast.ColumnDef( - name='0', - typename=pgast.TypeName( - name=pg_types.pg_type_from_ir_typeref( - inner_rtype - ) - ) - ) - ] + coldeflist=coldeflist ) ], lateral=True, @@ -3452,9 +3457,14 @@ def _process_set_func( else: colnames = [ctx.env.aliases.get('v')] - if _should_unwrap_polymorphic_return_array(expr): - # If we are unwrapping a previously nested array, its pg type is - # record and so we need to provide a column definition list. + if ( + _should_unwrap_polymorphic_return_array(expr) + and not expr.typeref.wrapped_array_id + ): + # Unwrapping a previously nested array. + # If the wrapped array (ie. tuple>) is not in the schema, + # its pg type is record and so we need to provide a column + # definition list. coldeflist = [ pgast.ColumnDef( name='v', @@ -3699,7 +3709,9 @@ def _compile_call_args( _should_wrap_polymorphic_array_args(expr) and _is_array_arg_as_simple_polymorphic(ir_arg) ): - arg_ref = pgast.RowExpr(args=[arg_ref]) + arg_ref = _wrap_array_of_array_element( + arg_ref, ir_arg.expr.typeref, + ) args.append(arg_ref) _compile_arg_null_check(expr, ir_arg, arg_ref, typemod, ctx=ctx) @@ -4102,13 +4114,30 @@ def process_set_as_agg_expr_inner( _should_wrap_polymorphic_array_args(expr) and _is_array_arg_as_simple_polymorphic(ir_call_arg) ): - # Wrap aggregated arrays in a tuple - arg_ref = pgast.RowExpr(args=[arg_ref]) - if aspect == pgce.PathAspect.SERIALIZED: + # In serialized json output, arg_ref will already be + # a postgres json and should not be wrapped in a tuple. + # + # Otherwise, wrap aggregated inner arrays in a tuple. + if argctx.env.output_format not in ( + context.OutputFormat.JSON, + context.OutputFormat.JSON_ELEMENTS, + context.OutputFormat.JSONB + ): + arg_ref = _wrap_array_of_array_element( + arg_ref, ir_arg.expr.typeref + ) + arg_ref = output.serialize_expr( arg_ref, path_id=ir_arg.path_id, env=argctx.env) + else: + # Not serialized, wrap aggregated inner arrays in + # a tuple + arg_ref = _wrap_array_of_array_element( + arg_ref, ir_arg.expr.typeref + ) + args.append(arg_ref) name = exprcomp.get_func_call_backend_name(expr, ctx=newctx) @@ -4327,14 +4356,15 @@ def process_set_as_json_object_pack( def build_array_expr( - ir_expr: irast.Base, - elements: list[pgast.BaseExpr], *, - ctx: context.CompilerContextLevel) -> pgast.BaseExpr: + ir_expr: irast.Array, + elements: list[pgast.BaseExpr], + *, + ctx: context.CompilerContextLevel +) -> pgast.BaseExpr: array = astutils.safe_array_expr(elements, ctx=ctx) if irutils.is_empty_array_expr(ir_expr): - assert isinstance(ir_expr, irast.Array) typeref = ir_expr.typeref if irtyputils.is_any(typeref.subtypes[0]): @@ -4355,6 +4385,20 @@ def build_array_expr( name=pg_type, ), ) + + elif irtyputils.is_persistent_array_of_array(ir_expr.typeref): + assert ir_expr.typeref.padded_array_type is not None + pg_type = pg_types.pg_type_from_ir_typeref( + ir_expr.typeref.padded_array_type + ) + + return pgast.TypeCast( + arg=array, + type_name=pgast.TypeName( + name=pg_type, + ), + ) + else: return array @@ -4376,7 +4420,9 @@ def process_set_as_array_expr( element = dispatch.compile(ir_element, ctx=ctx) if irtyputils.is_array(ir_element.typeref): # Wrap nested arrays in a tuple - element = pgast.RowExpr(args=[element]) + element = _wrap_array_of_array_element( + element, ir_element.typeref, + ) elements.append(element) if serializing: @@ -4419,6 +4465,29 @@ def process_set_as_array_expr( return new_stmt_set_rvar(ir_set, ctx.rel, ctx=ctx) +def _wrap_array_of_array_element( + element: pgast.BaseExpr, + typeref: irast.TypeRef, +) -> pgast.BaseExpr: + # Wrap aggregated arrays in a tuple + if typeref.wrapped_array_id: + # tuple in schema + name = cast( + tuple[str, str], + common.get_tuple_backend_name( + typeref.wrapped_array_id, catenate=False + ), + ) + return pgast.TypeCast( + arg=pgast.RowExpr(args=[element]), + type_name=pgast.TypeName( + name=name, + ), + ) + else: + return pgast.RowExpr(args=[element]) + + def process_encoded_param( param: irast.Param, *, ctx: context.CompilerContextLevel) -> pgast.BaseExpr: diff --git a/edb/pgsql/types.py b/edb/pgsql/types.py index a6fe1f7fcb2..69773cf3b92 100644 --- a/edb/pgsql/types.py +++ b/edb/pgsql/types.py @@ -383,7 +383,17 @@ def pg_type_from_ir_typeref( and irtyputils.is_scalar(ir_typeref.subtypes[0]))): return ('anyarray',) elif irtyputils.is_array(ir_typeref.subtypes[0]): - return ('record[]',) + if ( + ir_typeref.padded_array_type is not None + and ir_typeref.padded_array_type.in_schema + ): + tp = pg_type_from_ir_typeref( + ir_typeref.padded_array_type.subtypes[0], + serialized=serialized, + persistent_tuples=persistent_tuples) + return pg_type_array(tp) + else: + return ('record[]',) else: tp = pg_type_from_ir_typeref( ir_typeref.subtypes[0], diff --git a/edb/schema/expraliases.py b/edb/schema/expraliases.py index 76f2a550606..6cc9e4fa4c1 100644 --- a/edb/schema/expraliases.py +++ b/edb/schema/expraliases.py @@ -527,41 +527,59 @@ def _create_alias_types( created_type_shells: set[so.ObjectShell[s_types.Type]] = set() - for ty_id in irutils.collect_schema_types(ir.expr): - if schema.has_object(ty_id): - # this is not a new type, skip - continue - ty = new_schema.get_by_id(ty_id, type=s_types.Type) - - name = ty.get_name(new_schema) - if ( - not isinstance(ty, s_types.Collection) - and not _has_alias_name_prefix(classname, name) - ): - # not all created types are visible from the root, so they don't - # need to be created in the schema - continue - - new_schema = ty.update( - new_schema, - dict( - alias_is_persistent=True, - expr_type=s_types.ExprType.Select, - from_alias=True, - from_global=is_global, - internal=False, - builtin=False, - ), - ) - if isinstance(ty, s_types.Collection): - new_schema = ty.set_field_value(new_schema, 'is_persistent', True) + new_types: list[s_types.Type] = [ + new_schema.get_by_id(ty_id, type=s_types.Type) + for ty_id in irutils.collect_schema_types(ir.expr) + if not schema.has_object(ty_id) + ] + + while new_types: + additional_types: list[s_types.Type] = [] + + for ty in new_types: + name = ty.get_name(new_schema) + if ( + not isinstance(ty, s_types.Collection) + and not _has_alias_name_prefix(classname, name) + ): + # not all created types are visible from the root, so they don't + # need to be created in the schema + continue + + new_schema = ty.update( + new_schema, + dict( + alias_is_persistent=True, + expr_type=s_types.ExprType.Select, + from_alias=True, + from_global=is_global, + internal=False, + builtin=False, + ), + ) + if isinstance(ty, s_types.Collection): + new_schema = ty.set_field_value( + new_schema, 'is_persistent', True + ) + + # Since the type shells here are not created by create_shell, + # ensure the padded nested arrays are created for alias types. + additional_types.extend(ty.get_padded_types(new_schema)) - derived_delta.add( - ty.as_create_delta( - schema=new_schema, context=so.ComparisonContext() + derived_delta.add( + ty.as_create_delta( + schema=new_schema, context=so.ComparisonContext() + ) ) - ) - created_type_shells.add(so.ObjectShell(name=name, schemaclass=type(ty))) + created_type_shells.add( + so.ObjectShell(name=name, schemaclass=type(ty)) + ) + + new_types = [ + ty + for ty in set(additional_types) + if not schema.has_object(ty.get_id(new_schema)) + ] derived_delta = s_ordering.linearize_delta( derived_delta, old_schema=schema, new_schema=new_schema diff --git a/edb/schema/properties.py b/edb/schema/properties.py index 0ab43bc76ca..4bc1acbd3d9 100644 --- a/edb/schema/properties.py +++ b/edb/schema/properties.py @@ -276,6 +276,16 @@ def validate_object( f'{target_type.get_verbosename(schema)}', span=span, ) + if ( + target_type.contains_array_of_array(schema) + and self.get_attribute_value('expr') is None + ): + span = self.get_attribute_span('target') + raise errors.InvalidPropertyTargetError( + 'invalid property type: non-computed nested arrays are not ' + 'supported', + span=span, + ) def _check_field_errors(self, node: qlast.DDLOperation) -> None: for sub in node.commands: diff --git a/edb/schema/types.py b/edb/schema/types.py index c44e5845c4a..992f83c693d 100644 --- a/edb/schema/types.py +++ b/edb/schema/types.py @@ -1169,6 +1169,63 @@ def as_type_delete_if_unused( if_exists=True, ) + def get_padded_types(self, schema: s_schema.Schema) -> list[Type]: + """Get all types produced by padding arrays of arrays. + + Examples: + array> -> [ + array>>, + tuple>, + ] + tuple>> -> [ + tuple>>>, + array>>, + tuple>, + ] + array>> -> [ + array>>>>, + tuple>>>, + array>>, + tuple>, + ] + """ + + # TODO: It would be best for the get_padded_collection functions + # to call this function to prevent needless double recursion. + + if not self.contains_array_of_array(schema): + return [] + + result: list[Type] = [] + + padded_type: Type + if isinstance(self, Array): + element_type = self.get_element_type(schema) + padded_type = Array.get_padded_collection( + schema, element_type=element_type + ) + result.append(padded_type) + if self.is_array_of_arrays(schema): + result.append(padded_type.get_element_type(schema)) + + if isinstance(element_type, Collection): + result.extend(element_type.get_padded_types(schema)) + + elif isinstance(self, Tuple): + element_types = dict(self.get_element_types(schema).items(schema)) + padded_type = Tuple.get_padded_collection( + schema, + element_types=element_types, + named=self.get_named(schema), + ) + result.append(padded_type) + + for _, element_type in element_types.items(): + if isinstance(element_type, Collection): + result.extend(element_type.get_padded_types(schema)) + + return result + Dimensions = checked.FrozenCheckedList[int] Array_T = typing.TypeVar("Array_T", bound="Array") @@ -1283,6 +1340,20 @@ def create( result = schema.get_global(cls, name, default=None) if result is None: + if ( + isinstance(element_type, Array) + or ( + isinstance(element_type, Type) + and element_type.contains_array_of_array(schema) + ) + ): + # Nested arrays are represented as array> + # and so we need to ensure these types also exist. + schema, _ = Array.create_padded_collection( + schema, + element_type=element_type, + ) + schema, result = super().create_in_schema( schema, id=id, @@ -1296,6 +1367,82 @@ def create( return schema, result + @classmethod + def create_padded_collection( + cls, + schema: s_schema.Schema, + *, + element_type: Any, + ) -> tuple[s_schema.Schema, Array]: + # Create the corresponding array>> for a nested array + + # Recursively pad any inner nested arrays + if isinstance(element_type, Array): + schema, element_type = Array.create_padded_collection( + schema, + element_type=element_type.get_element_type(schema), + ) + schema, element_type = Tuple.create( + schema, + element_types={ + 'f1': element_type + }, + named=True, + ) + elif isinstance(element_type, Tuple): + schema, element_type = Tuple.create_padded_collection( + schema, + element_types=dict( + element_type.get_element_types(schema).items(schema) + ), + named=element_type.get_named(schema), + ) + + return Array.create( + schema, + element_type=element_type, + ) + + @classmethod + def get_padded_collection( + cls, + schema: s_schema.Schema, + *, + element_type: Any, + ) -> Array: + # Get the corresponding array>> for a nested array + + # Recursively pad any inner nested arrays + padded_element_name: s_name.Name + if isinstance(element_type, Array): + element_type = Array.get_padded_collection( + schema, + element_type=element_type.get_element_type(schema), + ) + padded_element_name = Tuple.generate_name( + {'f1': element_type.get_name(schema)}, + named=True, + ) + elif isinstance(element_type, Tuple): + element_type = Tuple.get_padded_collection( + schema, + element_types=dict( + element_type.get_element_types(schema).items(schema) + ), + named=element_type.get_named(schema), + ) + padded_element_name = element_type.get_name(schema) + else: + padded_element_name = element_type.get_name(schema) + + padded_collection_name = Array.generate_name( + padded_element_name + ) + return schema.get_global( + Array, + padded_collection_name, + ) + def get_generated_name(self, schema: s_schema.Schema) -> s_name.UnqualName: return type(self).generate_name( self.get_element_type(schema).get_name(schema), @@ -1491,12 +1638,53 @@ def create_shell( st = next(iter(subtypes)) + # If this type contains an array of array, recursively include the + # padded array>> in the shell. + padded_collection_type = None + if isinstance(st, ArrayTypeShell): + padded_subtype = ( + st + if st.padded_collection_type is None else + st.padded_collection_type + ) + wrapped_element_name = Tuple.generate_name( + {'f1': padded_subtype.name}, + named=True, + ) + wrapped_element_type = Tuple.create_shell( + schema, + subtypes={'f1': padded_subtype}, + typemods={'named': True}, + name=wrapped_element_name, + ) + padded_collection_name = Array.generate_name( + wrapped_element_name + ) + padded_collection_type = Array.create_shell( + schema, + subtypes=[wrapped_element_type], + name=padded_collection_name + ) + elif ( + isinstance(st, TupleTypeShell) + and st.padded_collection_type is not None + ): + padded_collection_name = Array.generate_name( + st.padded_collection_type.name + ) + padded_collection_type = Array.create_shell( + schema, + subtypes=[st.padded_collection_type], + name=padded_collection_name, + ) + return ArrayTypeShell( subtype=st, typemods=typemods, name=name, expr=expr, schemaclass=cls, + padded_collection_type=padded_collection_type, ) def as_shell( @@ -1543,6 +1731,7 @@ def __init__( subtype: TypeShell[Type], typemods: tuple[typing.Any, ...], schemaclass: type[Array_T_co], + padded_collection_type: Optional[TypeShell[Type]] = None, ) -> None: if name is None: name = schemaclass.generate_name(subtype.name) @@ -1550,6 +1739,7 @@ def __init__( super().__init__(name=name, schemaclass=schemaclass, expr=expr) self.subtype = subtype self.typemods = typemods + self.padded_collection_type = padded_collection_type def get_subtypes( self, @@ -1583,6 +1773,9 @@ def as_create_delta( if isinstance(el, CollectionTypeShell): cmd.add(el.as_create_delta(schema)) + if self.padded_collection_type: + cmd.add_prerequisite(self.padded_collection_type.as_create_delta(schema)) + ca.set_attribute_value('name', ca.classname) ca.set_attribute_value('element_type', el) ca.set_attribute_value('is_persistent', True) @@ -1678,6 +1871,18 @@ def create( result = schema.get_global(cls, name, default=None) if result is None: + if any( + st.contains_array_of_array(schema) + for st in element_types.values() + ): + # Nested arrays are represented as array> + # and so we need to ensure these types also exist. + schema, _ = Tuple.create_padded_collection( + schema, + element_types=element_types, + named=named, + ) + schema, result = super().create_in_schema( schema, id=id, @@ -1691,6 +1896,87 @@ def create( return schema, result + @classmethod + def create_padded_collection( + cls, + schema: s_schema.Schema, + *, + element_types: Mapping[str, Type], + named: bool = False, + ) -> tuple[s_schema.Schema, Collection]: + # If a tuple contains array>, create the coresponding + # tuple which contains array>> instead. + # + # eg. tuple>> + # -> tuple>>> + + # Recursively pad any inner nested arrays + padded_element_types = {} + for name, element_type in element_types.items(): + if isinstance(element_type, Array): + schema, element_type = Array.create_padded_collection( + schema, + element_type=element_type.get_element_type(schema), + ) + elif isinstance(element_type, Tuple): + schema, element_type = Tuple.create_padded_collection( + schema, + element_types=dict( + element_type.get_element_types(schema).items(schema) + ), + named=element_type.get_named(schema), + ) + + padded_element_types[name] = element_type + + return Tuple.create( + schema, + element_types=padded_element_types, + named=named, + ) + + @classmethod + def get_padded_collection( + cls, + schema: s_schema.Schema, + *, + element_types: Mapping[str, Type], + named: bool = False, + ) -> Tuple: + # If a tuple contains array>, get the coresponding + # tuple which contains array>> instead. + # + # eg. tuple>> + # -> tuple>>> + + # Recursively pad any inner nested arrays + padded_element_types = {} + for name, element_type in element_types.items(): + if isinstance(element_type, Array): + element_type = Array.get_padded_collection( + schema, + element_type=element_type.get_element_type(schema), + ) + elif isinstance(element_type, Tuple): + element_type = Tuple.get_padded_collection( + schema, + element_types=dict( + element_type.get_element_types(schema).items(schema) + ), + named=element_type.get_named(schema), + ) + + padded_element_types[name] = element_type.get_name(schema) + + padded_collection_name = Tuple.generate_name( + padded_element_types, + named=named, + ) + return schema.get_global( + Tuple, + padded_collection_name, + ) + def get_generated_name(self, schema: s_schema.Schema) -> s_name.UnqualName: els = {n: st.get_name(schema) for n, st in self.iter_subtypes(schema)} return type(self).generate_name(els, self.is_named(schema)) @@ -1820,11 +2106,50 @@ def create_shell( typemods: Any = None, name: Optional[s_name.Name] = None, ) -> TupleTypeShell[Tuple_T]: + + # If this type contains an array of array, recursively include the + # padded array>> in the shell. + padded_collection_type = None + if any( + ( + isinstance(st, (ArrayTypeShell, TupleTypeShell)) + and st.padded_collection_type is not None + ) + for _, st in subtypes.items() + ): + named = typemods is not None and typemods.get('named', False) + padded_subtypes = { + n: ( + st.padded_collection_type + if ( + isinstance(st, (ArrayTypeShell, TupleTypeShell)) + and st.padded_collection_type is not None + ) else + st + ) + for n, st in subtypes.items() + } + padded_subtype_names = { + n: st.name + for n, st in padded_subtypes.items() + } + padded_collection_name = Tuple.generate_name( + element_names=padded_subtype_names, + named=named, + ) + padded_collection_type = Tuple.create_shell( + schema, + subtypes=padded_subtypes, + typemods=typemods, + name=padded_collection_name, + ) + return TupleTypeShell( subtypes=subtypes, typemods=typemods, name=name, schemaclass=cls, + padded_collection_type=padded_collection_type, ) def as_shell( @@ -2080,6 +2405,7 @@ def __init__( subtypes: Mapping[str, TypeShell[Type]], typemods: Any = None, schemaclass: type[Tuple_T_co], + padded_collection_type: Optional[TypeShell[Type]] = None, ) -> None: if name is None: named = typemods is not None and typemods.get('named', False) @@ -2091,6 +2417,7 @@ def __init__( super().__init__(name=name, schemaclass=schemaclass) self.subtypes = subtypes self.typemods = typemods + self.padded_collection_type = padded_collection_type def get_displayname(self, schema: s_schema.Schema) -> str: st_names = ', '.join(st.get_displayname(schema) @@ -2135,6 +2462,9 @@ def as_create_delta( if view_name is not None: ct.add_prerequisite(plain_tuple) + if self.padded_collection_type: + ct.add_prerequisite(self.padded_collection_type.as_create_delta(schema)) + return ct def _as_plain_create_delta( diff --git a/edb/schema/utils.py b/edb/schema/utils.py index c59ad3302f8..d246900bdde 100644 --- a/edb/schema/utils.py +++ b/edb/schema/utils.py @@ -336,12 +336,6 @@ def ast_to_type_shell( span=node.span, ) - if isinstance(subtypes_list[0], s_types.ArrayTypeShell): - raise errors.UnsupportedFeatureError( - 'nested arrays are not supported', - span=node.subtypes[0].span, - ) - try: return coll.create_shell( # type: ignore schema, diff --git a/edb/server/compiler/sertypes.py b/edb/server/compiler/sertypes.py index 81975f644c5..9fec94e16bb 100644 --- a/edb/server/compiler/sertypes.py +++ b/edb/server/compiler/sertypes.py @@ -1654,7 +1654,28 @@ def _parse_multirange_descriptor( def _make_global_rep(typ: s_types.Type, ctx: Context) -> object: - if isinstance(typ, s_types.Tuple): + if typ.contains_array_of_array(ctx.schema): + # Nested array wraps inner array in a tuple + # Replace array> with array>> + if isinstance(typ, s_types.Array): + padded_collection_type = s_types.Array.get_padded_collection( + ctx.schema, + element_type=typ.get_element_type(ctx.schema), + ) + elif isinstance(type, s_types.Tuple): + padded_collection_type = s_types.Tuple.get_padded_collection( + ctx.schema, + element_types={ + n: st + for n, st in typ.get_element_types(ctx.schema) + }, + named=typ.get_named(ctx.schema), + ) + else: + raise NotImplementedError + + return _make_global_rep(padded_collection_type, ctx) + elif isinstance(typ, s_types.Tuple): subtyps = typ.get_subtypes(ctx.schema) return ( int(enums.TypeTag.TUPLE), diff --git a/edb/server/protocol/args_ser.pyx b/edb/server/protocol/args_ser.pyx index c42c76934d4..16e4d3110ab 100644 --- a/edb/server/protocol/args_ser.pyx +++ b/edb/server/protocol/args_ser.pyx @@ -406,6 +406,10 @@ cdef recode_array( data = frb_read(&sub_buf, in_len) out_buf.write_cstr(data, in_len) else: + # Nested arrays (eg. array>) are handled + # as array>>. However, this doesn't need to be + # handled here since this is already incorporated in the + # tuple_info during _make_global_rep _recode_global(dbv, &sub_buf, out_buf, in_len, tuple_info) if frb_get_len(&sub_buf): raise errors.InputDataError('unexpected trailing data in buffer') diff --git a/tests/test_edgeql_ddl.py b/tests/test_edgeql_ddl.py index b32b7062849..79d4687341b 100644 --- a/tests/test_edgeql_ddl.py +++ b/tests/test_edgeql_ddl.py @@ -3806,16 +3806,6 @@ async def test_edgeql_ddl_bad_03(self): }; """) - async def test_edgeql_ddl_bad_04(self): - with self.assertRaisesRegex( - edgedb.UnsupportedFeatureError, - r'nested arrays are not supported'): - await self.con.execute(r""" - CREATE TYPE Foo { - CREATE PROPERTY bar -> array>; - }; - """) - async def test_edgeql_ddl_bad_05(self): with self.assertRaisesRegex( edgedb.EdgeQLSyntaxError, diff --git a/tests/test_edgeql_globals.py b/tests/test_edgeql_globals.py index 19d2e3c336c..bbd6785b48a 100644 --- a/tests/test_edgeql_globals.py +++ b/tests/test_edgeql_globals.py @@ -74,6 +74,13 @@ class TestEdgeQLGlobals(tb.QueryTestCase): create global arrayTuple2 -> array>>; create global arrayTuple3 -> array, array>>; + create global compositeAA -> array>; + + create global computedArrayArray := [[1, 2], [3, 4]]; + create global computedArrayArrayTuple := [ + [(1, 'A'), (2, 'B')], + [(3, 'C'), (4, 'D')], + ]; ''' ] @@ -587,7 +594,7 @@ async def test_edgeql_globals_state_cardinality(self): await self.con.execute("select global cur_user") self.con._protocol.last_state = None - async def test_edgeql_globals_composite(self): + async def test_edgeql_globals_composite_01(self): # Test various composite global variables. # HACK: Using with_globals on testbase.Connection doesn't @@ -606,6 +613,14 @@ async def test_edgeql_globals_composite(self): arrayTuple=[(10, 20), (30, 40)], arrayTuple2=[('a', [1]), ('b', [3, 4])], arrayTuple3=[(('a', True), [1]), (('b', False), [3, 4])], + compositeAA=[[1, 2], [3, 4]], + # compositeAAA=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]], + # compositeAAT=[[(1, 'A'), (2, 'B')], [(3, 'C'), (4, 'D')]], + # compositeTAA=([[1, 2], [3, 4]], [['A', 'B'], ['C', 'D']]), + # compositeTTAA=( + # ([[1, 2], [3, 4]], [['A', 'B'], ['C', 'D']]), + # ([[True, False], [False, True]]), + # ), ) scon = con.with_globals(**globs) chunks = [f'{name} := global {name}' for name in globs] @@ -615,6 +630,16 @@ async def test_edgeql_globals_composite(self): finally: await con.aclose() + async def test_edgeql_globals_composite_02(self): + await self.assert_query_result( + r'''select global computedArrayArray''', + [[[1, 2], [3, 4]]] + ) + await self.assert_query_result( + r'''select global computedArrayArrayTuple''', + [[[(1, 'A'), (2, 'B')], [(3, 'C'), (4, 'D')]]] + ) + async def test_edgeql_globals_schema_types_01(self): # Non-computed globals don't add a schema type await self.con.execute(''' diff --git a/tests/test_schema.py b/tests/test_schema.py index 95cc55729b5..ce2a0f152b1 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -3734,18 +3734,14 @@ def test_schema_with_module_06(self): "abs' does not exist", ) - @tb.must_fail( - errors.UnsupportedFeatureError, - "nested arrays are not supported", - ) def test_schema_array_of_array_01(self): """ global foo: array>; """ @tb.must_fail( - errors.UnsupportedFeatureError, - "nested arrays are not supported", + errors.SchemaDefinitionError, + "non-computed globals may not have have object type", ) def test_schema_array_of_array_02(self): """ @@ -3764,8 +3760,8 @@ def test_schema_array_of_array_04(self): """ @tb.must_fail( - errors.UnsupportedFeatureError, - "nested arrays are not supported", + errors.InvalidPropertyTargetError, + "invalid property type: non-computed nested arrays are not supported", ) def test_schema_array_of_array_05(self): """ @@ -3775,8 +3771,9 @@ def test_schema_array_of_array_05(self): """ @tb.must_fail( - errors.UnsupportedFeatureError, - "nested arrays are not supported", + errors.InvalidPropertyTargetError, + "invalid property type: expected a scalar type, " + "or a scalar collection, got collection 'array>'", ) def test_schema_array_of_array_06(self): """ @@ -3792,19 +3789,11 @@ def test_schema_array_of_array_07(self): }; """ - @tb.must_fail( - errors.UnsupportedFeatureError, - "nested arrays are not supported", - ) def test_schema_array_of_array_08(self): """ function foo(x: array>) -> int64 using (1); """ - @tb.must_fail( - errors.UnsupportedFeatureError, - "nested arrays are not supported", - ) def test_schema_array_of_array_09(self): """ type Foo;