Merge branch 'main' of https://github.com/microsoft/Power-Fx into yucl/udf_dataSource_delegatable

Author: yuchenglin

src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs

@@ -108,31 +108,36 @@ private async Task WriteObjectAsync(string objectName, ISwaggerSchema schema, IE
                     await WritePropertyAsync(
                         nv.Name,
                         new SwaggerSchema(
-                            type: nv.Value.Type._type.Kind switch
-                            {
-                                DKind.Number => "number",
-                                DKind.Decimal => "number",
-                                DKind.String or
-                                DKind.Date or
-                                DKind.DateTime or
-                                DKind.DateTimeNoTimeZone => "string",
-                                DKind.Boolean => "boolean",
-                                DKind.Record => "object",
-                                DKind.Table => "array",
-                                DKind.ObjNull => "null",
-                                _ => $"type: unknown_dkind {nv.Value.Type._type.Kind}"
-                            },
-                            format: GetDateFormat(nv.Value.Type._type.Kind)),
+                            type: GetType(nv.Value.Type),
+                            format: GetFormat(nv.Value.Type)),
                         nv.Value).ConfigureAwait(false);
                 }
             }
 
             EndObject(objectName);
         }
 
-        private static string GetDateFormat(DKind kind)
+        internal static string GetType(FormulaType type)
         {
-            return kind switch
+            return type._type.Kind switch
+            {
+                DKind.Number => "number",
+                DKind.Decimal => "number",
+                DKind.String or
+                DKind.Date or
+                DKind.DateTime or
+                DKind.DateTimeNoTimeZone => "string",
+                DKind.Boolean => "boolean",
+                DKind.Record => "object",
+                DKind.Table => "array",
+                DKind.ObjNull => "null",
+                _ => $"type: unknown_dkind {type._type.Kind}"
+            };
+        }
+
+        internal static string GetFormat(FormulaType type)
+        {
+            return type._type.Kind switch
             {
                 DKind.Date => "date",
                 DKind.DateTime => "date-time",

src/libraries/Microsoft.PowerFx.Connectors/Execution/FormulaValueSerializer.cs

@@ -57,15 +57,28 @@ public async Task<HttpRequestMessage> BuildRequest(FormulaValue[] args, IConvert
             // Header names are not case sensitive.
             // From RFC 2616 - "Hypertext Transfer Protocol -- HTTP/1.1", Section 4.2, "Message Headers"
             var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-            Dictionary<string, (ISwaggerSchema, FormulaValue)> bodyParts = new ();
+            Dictionary<string, (ISwaggerSchema, FormulaValue)> bodyParts = new ();            
             Dictionary<string, FormulaValue> incomingParameters = ConvertToNamedParameters(args);
             string contentType = null;
 
             foreach (KeyValuePair<ConnectorParameter, FormulaValue> param in _function._internals.OpenApiBodyParameters)
-            {
+            {                
                 if (incomingParameters.TryGetValue(param.Key.Name, out var paramValue))
                 {
-                    bodyParts.Add(param.Key.Name, (param.Key.Schema, paramValue));
+                    if (_function._internals.SpecialBodyHandling && paramValue is RecordValue rv)
+                    {
+                        foreach (NamedValue field in rv.Fields)
+                        {
+                            string type = FormulaValueSerializer.GetType(field.Value.Type);
+                            string format = FormulaValueSerializer.GetFormat(field.Value.Type);
+
+                            bodyParts.Add(field.Name, (new SwaggerSchema(type, format), field.Value));
+                        }
+                    }
+                    else
+                    {
+                        bodyParts.Add(param.Key.Name, (param.Key.Schema, paramValue));
+                    }
                 }
                 else if (param.Key.Schema.Default != null && param.Value != null)
                 {
@@ -200,6 +213,7 @@ public Dictionary<string, FormulaValue> ConvertToNamedParameters(FormulaValue[]
             // Parameter names are case sensitive.
 
             Dictionary<string, FormulaValue> map = new ();
+            bool specialBodyHandling = _function._internals.SpecialBodyHandling;
 
             // Seed with default values. This will get overwritten if provided.
             foreach (KeyValuePair<string, (bool required, FormulaValue fValue, DType dType)> kv in _function._internals.ParameterDefaultValues)
@@ -217,9 +231,9 @@ public Dictionary<string, FormulaValue> ConvertToNamedParameters(FormulaValue[]
             {
                 string parameterName = _function.RequiredParameters[i].Name;
                 FormulaValue paramValue = args[i];
-
+                
                 // Objects are always flattenned
-                if (paramValue is RecordValue record && !_function.RequiredParameters[i].IsBodyParameter)
+                if (paramValue is RecordValue record && (specialBodyHandling || !_function.RequiredParameters[i].IsBodyParameter))
                 {
                     foreach (NamedValue field in record.Fields)
                     {

src/libraries/Microsoft.PowerFx.Connectors/Execution/HttpFunctionInvoker.cs

@@ -20,5 +20,7 @@ internal class ConnectorParameterInternals
         internal string BodySchemaReferenceId { get; init; }
 
         internal Dictionary<string, (bool, FormulaValue, DType)> ParameterDefaultValues { get; init; }
+
+        internal bool SpecialBodyHandling { get; init; }
     }
 }

src/libraries/Microsoft.PowerFx.Connectors/Internal/ConnectorParameterInternals.cs

@@ -44,6 +44,11 @@ internal CdpTableValue(IRContext irContext)
 
         public override IEnumerable<DValue<RecordValue>> Rows => GetRowsAsync(null, null, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult();
 
+        public DelegationParameterFeatures SupportedFeatures => DelegationParameterFeatures.Filter |
+                DelegationParameterFeatures.Top |
+                DelegationParameterFeatures.Columns | // $select
+                DelegationParameterFeatures.Sort; // $orderby
+
         public async Task<IReadOnlyCollection<DValue<RecordValue>>> GetRowsAsync(IServiceProvider services, DelegationParameters parameters, CancellationToken cancel)
         {
             if (parameters == null && _cachedRows != null)
@@ -66,6 +71,11 @@ public void Refresh()
         {
             _cachedRows = null;
         }
+
+        public Task<FormulaValue> ExecuteQueryAsync(IServiceProvider services, DelegationParameters parameters, CancellationToken cancel)
+        {
+            throw new NotImplementedException();
+        }
     }   
 
     internal static class ODataParametersExtensions

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpTableValue.cs

@@ -92,16 +92,17 @@ public bool ExposeInternalParamsWithoutDefaultValue
         /// This flag will force all enums to be returns as FormulaType.String or FormulaType.Decimal regardless of x-ms-enum-*.
         /// This flag is only in effect when SupportXMsEnumValues is true.
         /// </summary>
-        public bool ReturnEnumsAsPrimitive { get; init; } = false;
-
+        public bool ReturnEnumsAsPrimitive { get; init; } = false;
+
         /// <summary>
-        /// In Power Apps, when a body parameter is used it's flattened and we create one parameter for each
-        /// body object property. With that logic each parameter name will be the object property name.
-        /// When set, this setting will use the real body name specified in the swagger instead of the property name
-        /// of the object, provided there is only one property.
+        /// This flag enables some special handling for the body parameter, when
+        /// - body name is 'item'
+        /// - body inner object is 'dynamicProperties'
+        /// - there is only one property in inner object
+        /// In that base the body will be fully flattened and we will retain the 'body' name for the parameter.
         /// </summary>
-        public bool UseDefaultBodyNameForSinglePropertyObject { get; init; } = false;
-
+        public bool UseItemDynamicPropertiesSpecialHandling { get; init; } = false;
+        
         public ConnectorCompatibility Compatibility { get; init; } = ConnectorCompatibility.Default;
     }
 

src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSettings.cs

@@ -12,13 +12,14 @@
 using Microsoft.PowerFx.Core.Localization;
 using Microsoft.PowerFx.Core.Types;
 using Microsoft.PowerFx.Core.Utils;
+using Microsoft.PowerFx.Functions;
 using Microsoft.PowerFx.Intellisense;
 using Microsoft.PowerFx.Types;
 using static Microsoft.PowerFx.Connectors.ConnectorHelperFunctions;
 
 namespace Microsoft.PowerFx.Connectors
 {
-    internal class ConnectorTexlFunction : TexlFunction, IAsyncConnectorTexlFunction, IHasUnsupportedFunctions
+    internal class ConnectorTexlFunction : TexlFunction, IFunctionInvoker, IHasUnsupportedFunctions
     {
         public ConnectorFunction ConnectorFunction { get; }
 
@@ -85,8 +86,11 @@ public override bool TryGetParamDescription(string paramName, out string paramDe
 
         public override bool HasSuggestionsForParam(int argumentIndex) => argumentIndex <= MaxArity;
 
-        public async Task<FormulaValue> InvokeAsync(FormulaValue[] args, IServiceProvider serviceProvider, CancellationToken cancellationToken)
+        public async Task<FormulaValue> InvokeAsync(FunctionInvokeInfo invokeInfo, CancellationToken cancellationToken)
         {
+            FormulaValue[] args = invokeInfo.Args.ToArray(); // https://github.com/microsoft/Power-Fx/issues/2817
+            var serviceProvider = invokeInfo.FunctionServices;
+
             cancellationToken.ThrowIfCancellationRequested();
             BaseRuntimeConnectorContext runtimeContext = serviceProvider.GetService(typeof(BaseRuntimeConnectorContext)) as BaseRuntimeConnectorContext ?? throw new InvalidOperationException("RuntimeConnectorContext is missing from service provider");
 

src/libraries/Microsoft.PowerFx.Connectors/Texl/ConnectorTexlFunction.cs

@@ -26,6 +26,9 @@ public abstract class TableDelegationInfo
         [Obsolete("preview")]
         public SummarizeCapabilities SummarizeCapabilities { get; init; }
 
+        [Obsolete("preview")]
+        public CountCapabilities CountCapabilities { get; init; }
+
         // Defines ungroupable columns
         public GroupRestrictions GroupRestriction { get; init; }
 
@@ -299,6 +302,54 @@ public enum SummarizeMethod
         CountRows
     }
 
+    [Obsolete("preview")]
+    public class CountCapabilities
+    {
+        public CountCapabilities()
+        {
+        }
+
+        /// <summary>
+        /// If the table is countable, return true. 
+        /// Relevant expression: CountRows(Table).
+        /// </summary>
+        /// <returns></returns>
+        public virtual bool IsCountableTable()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// If the table is countable after filter, return true.
+        /// Relevant expression: CountRows(Filter(Table, Condition)); / CountIf(Table, Condition).
+        /// </summary>
+        /// <returns></returns>
+        public virtual bool IsCountableAfterFilter()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// If the table is countable after join, return true.
+        /// Relevant expression: CountRows(Join(Table1, Table2, ...)).
+        /// </summary>
+        /// <returns></returns>
+        public virtual bool IsCountableAfterJoin()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// If the table is countable after summarize, return true.
+        /// Relevant expression: CountRows(Summarize(Table, ...)).
+        /// </summary>
+        /// <returns></returns>
+        public virtual bool IsCountableAfterSummarize()
+        {
+            return false;
+        }
+    }
+
     public sealed class FilterRestrictions
     {
         // List of required properties

src/libraries/Microsoft.PowerFx.Core/Entities/External/TableDelegationInfo.cs

@@ -8,6 +8,7 @@
 namespace Microsoft.PowerFx.Core.Functions
 {
     // A Texl function capable of async invokes. 
+    // This only lives in Core to enable Fx.Json funcs impl (which doesn't depend on interpreter).    
     internal interface IAsyncTexlFunction
     {
         Task<FormulaValue> InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken);

src/libraries/Microsoft.PowerFx.Core/Functions/IAsyncTexlFunction.cs

@@ -9,6 +9,7 @@
 namespace Microsoft.PowerFx.Core.Functions
 {
     // A Texl function capable of async invokes, using TimeZoneInfo and IRContext. 
+    // Remove this: https://github.com/microsoft/Power-Fx/issues/2818
     internal interface IAsyncTexlFunction4
     {
         Task<FormulaValue> InvokeAsync(TimeZoneInfo timezoneInfo, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken);

src/libraries/Microsoft.PowerFx.Core/Functions/IAsyncTexlFunction3.cs

@@ -7,8 +7,10 @@
 using Microsoft.PowerFx.Types;
 
 namespace Microsoft.PowerFx.Core.Functions
-{    
+{
     // Texl function interface with IServiceProvider
+    // Only product impl is JsonFunctionImpl.
+    // Remove this: https://github.com/microsoft/Power-Fx/issues/2818
     internal interface IAsyncTexlFunction5
     {
         Task<FormulaValue> InvokeAsync(IServiceProvider runtimeServiceProvider, FormulaType irContext, FormulaValue[] args, CancellationToken cancellationToken);        

src/libraries/Microsoft.PowerFx.Core/Functions/IAsyncTexlFunction4.cs

@@ -692,6 +692,7 @@ internal static class TexlStrings
         public static ErrorResourceKey ErrCannotCoerce_SourceType_TargetType = new ErrorResourceKey("ErrCannotCoerce_SourceType_TargetType");
         public static ErrorResourceKey ErrNumberOrStringExpected = new ErrorResourceKey("ErrNumberOrStringExpected");
         public static ErrorResourceKey ErrClosingBracketExpected = new ErrorResourceKey("ErrClosingBracketExpected");
+        public static ErrorResourceKey ErrClosingIdentifierExpected = new ErrorResourceKey("ErrClosingIdentifierExpected");
         public static ErrorResourceKey ErrEmptyInvalidIdentifier = new ErrorResourceKey("ErrEmptyInvalidIdentifier");
         public static ErrorResourceKey ErrIncompatibleTypes = new ErrorResourceKey("ErrIncompatibleTypes");
         public static ErrorResourceKey ErrIncompatibleTypesForEquality_Left_Right = new ErrorResourceKey("ErrIncompatibleTypesForEquality_Left_Right");

src/libraries/Microsoft.PowerFx.Core/Functions/IAsyncTexlFunction5.cs

@@ -1429,13 +1429,13 @@ private Identifier ParseIdentifier(Token at = null)
             if (_curs.TidCur == TokKind.Ident)
             {
                 tok = _curs.TokMove().As<IdentToken>();
-                if (tok.HasDelimiterStart && !tok.HasDelimiterEnd)
+                if (tok.IsModified)
                 {
-                    PostError(tok, TexlStrings.ErrClosingBracketExpected);
+                    PostError(tok, TexlStrings.ErrEmptyInvalidIdentifier);
                 }
-                else if (tok.IsModified)
+                else if (tok.HasDelimiterStart && !tok.HasDelimiterEnd)
                 {
-                    PostError(tok, TexlStrings.ErrEmptyInvalidIdentifier);
+                    PostError(tok, TexlStrings.ErrClosingIdentifierExpected);
                 }
             }
             else

src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs

@@ -52,6 +52,7 @@ public SymbolTable SymbolTable
             set => _symbolTable = value;
         }
 
+        // Remove this: https://github.com/microsoft/Power-Fx/issues/2821
         internal readonly Dictionary<TexlFunction, IAsyncTexlFunction> AdditionalFunctions = new ();
 
         [Obsolete("Use Config.EnumStore or symboltable directly")]

src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs

@@ -76,15 +76,15 @@ public enum DelegationParameterFeatures
         // $apply = groupby((field1, ..), field with sum as TotalSum)
         ApplyGroupBy = 1 << 5,
 
+        // $count
+        Count = 1 << 6,
+
         /*
           To be implemented later when needed
          
         // $compute
         Compute = 1 << 5,
 
-        // $count
-        Count = 1 << 6,
-
         // $expand
         Expand = 1 << 7,
 

src/libraries/Microsoft.PowerFx.Core/Public/Config/PowerFxConfig.cs

@@ -27,7 +27,21 @@ public interface IDelegatableTableValue
         /// <param name="parameters">delegation parameters.</param>
         /// <param name="cancel"></param>
         /// <returns></returns>
+        [Obsolete($"Use  {nameof(ExecuteQueryAsync)} instead.")]
         Task<IReadOnlyCollection<DValue<RecordValue>>> GetRowsAsync(IServiceProvider services, DelegationParameters parameters, CancellationToken cancel);
+
+        /// <summary>
+        /// Evaluation will invoke this method on aggregation calls where return value is scaler.
+        /// </summary>
+        /// <param name="services">Pre-eval services.</param>
+        /// <param name="parameters">Delegation parameters.</param>
+        /// <param name="cancel"></param>
+        Task<FormulaValue> ExecuteQueryAsync(IServiceProvider services, DelegationParameters parameters, CancellationToken cancel);
+
+        /// <summary>
+        /// Supported features for delegation to verify correct Features are delegated at runtime.
+        /// </summary>
+        DelegationParameterFeatures SupportedFeatures { get; }
     }
 
     /// <summary>