tuplex · aletheia094 · Jun 2, 2022 · Jun 2, 2022 · Jun 14, 2022 · Jun 30, 2022
diff --git a/tuplex/codegen/include/ASTAnnotation.h b/tuplex/codegen/include/ASTAnnotation.h
@@ -48,8 +48,13 @@ class Symbol : public std::enable_shared_from_this<Symbol> {
     std::shared_ptr<Symbol> parent;
 
     ///! an optional abstract typer function which can be applied if the symboltype is function
+    ///! to deliver a concretely typed type based on the paramter type
     std::function<python::Type(const python::Type&)> functionTyper;
 
+    ///! an optional abstract typer that takes the original type of the caller (e.g., for an attribute)
+    ///! and provides then together with the parameterType symilar to functionTyper a concrete type for the attribute function
+    std::function<python::Type(const python::Type&,const python::Type&)> attributeFunctionTyper;
+
     ///! optionally constant data associated with that symbol
     tuplex::Field constantData;
 
@@ -90,55 +95,43 @@ class Symbol : public std::enable_shared_from_this<Symbol> {
         auto generic_result = functionTyper(parameterType);
         if(generic_result != python::Type::UNKNOWN) {
             specializedFunctionType = generic_result;
-
             assertFunctionDoesNotReturnGeneric(specializedFunctionType);
             return true;
         }
 
-        for(auto& type : types) {
-            // found symbol, now check its type
-            if(!type.isFunctionType())
-                continue;
-
-            auto tupleArgType = getTupleArg(type.getParamsType());
+        // typer did not yield a result, hence try stored funciton types incl. upcasting
+        return findStoredTypedFunction(parameterType, specializedFunctionType);
+    }
 
-            // check if there's a direct type match => use that function then!
-            if(parameterType == tupleArgType) {
-                specializedFunctionType = type;
-                assertFunctionDoesNotReturnGeneric(specializedFunctionType);
-                return true;
-            }
+    /*!
+     * the typing of an attribute which is a function may be based on both the callerType and the parameters. I.e.,
+     * this function helps to type an attribute x.a(p) where callerType = type(x) and parameterType = type(p) for some
+     * symbol a which is this.
+     * @param callerType
+     * @param parameterType
+     * @param specializedFunctionType where to store the concrete (non-generic!) output type!
+     * @return true if a specialized function type could be generated, false else.
+     */
+    inline bool findAttributeFunctionType(const python::Type& callerType,
+                                          const python::Type& parameterType,
+                                          python::Type& specializedFunctionType) {
+        // fallback based typing:
+        // 1. check attribute typer
+        // 2. check general typer
+        auto typed_result = attributeFunctionTyper(callerType, parameterType);
+        if(python::Type::UNKNOWN == typed_result) {
+            typed_result = functionTyper(parameterType);
         }
 
-        // no direct match was found. Check whether casting would work or partial matching.
-        for(auto& type : types) {
-            // found symbol, now check its type
-            if (!type.isFunctionType())
-                continue;
-
-            auto tupleArgType = getTupleArg(type.getParamsType());
-
-            // check if given parameters type is compatible with function type?
-            // actual invocation is with parameterType
-            // ==> can we upcast them to fit the defined one OR does is partially work?
-            // e.g., when the function is defined for NULL, but we have opt?
-            if (isTypeCompatible(parameterType, tupleArgType)) {
-                specializedFunctionType = type;
-
-                // specialize according to parameterType if it's a generic function so further typing works
-                assert(!specializedFunctionType.getReturnType().isGeneric());
-                if(specializedFunctionType.getParamsType().isGeneric()) {
-                    auto specializedParams = python::specializeGenerics(parameterType, tupleArgType);
-                    specializedFunctionType = python::Type::makeFunctionType(specializedParams,
-                                                                             specializedFunctionType.getReturnType());
-                }
-
-                assertFunctionDoesNotReturnGeneric(specializedFunctionType);
-                return true;
-            }
+        // check if result is valid, then take it
+        if(typed_result != python::Type::UNKNOWN) {
+            specializedFunctionType = typed_result;
+            assertFunctionDoesNotReturnGeneric(specializedFunctionType);
+            return true;
         }
 
-        return false;
+        // typer did not yield a result, hence try stored funciton types incl. upcasting
+        return findStoredTypedFunction(parameterType, specializedFunctionType);
     }
 
     /*!
@@ -169,7 +162,8 @@ class Symbol : public std::enable_shared_from_this<Symbol> {
         return full_name;
     }
 
-    Symbol() {}
+    Symbol() : functionTyper([](const python::Type&){return python::Type::UNKNOWN;}),
+               attributeFunctionTyper([](const python::Type&, const python::Type&){return python::Type::UNKNOWN;}) {}
     virtual ~Symbol()  {
         _attributes.clear();
         parent.reset();
@@ -216,24 +210,76 @@ class Symbol : public std::enable_shared_from_this<Symbol> {
 
     Symbol(std::string _name,
            std::function<python::Type(const python::Type&)> typer) : name(_name), qualifiedName(_name),
-           functionTyper(std::move(typer)), symbolType(SymbolType::FUNCTION) {}
+           functionTyper(std::move(typer)), attributeFunctionTyper([](const python::Type&, const python::Type&){return python::Type::UNKNOWN;}), symbolType(SymbolType::FUNCTION) {}
 
     Symbol(std::string _name, python::Type _type) : name(_name), qualifiedName(_name),
     types{_type},
-    symbolType(_type.isFunctionType() ? SymbolType::FUNCTION : SymbolType::VARIABLE), functionTyper([](const python::Type&) { return python::Type::UNKNOWN; }) {}
+    symbolType(_type.isFunctionType() ? SymbolType::FUNCTION : SymbolType::VARIABLE),
+    functionTyper([](const python::Type&) { return python::Type::UNKNOWN; }),
+    attributeFunctionTyper([](const python::Type&, const python::Type&){return python::Type::UNKNOWN;}) {}
 
     Symbol(std::string _name, std::string _qualifiedName, python::Type _type, SymbolType _symbolType) : name(_name),
     qualifiedName(_qualifiedName),
     types{_type},
     symbolType(_symbolType),
-    functionTyper([](const python::Type&) { return python::Type::UNKNOWN; }) {}
+    functionTyper([](const python::Type&) { return python::Type::UNKNOWN; }),
+    attributeFunctionTyper([](const python::Type&, const python::Type&){return python::Type::UNKNOWN;}) {}
 
 private:
     ///! i.e. to store something like re.search. re is then of module type. search will have a concrete function type.
     std::vector<std::shared_ptr<Symbol>> _attributes;
 
     /********* HELPER FUNCTIONS *************/
 
+    inline bool findStoredTypedFunction(const python::Type& parameterType, python::Type& specializedFunctionType) {
+
+        // typing using typer functions above failed, hence now search for concrete stored types.
+        for(auto& type : types) {
+            // found symbol, now check its type
+            if(!type.isFunctionType())
+                continue;
+
+            auto tupleArgType = getTupleArg(type.getParamsType());
+
+            // check if there's a direct type match => use that function then!
+            if(parameterType == tupleArgType) {
+                specializedFunctionType = type;
+                assertFunctionDoesNotReturnGeneric(specializedFunctionType);
+                return true;
+            }
+        }
+
+        // no direct match was found. Check whether casting would work or partial matching.
+        for(auto& type : types) {
+            // found symbol, now check its type
+            if (!type.isFunctionType())
+                continue;
+
+            auto tupleArgType = getTupleArg(type.getParamsType());
+
+            // check if given parameters type is compatible with function type?
+            // actual invocation is with parameterType
+            // ==> can one upcast them to fit the defined one OR does is partially work?
+            // e.g., when the function is defined for NULL, but we have opt?
+            if (isTypeCompatible(parameterType, tupleArgType)) {
+                specializedFunctionType = type;
+
+                // specialize according to parameterType if it's a generic function so further typing works
+                assert(!specializedFunctionType.getReturnType().isGeneric());
+                if(specializedFunctionType.getParamsType().isGeneric()) {
+                    auto specializedParams = python::specializeGenerics(parameterType, tupleArgType);
+                    specializedFunctionType = python::Type::makeFunctionType(specializedParams,
+                                                                             specializedFunctionType.getReturnType());
+                }
+
+                assertFunctionDoesNotReturnGeneric(specializedFunctionType);
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     /*!
      * helper function to check for compatibility, i.e. whether from type can be cast to to type.
      * @param from source type

diff --git a/tuplex/codegen/include/AnnotatedAST.h b/tuplex/codegen/include/AnnotatedAST.h
@@ -177,7 +177,7 @@ namespace tuplex {
 
             /*!
              * annotates the tree with final types. If this is not possible, returns false
-             * @param pokicy compiler policy
+             * @param policy compiler policy
              * @param silentMode determines whether the type inference should log out problems or not
              * @param removeBranches whether to use RemoveDeadBranchesVisitor to prune AST
              * @return whether types could be successfully annotated/defined for all AST nodes

diff --git a/tuplex/codegen/include/BuiltinDictProxy.h b/tuplex/codegen/include/BuiltinDictProxy.h
@@ -0,0 +1,90 @@
+//--------------------------------------------------------------------------------------------------------------------//
+//                                                                                                                    //
+//                                      Tuplex: Blazing Fast Python Data Science                                      //
+//                                                                                                                    //
+//                                                                                                                    //
+//  (c) 2017 - 2021, Tuplex team                                                                                      //
+//  Created by Leonhard Spiegelberg first on 8/9/2021                                                                 //
+//  License: Apache 2.0                                                                                               //
+//--------------------------------------------------------------------------------------------------------------------//
+#ifndef TUPLEX_BUILTINDICTPROXY_H
+#define TUPLEX_BUILTINDICTPROXY_H
+
+#include <memory>
+
+#include <TypeSystem.h>
+#include <BuiltinDictProxyImpl.h>
+#include <cJSONDictProxyImpl.h>
+
+// TODO: Could also use a general object based system which would make things easier...
+// -> i.e., sequence protocol strings/lists/...
+
+// basically for each object we need
+// 1.) representation as C++ object (field)
+// 2.) code-generated logic (i.e., codegen specialization)
+// 3.) to/from python object
+
+namespace tuplex {
+    namespace codegen {
+        class BuiltinDictProxy {
+        public:
+            // BuiltinDictProxy (--> specializedDictType)
+            BuiltinDictProxy(const python::Type& specializedDictType) : _specializedType(specializedDictType) {
+                // use cJSON as default for now...
+                _impl = std::make_shared<cJSONDictProxyImpl>();
+            }
+
+            // use both codegen/non-codegen version
+            // putItem
+            BuiltinDictProxy& putItem(const Field& key, const Field& value) { assert(_impl); _impl->putItem(key, value); return *this; }
+            BuiltinDictProxy& putItem(const python::Type& keyType, const SerializableValue& key, const python::Type& valueType, const SerializableValue& value) { assert(_impl); _impl->putItem(keyType, key, valueType, value); return *this; }
+
+//            // getItem
+//            BuiltinDictProxy& getItem(const Field& key);
+//            BuiltinDictProxy& getItem(const python::Type& keyType, const SerializableValue& key);
+//
+//            // delItem
+//            BuiltinDictProxy& delItem(const Field& key);
+//            BuiltinDictProxy& delItem(const python::Type& keyType, const SerializableValue& key);
+//
+//            // allocSize() --> helpful when dict size is known upfront, can be used for optimization.
+//            BuiltinDictProxy& allocSize(llvm::Value* size);
+
+            // getKeysView() --> codegen object
+
+            // getValuesView() --> codegen object
+
+            python::Type dictType() const {
+                throw std::runtime_error("not yet implemented");
+            }
+
+            python::Type specializedDictType() const {
+                return _specializedType;
+            }
+
+            // codegenToMemory
+
+            // codegenFromMemory
+            // static function?
+
+            // codegenSerializedLength
+
+            // toMemory
+
+            // fromMemory
+            // static function?
+
+            // serializedLength
+        private:
+            python::Type _specializedType;
+
+            // implementation...
+            // -> cJSON
+            // -> ...
+            // -> ...
+            std::shared_ptr<BuiltinDictProxyImpl> _impl;
+        };
+    }
+}
+
+#endif //TUPLEX_BUILTINDICTPROXY_H
diff --git a/tuplex/codegen/include/BuiltinDictProxyImpl.h b/tuplex/codegen/include/BuiltinDictProxyImpl.h
@@ -0,0 +1,40 @@
+//--------------------------------------------------------------------------------------------------------------------//
+//                                                                                                                    //
+//                                      Tuplex: Blazing Fast Python Data Science                                      //
+//                                                                                                                    //
+//                                                                                                                    //
+//  (c) 2017 - 2021, Tuplex team                                                                                      //
+//  Created by Leonhard Spiegelberg first on 8/9/2021                                                                 //
+//  License: Apache 2.0                                                                                               //
+//--------------------------------------------------------------------------------------------------------------------//
+
+#ifndef TUPLEX_BUILTINDICTPROXYIMPL_H
+#define TUPLEX_BUILTINDICTPROXYIMPL_H
+
+#include <TypeSystem.h>
+#include <Row.h>
+#include <CodegenHelper.h>
+
+namespace tuplex {
+    namespace codegen {
+        class BuiltinDictProxyImpl {
+        public:
+            virtual void putItem(const Field& key, const Field& value) = 0;
+            virtual void putItem(const python::Type& keyType, const SerializableValue& key, const python::Type& valueType, const SerializableValue& value) = 0;
+
+            virtual bool keyExists(const Field& key) = 0;
+
+            virtual Field getItem(const Field& key) = 0;
+
+            virtual void replaceItem(const Field& key, const Field& value) = 0;
+
+            virtual void deleteItem(const Field& key) = 0;
+
+            virtual std::vector<Field> getKeysView() = 0;
+
+            virtual std::vector<Field> getValuesView() = 0;
+        };
+    }
+}
+
+#endif //TUPLEX_BUILTINDICTPROXYIMPL_H
diff --git a/tuplex/codegen/include/SymbolTable.h b/tuplex/codegen/include/SymbolTable.h
@@ -171,9 +171,36 @@ namespace tuplex {
          * add an attribute to a builtin type, e.g. str.lower
          * @param builtinType
          * @param name
-         * @param type
+         * @param type type of the attribute, i.e. if it is a function type then a function symbol will be added.
+         * If it is not a function type a variable symbol will be added.
+         */
+        void addBuiltinTypeAttribute(const python::Type& builtinType,
+                                     const std::string& name,
+                                     const python::Type& type);
+
+        /*!
+         * add an attribute to a builtin type, e.g. dict.keys()
+         * @param builtinType to which type to add the function
+         * @param name name of the attribute
+         * @param typer a dynamic typing function
+         * @param sym_type what kind of symbol it is (function? variable?), needed because typer works for both.
+         */
+        void addBuiltinTypeAttribute(const python::Type& builtinType,
+                                     const std::string& name,
+                                     std::function<python::Type(const python::Type&)> typer,
+                                     const SymbolType& sym_type);
+
+        /*!
+         * add an attribute to a builtin type, e.g. dict.keys()
+         * @param builtinType to which type to add the function
+         * @param name name of the attribute
+         * @param typer a dynamic typing function
+         * @param sym_type what kind of symbol it is (function? variable?), needed because typer works for both.
          */
-        void addBuiltinTypeAttribute(const python::Type& builtinType, const std::string& name, const python::Type& type);
+        void addBuiltinTypeAttribute(const python::Type& builtinType,
+                                     const std::string& name,
+                                     std::function<python::Type(const python::Type&, const python::Type&)> attributeTyper,
+                                     const SymbolType& sym_type=SymbolType::FUNCTION);
 
         /*!
          * checks whether a symbol can be looked up or not

diff --git a/tuplex/codegen/include/TypeAnnotatorVisitor.h b/tuplex/codegen/include/TypeAnnotatorVisitor.h
@@ -53,6 +53,10 @@ namespace tuplex {
                                        const TokenType tt, ASTNode* right,
                                        const python::Type& b);
         void assignHelper(NIdentifier *id, python::Type type);
+
+        bool is_nested_subscript_target(ASTNode* target);
+        void recursive_set_subscript_types(NSubscription* target, python::Type value_type);
+
         void checkRetType(python::Type t);
         /*!
          * Annotate iterator-related NCall with iterator-specific info