tuplex · LeonhardFS · Oct 28, 2021 · Oct 22, 2021 · Oct 22, 2021 · Oct 22, 2021
diff --git a/tuplex/codegen/include/ASTNodes.h b/tuplex/codegen/include/ASTNodes.h
@@ -1043,8 +1043,8 @@ namespace tuplex {
     public:
 
         ASTNode *_left;
-        std::vector<TokenType> _ops; // operands
-        std::vector<ASTNode*> _comps; // comparators
+        std::vector<TokenType> _ops; // operators (TokenType::IS, TokenType::NOTEQUAL, etc.)
+        std::vector<ASTNode*> _comps; // operands (54, "hello", False, etc.)
 
         NCompare() : _left(nullptr) {}
 

diff --git a/tuplex/codegen/include/IFailable.h b/tuplex/codegen/include/IFailable.h
@@ -30,6 +30,7 @@ enum class CompileError {
     TYPE_ERROR_RETURN_ITERATOR,
     TYPE_ERROR_NEXT_CALL_DIFFERENT_DEFAULT_TYPE,
     TYPE_ERROR_MIXED_ASTNODETYPE_IN_FOR_LOOP_EXPRLIST, // exprlist contains a mix of tuple/list of identifiers and single identifier
+    TYPE_ERROR_INCOMPATIBLE_TYPES_FOR_IS_COMPARISON, // incompatible types for `is` comparison (one of the types is not BOOLEAN/NULLVALUE).
 };
 
 /*!

diff --git a/tuplex/codegen/include/TokenType.h b/tuplex/codegen/include/TokenType.h
@@ -51,6 +51,7 @@ enum class TokenType {
     IN,
     NOTIN,
     IS,
+    ISNOT,
     LAMBDA,
     NONLOCAL,
     NOT,

diff --git a/tuplex/codegen/src/BlockGeneratorVisitor.cc b/tuplex/codegen/src/BlockGeneratorVisitor.cc
@@ -959,6 +959,20 @@ namespace tuplex {
 
             assert(L);
             assert(R);
+
+            if(tt == TokenType::IS || tt == TokenType::ISNOT) {
+                assert(leftType == python::Type::BOOLEAN || rightType == python::Type::BOOLEAN);
+                // one of the types must be boolean, otherwise compareInst with _isnull would've taken care.
+                if((leftType == python::Type::BOOLEAN) != (rightType == python::Type::BOOLEAN)) {
+                    // one of the types is boolean, other isn't. comparison results in false.
+                    return _env->boolConst(tt == TokenType::ISNOT);
+                } 
+
+                // both must be boolean.
+                auto cmpPredicate = (tt == TokenType::ISNOT) ? llvm::CmpInst::Predicate::ICMP_NE : llvm::CmpInst::Predicate::ICMP_EQ;
+                return _env->upcastToBoolean(builder, builder.CreateICmp(cmpPredicate, L, R));              
+            }
+
             // comparison of values without null
             auto superType = python::Type::superType(leftType.withoutOptions(), rightType.withoutOptions());
             if (superType == python::Type::UNKNOWN) {
@@ -981,10 +995,11 @@ namespace tuplex {
 
 
         llvm::Value* BlockGeneratorVisitor::oneSidedNullComparison(llvm::IRBuilder<>& builder, const python::Type& type, const TokenType& tt, llvm::Value* isnull) {
-            assert(tt == TokenType::EQEQUAL || tt == TokenType::NOTEQUAL); // only for == or !=!
+            assert(tt == TokenType::EQEQUAL || tt == TokenType::NOTEQUAL || tt == TokenType::IS || tt == TokenType::ISNOT); // only for == or != or IS or ISNOT!
 
+            // we're comparing null to null, should only return true if operators are EQEQUAL or IS. 
             if(type == python::Type::NULLVALUE)
-                return _env->boolConst(tt == TokenType::EQEQUAL); // if == then true, if != then false
+                return _env->boolConst(tt == TokenType::EQEQUAL || tt == TokenType::IS); // if == then true, if != then false
 
             // option type? check isnull
             // else, super simple. Decide on tokentype
@@ -996,15 +1011,22 @@ namespace tuplex {
                 // the other side is null
                 // if isnull is true && equal => true
                 // if isnull is false && notequal => false (case 12 != None)
-                if(tt == TokenType::NOTEQUAL)
+
+                // for IS NOT, if isnull is true, we want to return false.
+                // if isnull is false, we want to return true.
+                // therefore we negate. (similar to logic for NOTEQUAL).  
+                if(tt == TokenType::NOTEQUAL || tt == TokenType::ISNOT)
                     return _env->upcastToBoolean(builder, _env->i1neg(builder, isnull));
                 else
                     return _env->upcastToBoolean(builder, isnull);
             } else {
                 // the other side is null
                 // => 12 != null => true
                 // => 12 == null => false
-                return _env->boolConst(tt == TokenType::NOTEQUAL);
+
+                // we are now comparing a non-null type to null.
+                // so we return true only if token is IS NOT or NOTEQUAL.
+                return _env->boolConst(tt == TokenType::NOTEQUAL || tt == TokenType::ISNOT);
             }
         }
 
@@ -1014,7 +1036,8 @@ namespace tuplex {
                                            const python::Type &rightType) {
 
             // None comparisons only work for == or !=, i.e. for all other ops throw exception
-            if (tt == TokenType::EQEQUAL || tt == TokenType::NOTEQUAL) {
+            if (tt == TokenType::EQEQUAL || tt == TokenType::NOTEQUAL || tt == TokenType::IS || tt == TokenType::ISNOT) {
+
                 // special case: one side is None
                 if(leftType == python::Type::NULLVALUE || rightType == python::Type::NULLVALUE) {
 
@@ -1043,7 +1066,7 @@ namespace tuplex {
                         assert(L);
                         assert(R);
 
-                        auto resVal = _env->CreateTernaryLogic(builder, L_isnull, [&] (llvm::IRBuilder<>& builder) { return _env->boolConst(tt == TokenType::NOTEQUAL); },
+                        auto resVal = _env->CreateTernaryLogic(builder, L_isnull, [&] (llvm::IRBuilder<>& builder) { return _env->boolConst(tt == TokenType::NOTEQUAL || tt == TokenType::ISNOT); },
                                                                [&] (llvm::IRBuilder<>& builder) { return compareInst(builder, L, leftType.withoutOptions(), tt, R, rightType); });
                         _lfb->setLastBlock(builder.GetInsertBlock());
                         return resVal;
@@ -1059,7 +1082,7 @@ namespace tuplex {
                         assert(L);
                         assert(R);
 
-                        auto resVal = _env->CreateTernaryLogic(builder, R_isnull, [&] (llvm::IRBuilder<>& builder) { return _env->boolConst(tt == TokenType::NOTEQUAL); },
+                        auto resVal = _env->CreateTernaryLogic(builder, R_isnull, [&] (llvm::IRBuilder<>& builder) { return _env->boolConst(tt == TokenType::NOTEQUAL || tt == TokenType::ISNOT); },
                                                                [&] (llvm::IRBuilder<>& builder) { return compareInst(builder, L, leftType, tt, R, rightType.withoutOptions()); });
                         _lfb->setLastBlock(builder.GetInsertBlock());
                         return resVal;
@@ -1076,7 +1099,7 @@ namespace tuplex {
                         // compareInst if both are NOT none
                         auto bothValid = builder.CreateAnd(L_isnull, R_isnull);
                         auto xorResult = builder.CreateXor(L_isnull, R_isnull);
-                        if (TokenType::EQEQUAL == tt)
+                        if (tt == TokenType::EQEQUAL || tt == TokenType::IS)
                             xorResult = builder.CreateNot(xorResult);
 
                         auto resVal = _env->CreateTernaryLogic(builder, bothValid, [&] (llvm::IRBuilder<>& builder) { return compareInst(builder, L, leftType.withoutOptions(), tt, R,

diff --git a/tuplex/codegen/src/IFailable.cc b/tuplex/codegen/src/IFailable.cc
@@ -71,6 +71,9 @@ std::string IFailable::compileErrorToStr(const CompileError &err) {
         case CompileError::TYPE_ERROR_MIXED_ASTNODETYPE_IN_FOR_LOOP_EXPRLIST:
             errMsg = "mixed use of tuple/list of identifiers and single identifier in exprlist not yet supported";
             break;
+        case CompileError::TYPE_ERROR_INCOMPATIBLE_TYPES_FOR_IS_COMPARISON:
+            errMsg = "use of is comparison only supported with types boolean and null";
+            break;
         default:
             break;
     }

diff --git a/tuplex/codegen/src/Token.cc b/tuplex/codegen/src/Token.cc
@@ -45,6 +45,7 @@ std::ostream& operator<< (std::ostream& os, const TokenType tt)
         case TokenType::IMPORT: return os<<"IMPORT";
         case TokenType::IN: return os<<"IN";
         case TokenType::IS: return os<<"IS";
+        case TokenType::ISNOT: return os<<"ISNOT";
         case TokenType::LAMBDA: return os<<"LAMBDA";
         case TokenType::NONLOCAL: return os<<"NONLOCAL";
         case TokenType::NOT: return os<<"NOT";
@@ -112,7 +113,10 @@ TokenType stringToToken(const std::string& s) {
         return TokenType::IN;
     if(s == "notin")
         return TokenType::NOTIN;
-
+    if(s == "is")
+        return TokenType::IS;
+    if(s == "isnot")
+        return TokenType::ISNOT;
     if(s == "&&" || s == "and")
         return TokenType::AND;
     if(s == "!")
@@ -267,6 +271,9 @@ std::string opToString(const TokenType& tt) {
         case TokenType::DOUBLESLASHEQUAL: return "//=";
         case TokenType::ELLIPSIS: return "...";
         case TokenType::IN: return "in";
+        case TokenType::IS: return "is";
+        case TokenType::ISNOT: return "is not";
+
         default: return std::to_string(static_cast<std::uint16_t>(tt));
     }
 }
diff --git a/tuplex/codegen/src/TypeAnnotatorVisitor.cc b/tuplex/codegen/src/TypeAnnotatorVisitor.cc
@@ -535,6 +535,32 @@ namespace tuplex {
         else
             // else it is a bool (b.c. it is a compare statement)
             cmp->setInferredType(python::Type::BOOLEAN);
+
+
+        // check if `is` comparison is valid
+        std::unordered_set<python::Type> validTypes = {python::Type::NULLVALUE, python::Type::BOOLEAN};
+        // one of every two types must be in validTypes.
+
+        if(cmp->_comps.size() >= 1) {
+            if(cmp->_ops[0] == TokenType::IS && !validTypes.count(cmp->_left->getInferredType()) && !validTypes.count(cmp->_comps[0]->getInferredType())) {
+                // invalid types for lhs and rhs to do an `is` comparison.
+                addCompileError(CompileError::TYPE_ERROR_INCOMPATIBLE_TYPES_FOR_IS_COMPARISON);
+                return;
+            }
+        }
+
+        bool lastValid = false;
+        // if more that one operand, check if all types would be valid.
+        for(int i = 0; i < cmp->_comps.size() - 1; i++) {
+            auto currType = cmp->_comps[i]->getInferredType();
+            auto nextType = cmp->_comps[i+1]->getInferredType();
+
+            // type error only if previous comparison is invalid
+            if(!validTypes.count(currType) && !validTypes.count(nextType) && cmp->_ops[i] == TokenType::IS) {
+                // neither type is valid for an is comparison. 
+                addCompileError(CompileError::TYPE_ERROR_INCOMPATIBLE_TYPES_FOR_IS_COMPARISON);
+            }
+        }
     }
 
     python::Type TypeAnnotatorVisitor::binaryOpInference(ASTNode* left, const python::Type& a,

diff --git a/tuplex/core/src/TraceVisitor.cc b/tuplex/core/src/TraceVisitor.cc
@@ -388,6 +388,7 @@ namespace tuplex {
         // now truth value testing, single element?
         auto res = ti_vals.front();
 
+        // IS and IS NOT are equivalent to id(L) == id(R) and id(L) != id(R).
         std::unordered_map<TokenType, int> cmpLookup{{TokenType::EQEQUAL, Py_EQ},
                                                      {TokenType::NOTEQUAL, Py_NE},
                                                      {TokenType::LESS, Py_LT},
@@ -398,18 +399,25 @@ namespace tuplex {
         // eval
         for(int i = 0; i < node->_ops.size(); ++i) {
             auto op = node->_ops[i];
-            auto it = cmpLookup.find(op);
-            if(it == cmpLookup.end())
-                   throw std::runtime_error("Operator " + opToString(op) + " not yet supported in TraceVisitor/NCompare");
-            int opid = it->second;
 
-            // debug:
-            auto info = python::PyString_AsString(res.value) + " " + opToString(op) + " " +
-                    python::PyString_AsString(ti_vals[i+1].value);
-
-            res.value = PyObject_RichCompare(res.value, ti_vals[i + 1].value, opid);
+            // based on op, decide value of result.
+            if(op == TokenType::IS || op == TokenType::ISNOT) {
+                // `x is y` in Python is equivalent to `id(x) == id(y)`
+                // so we just compare pointers for equality and return the corresponding PyBool.
+
+                assert(i+1 < ti_vals.size());
+                bool finalResult = (ti_vals[i+1].value == res.value);
+                // invert result if op is ISNOT.
+                finalResult = (op == TokenType::IS) ? finalResult : !finalResult;
+                res.value = finalResult ? Py_True : Py_False;
+            } else {
+                auto it = cmpLookup.find(op);
+                if(it == cmpLookup.end())
+                        throw std::runtime_error("Operator " + opToString(op) + " not yet supported in TraceVisitor/NCompare");
+                int opid = it->second;
 
-            auto res_info = "is: " + python::PyString_AsString(res.value);
+                res.value = PyObject_RichCompare(res.value, ti_vals[i + 1].value, opid);
+            }
 
             // NULL? ==> failure!
             assert(res.value);

diff --git a/tuplex/python/tests/test_is.py b/tuplex/python/tests/test_is.py
@@ -0,0 +1,38 @@
+import tuplex
+from unittest import TestCase
+"""
+Tests functionality for `is` keyword. 
+"""
+class TestIs(TestCase):
+
+    def setUp(self):
+        self.conf = {"webui.enable": False, "executorCount": "0"}
+        self.c = tuplex.Context(self.conf)
+
+    def test_boolIsBool(self):
+        res = self.c.parallelize([False, True, False, False, True]).map(lambda x: x is False).collect()
+        self.assertEqual(res, [True, False, True, True, False])
+
+    def test_boolIsNotBool(self):
+        res = self.c.parallelize([True, False, True, False, True]).map(lambda x: x is not False).collect()
+        self.assertEqual(res, [True, False, True, False, True])
+
+    def test_boolIsNone(self):
+        res = self.c.parallelize([True, False, False, True]).map(lambda x: x is None).collect()
+        self.assertEqual(res, [False] * 4)
+
+    def test_mixedIsNone(self):
+        res = self.c.parallelize([None, 255, 400, False, 2.3]).map(lambda x: x is None).collect()
+        self.assertEqual(res, [True, False, False, False, False])
+
+    def test_mixedIsNotNone(self):
+        res = self.c.parallelize([None, None, None]).map(lambda x: x is not None).collect()
+        self.assertEqual(res, [False, False, False])
+
+    def test_mixedIsNotNone2(self):
+        res = self.c.parallelize([None, True, False]).map(lambda x: x is not None).collect()
+        self.assertEqual(res, [False, True, True])
+
+    def test_mixedIsNotNone3(self):
+        res = self.c.parallelize([2, False, None]).map(lambda x: x is not None).collect()
+        self.assertEqual(res, [True, True, False])
-Original file line number
+Diff line change
@@ Expand Up / @@ -51,6 +51,7 @@ enum class TokenType { @@
         IN,
         NOTIN,
         IS,
+        ISNOT,
         LAMBDA,
         NONLOCAL,
         NOT,
@@ Expand Down @@