tuplex · LeonhardFS · Sep 4, 2022 · Sep 5, 2022 · Sep 5, 2022 · Sep 5, 2022
diff --git a/benchmarks/311-demo/runtuplex.py b/benchmarks/311-demo/runtuplex.py
@@ -69,7 +69,7 @@ def fix_zip_codes(zips):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": True,
         "optimizer.nullValueOptimization": True,
-        "csv.selectionPushdown": True,
+        "optimizer.selectionPushdown": True,
         "optimizer.generateParser": True,
         "tuplex.optimizer.mergeExceptionsInOrder": False,
         "csv.filterPushdown": True,

diff --git a/benchmarks/311-demo/tuplex_config.json b/benchmarks/311-demo/tuplex_config.json
@@ -1 +1 @@
-{"webui.enable": false, "executorMemory": "4G", "executorCount": 63, "driverMemory": "4G", "partitionSize": "32MB", "runTimeMemory": "4MB", "inputSplitSize": "8MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "csv.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
+{"webui.enable": false, "executorMemory": "4G", "executorCount": 63, "driverMemory": "4G", "partitionSize": "32MB", "runTimeMemory": "4MB", "inputSplitSize": "8MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "optimizer.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
diff --git a/benchmarks/311/create_conf.py b/benchmarks/311/create_conf.py
@@ -29,7 +29,7 @@
             'inputSplitSize' : args.input_split_size,
             'useLLVMOptimizer' : args.opt_llvm,
             'optimizer.nullValueOptimization' : args.opt_null,
-            'csv.selectionPushdown' : args.opt_pushdown,
+            'optimizer.selectionPushdown' : args.opt_pushdown,
             'optimizer.generateParser' : args.opt_parser,
             'optimizer.mergeExceptionsInOrder' : False,
             'optimizer.filterPushdown' : args.opt_filter}

diff --git a/benchmarks/311/runtuplex.py b/benchmarks/311/runtuplex.py
@@ -69,7 +69,7 @@ def fix_zip_codes(zips):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": True,
         "optimizer.nullValueOptimization": True,
-        "csv.selectionPushdown": True,
+        "optimizer.selectionPushdown": True,
         "optimizer.generateParser": True,
         "tuplex.optimizer.mergeExceptionsInOrder": False,
         "csv.filterPushdown": True,

diff --git a/benchmarks/311/tuplex_config_mt.json b/benchmarks/311/tuplex_config_mt.json
@@ -1 +1 @@
-{"webui.enable": false, "executorMemory": "6G", "executorCount": 15, "driverMemory": "10G", "partitionSize": "32MB", "runTimeMemory": "64MB", "inputSplitSize": "64MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "csv.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
+{"webui.enable": false, "executorMemory": "6G", "executorCount": 15, "driverMemory": "10G", "partitionSize": "32MB", "runTimeMemory": "64MB", "inputSplitSize": "64MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "optimizer.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
diff --git a/benchmarks/311/tuplex_config_st.json b/benchmarks/311/tuplex_config_st.json
@@ -1 +1 @@
-{"webui.enable": false, "executorMemory": "6G", "executorCount": 0, "driverMemory": "100G", "partitionSize": "32MB", "runTimeMemory": "64MB", "inputSplitSize": "64MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "csv.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
+{"webui.enable": false, "executorMemory": "6G", "executorCount": 0, "driverMemory": "100G", "partitionSize": "32MB", "runTimeMemory": "64MB", "inputSplitSize": "64MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "optimizer.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -262,7 +262,7 @@ Use following config for flights, to get it fast...
 "autoUpcast":true,
 "optimizer.generateParser":true,
 "optimizer.nullValueOptimization": true,
-"csv.selectionPushdown": true,
+"optimizer.selectionPushdown": true,
 "resolveWithInterpreterOnly":false}
 ```
 

diff --git a/benchmarks/dirty_zillow/create_conf.py b/benchmarks/dirty_zillow/create_conf.py
@@ -29,7 +29,7 @@
             'inputSplitSize' : args.input_split_size,
             'useLLVMOptimizer' : args.opt_llvm,
             'optimizer.nullValueOptimization' : args.opt_null,
-            'csv.selectionPushdown' : args.opt_pushdown,
+            'optimizer.selectionPushdown' : args.opt_pushdown,
             'optimizer.generateParser' : args.opt_parser,
             'optimizer.mergeExceptionsInOrder' : False,
             'optimizer.filterPushdown' : args.opt_filter}

diff --git a/benchmarks/dirty_zillow/runtuplex.py b/benchmarks/dirty_zillow/runtuplex.py
@@ -471,7 +471,7 @@ def extractZipcode(x):
             "runTimeMemory" : "128MB",
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : False,
-            "csv.selectionPushdown" : True,
+            "optimizer.selectionPushdown" : True,
             "optimizer.generateParser" : False} # bug when using generated parser. Need to fix that.
 
     if os.path.exists('tuplex_config.json'):

diff --git a/benchmarks/dirty_zillow/tuplex_config.json b/benchmarks/dirty_zillow/tuplex_config.json
@@ -1 +1 @@
-{"webui.enable": false, "executorMemory": "6G", "executorCount": 15, "driverMemory": "10G", "partitionSize": "32MB", "runTimeMemory": "64MB", "inputSplitSize": "64MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "csv.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
+{"webui.enable": false, "executorMemory": "6G", "executorCount": 15, "driverMemory": "10G", "partitionSize": "32MB", "runTimeMemory": "64MB", "inputSplitSize": "64MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "optimizer.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
diff --git a/benchmarks/distributed/tuplex/run_tuplex.py b/benchmarks/distributed/tuplex/run_tuplex.py
@@ -180,7 +180,7 @@ def filterBd(x):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": True,
         "optimizer.nullValueOptimization": False,
-        "csv.selectionPushdown": True,
+        "optimizer.selectionPushdown": True,
     }
 
     # Begin pipeline

diff --git a/benchmarks/flights/create_conf.py b/benchmarks/flights/create_conf.py
@@ -29,7 +29,7 @@
             'inputSplitSize' : args.input_split_size,
             'useLLVMOptimizer' : args.opt_llvm,
             'optimizer.nullValueOptimization' : args.opt_null,
-            'csv.selectionPushdown' : args.opt_pushdown,
+            'optimizer.selectionPushdown' : args.opt_pushdown,
             'optimizer.generateParser' : args.opt_parser,
             'optimizer.mergeExceptionsInOrder' : False,
             'optimizer.filterPushdown' : args.opt_filter,

diff --git a/benchmarks/flights/run_on_aws.py b/benchmarks/flights/run_on_aws.py
@@ -289,7 +289,7 @@ def run_tuplex(instances, branch, enableLLVMOpt, enablePushdown, num_runs=5, num
             "partitionSize": "32MB",
             "runTimeMemory": "256MB",
             "useLLVMOptimizer": enableLLVMOpt,
-            "csv.selectionPushdown": enablePushdown}
+            "optimizer.selectionPushdown": enablePushdown}
     with open(config_file, 'w') as fp:
         json.dump(conf, fp)
 

diff --git a/benchmarks/flights/runtuplex.py b/benchmarks/flights/runtuplex.py
@@ -77,7 +77,7 @@
         "runTimeMemory" : "128MB",
         "useLLVMOptimizer" : False,
         "optimizer.nullValueOptimization" : False,
-        "csv.selectionPushdown" : True,
+        "optimizer.selectionPushdown" : True,
         "tuplex.optimizer.generateParser": True}
 
 
@@ -105,7 +105,7 @@
     df = df.renameColumn(c, renamed_cols[i])
 
 # if cache/nosf mode is active, prefilter
-if args.simulate_spark and conf["csv.selectionPushdown"]:
+if args.simulate_spark and conf["optimizer.selectionPushdown"]:
     df = df.selectColumns(time_req_cols)
 if args.simulate_spark:
     df = df.cache()

diff --git a/benchmarks/flights/tuplex_config.json b/benchmarks/flights/tuplex_config.json
@@ -6,6 +6,6 @@
 "runTimeMemory": "8MB",
 "useLLVMOptimizer": true,
 "optimizer.nullValueOptimization": true,
-"csv.selectionPushdown": true,
+"optimizer.selectionPushdown": true,
 "resolveWithInterpreterOnly":false,
 "mergeRowsInOrder":false}
diff --git a/benchmarks/flights/tuplex_config_template.json b/benchmarks/flights/tuplex_config_template.json
@@ -6,6 +6,6 @@
 "runTimeMemory": "8MB",
 "useLLVMOptimizer": true,
 "optimizer.nullValueOptimization": true,
-"csv.selectionPushdown": true,
+"optimizer.selectionPushdown": true,
 "resolveWithInterpreterOnly":false,
 "mergeRowsInOrder":false}
diff --git a/benchmarks/logs/create_conf.py b/benchmarks/logs/create_conf.py
@@ -29,7 +29,7 @@
             'inputSplitSize' : args.input_split_size,
             'useLLVMOptimizer' : args.opt_llvm,
             'optimizer.nullValueOptimization' : args.opt_null,
-            'csv.selectionPushdown' : args.opt_pushdown,
+            'optimizer.selectionPushdown' : args.opt_pushdown,
             'optimizer.generateParser' : args.opt_parser,
             'optimizer.mergeExceptionsInOrder' : False,
             'optimizer.filterPushdown' : args.opt_filter}

diff --git a/benchmarks/logs/manual_reorder/runtuplex-io.py b/benchmarks/logs/manual_reorder/runtuplex-io.py
@@ -212,7 +212,7 @@ def randomize_udf(x):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": False,
         "nullValueOptimization": False,
-        "csv.selectionPushdown": False,
+        "optimizer.selectionPushdown": False,
         "optimizer.generateParser": False
     }
 

diff --git a/benchmarks/logs/manual_reorder/runtuplex.py b/benchmarks/logs/manual_reorder/runtuplex.py
@@ -220,7 +220,7 @@ def randomize_udf(x):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": False,
         "nullValueOptimization": False,
-        "csv.selectionPushdown": False,
+        "optimizer.selectionPushdown": False,
         "optimizer.generateParser": False
     }
 

diff --git a/benchmarks/logs/manual_reorder/tuplex_config.json b/benchmarks/logs/manual_reorder/tuplex_config.json
@@ -7,7 +7,7 @@
   "inputSplitSize": "64MB",
   "useLLVMOptimizer": true,
   "optimizer.nullValueOptimization": true,
-  "csv.selectionPushdown": true,
+  "optimizer.selectionPushdown": true,
   "optimizer.generateParser": false,
   "optimizer.mergeExceptionsInOrder": false,
   "optimizer.filterPushdown": true}
diff --git a/benchmarks/logs/microbenchmarks/single_regex/runtuplex.py b/benchmarks/logs/microbenchmarks/single_regex/runtuplex.py
@@ -70,7 +70,7 @@ def ParseWithRegex(logline):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": False,
         "nullValueOptimization": False,
-        "csv.selectionPushdown": False,
+        "optimizer.selectionPushdown": False,
         "optimizer.generateParser": False
     }
 

diff --git a/benchmarks/logs/microbenchmarks/single_regex/tuplex_config.json b/benchmarks/logs/microbenchmarks/single_regex/tuplex_config.json
@@ -7,6 +7,6 @@
   "runTimeMemory": "256MB",
   "useLLVMOptimizer": true,
   "optimizer.nullValueOptimization": false,
-  "csv.selectionPushdown": false,
+  "optimizer.selectionPushdown": false,
   "optimizer.generateParser": false
 }
diff --git a/benchmarks/logs/runtuplex-io.py b/benchmarks/logs/runtuplex-io.py
@@ -156,7 +156,7 @@ def randomize_udf(x):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": False,
         "nullValueOptimization": False,
-        "csv.selectionPushdown": False,
+        "optimizer.selectionPushdown": False,
         "optimizer.generateParser": False
     }
 

diff --git a/benchmarks/logs/runtuplex.py b/benchmarks/logs/runtuplex.py
@@ -220,7 +220,7 @@ def randomize_udf(x):
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": False,
         "nullValueOptimization": False,
-        "csv.selectionPushdown": False,
+        "optimizer.selectionPushdown": False,
         "optimizer.generateParser": False
     }
 

diff --git a/benchmarks/logs/tuplex_config.json b/benchmarks/logs/tuplex_config.json
@@ -7,7 +7,7 @@
   "inputSplitSize": "64MB",
   "useLLVMOptimizer": true,
   "optimizer.nullValueOptimization": true,
-  "csv.selectionPushdown": true,
+  "optimizer.selectionPushdown": true,
   "optimizer.generateParser": false,
   "optimizer.mergeExceptionsInOrder": false,
   "optimizer.filterPushdown": true}
diff --git a/benchmarks/orc/runcsv-read-single.py b/benchmarks/orc/runcsv-read-single.py
@@ -41,7 +41,7 @@
             "runTimeMemory" : "128MB",
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : False,
-            "csv.selectionPushdown" : True}
+            "optimizer.selectionPushdown" : True}
 
     if os.path.exists('tuplex_config.json'):
         with open('tuplex_config.json') as fp:

diff --git a/benchmarks/orc/runcsv-write.py b/benchmarks/orc/runcsv-write.py
@@ -41,7 +41,7 @@
             "runTimeMemory" : "128MB",
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : False,
-            "csv.selectionPushdown" : True}
+            "optimizer.selectionPushdown" : True}
 
     if os.path.exists('tuplex_config.json'):
         with open('tuplex_config.json') as fp:

diff --git a/benchmarks/orc/runorc-read-single.py b/benchmarks/orc/runorc-read-single.py
@@ -41,7 +41,7 @@
             "runTimeMemory" : "128MB",
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : False,
-            "csv.selectionPushdown" : True}
+            "optimizer.selectionPushdown" : True}
 
     if os.path.exists('tuplex_config.json'):
         with open('tuplex_config.json') as fp:

diff --git a/benchmarks/orc/runorc-write.py b/benchmarks/orc/runorc-write.py
@@ -41,7 +41,7 @@
             "runTimeMemory" : "128MB",
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : False,
-            "csv.selectionPushdown" : True}
+            "optimizer.selectionPushdown" : True}
 
     if os.path.exists('tuplex_config.json'):
         with open('tuplex_config.json') as fp:

diff --git a/benchmarks/sigmod21-reproducibility/plot_scripts/figure10.py b/benchmarks/sigmod21-reproducibility/plot_scripts/figure10.py
@@ -121,7 +121,7 @@ def extractType(row):
             t += 'sf'
         else:
             t += 'nosf'
-        if row['csv.selectionPushdown']:
+        if row['optimizer.selectionPushdown']:
             t += '+logical'
         if row['optimizer.nullValueOptimization']:
             t += '+null'

diff --git a/benchmarks/tpch/Q06/create_conf.py b/benchmarks/tpch/Q06/create_conf.py
@@ -30,7 +30,7 @@
             'inputSplitSize' : args.input_split_size,
             'useLLVMOptimizer' : args.opt_llvm,
             'optimizer.nullValueOptimization' : args.opt_null,
-            'csv.selectionPushdown' : args.opt_pushdown,
+            'optimizer.selectionPushdown' : args.opt_pushdown,
             'optimizer.generateParser' : args.opt_parser,
             'optimizer.mergeExceptionsInOrder' : False,
             'optimizer.filterPushdown' : args.opt_filter}

diff --git a/benchmarks/tpch/Q06/runtuplex.py b/benchmarks/tpch/Q06/runtuplex.py
@@ -36,7 +36,7 @@
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": True,
         "nullValueOptimization": True,
-        "csv.selectionPushdown": True,
+        "optimizer.selectionPushdown": True,
         "optimizer.generateParser": True,
     }
 

diff --git a/benchmarks/tpch/Q19/create_conf.py b/benchmarks/tpch/Q19/create_conf.py
@@ -30,7 +30,7 @@
             'inputSplitSize' : args.input_split_size,
             'useLLVMOptimizer' : args.opt_llvm,
             'optimizer.nullValueOptimization' : args.opt_null,
-            'csv.selectionPushdown' : args.opt_pushdown,
+            'optimizer.selectionPushdown' : args.opt_pushdown,
             'optimizer.generateParser' : args.opt_parser,
             'optimizer.mergeExceptionsInOrder' : False,
             'optimizer.filterPushdown' : args.opt_filter}

diff --git a/benchmarks/tpch/Q19/runtuplex.py b/benchmarks/tpch/Q19/runtuplex.py
@@ -69,7 +69,7 @@
         "runTimeMemory": "128MB",
         "useLLVMOptimizer": True,
         "nullValueOptimization": True,
-        "csv.selectionPushdown": True,
+        "optimizer.selectionPushdown": True,
         "optimizer.generateParser": True,
     }
 

diff --git a/benchmarks/zillow/Z1/baseline/tuplex_config.json b/benchmarks/zillow/Z1/baseline/tuplex_config.json
@@ -8,7 +8,7 @@
   "runTimeMemory": "256MB",
   "useLLVMOptimizer": true,
   "optimizer.nullValueOptimization": true,
-  "csv.selectionPushdown": true,
+  "optimizer.selectionPushdown": true,
   "optimizer.generateParser": true,
   "optimizer.filterPushdown": true
 }
diff --git a/benchmarks/zillow/Z1/runtuplex-io.py b/benchmarks/zillow/Z1/runtuplex-io.py
@@ -149,7 +149,7 @@ def filterBd(x):
             "runTimeMemory" : "128MB",
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : True,
-            "csv.selectionPushdown" : True}
+            "optimizer.selectionPushdown" : True}
 
     # # conf for r5d.4xlarge (in total: 100GB, same as spark)
     # conf = {"webui.enable" : False,

diff --git a/benchmarks/zillow/Z1/runtuplex.py b/benchmarks/zillow/Z1/runtuplex.py
@@ -155,7 +155,7 @@ def filterBd(x):
             "runTimeMemory" : "128MB",
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : False,
-            "csv.selectionPushdown" : True}
+            "optimizer.selectionPushdown" : True}
 
     # # conf for r5d.4xlarge (in total: 100GB, same as spark)
     # conf = {"webui.enable" : False,

diff --git a/benchmarks/zillow/Z1/tuplex_config.json b/benchmarks/zillow/Z1/tuplex_config.json
@@ -6,4 +6,4 @@
   "runTimeMemory": "8MB",
   "useLLVMOptimizer": true,
   "optimizer.nullValueOptimization": true,
-  "csv.selectionPushdown": true}
+  "optimizer.selectionPushdown": true}
diff --git a/benchmarks/zillow/Z2/baseline/tuplex_config.json b/benchmarks/zillow/Z2/baseline/tuplex_config.json
@@ -8,7 +8,7 @@
   "runTimeMemory": "256MB",
   "useLLVMOptimizer": true,
   "optimizer.nullValueOptimization": true,
-  "csv.selectionPushdown": true,
+  "optimizer.selectionPushdown": true,
   "optimizer.generateParser": true,
   "optimizer.filterPushdown": true
 }
diff --git a/benchmarks/zillow/Z2/runtuplex.py b/benchmarks/zillow/Z2/runtuplex.py
@@ -158,7 +158,7 @@ def filterBd(x):
             "useLLVMOptimizer" : True,
             "optimizer.nullValueOptimization" : False,
             "tuplex.allowUndefinedBehavior" : False,
-            "csv.selectionPushdown" : True,
+            "optimizer.selectionPushdown" : True,
             "optimizer.filterPushdown" : True,
             "optimizer.generateParser": False}
 

diff --git a/benchmarks/zillow/Z2/tuplex_config.json b/benchmarks/zillow/Z2/tuplex_config.json
@@ -6,4 +6,4 @@
   "runTimeMemory": "8MB",
   "useLLVMOptimizer": true,
   "optimizer.nullValueOptimization": true,
-  "csv.selectionPushdown": true}
+  "optimizer.selectionPushdown": true}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"webui.enable": false, "executorMemory": "4G", "executorCount": 63, "driverMemory": "4G", "partitionSize": "32MB", "runTimeMemory": "4MB", "inputSplitSize": "8MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "csv.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}
		{"webui.enable": false, "executorMemory": "4G", "executorCount": 63, "driverMemory": "4G", "partitionSize": "32MB", "runTimeMemory": "4MB", "inputSplitSize": "8MB", "useLLVMOptimizer": true, "optimizer.nullValueOptimization": false, "optimizer.selectionPushdown": true, "optimizer.generateParser": false, "optimizer.mergeExceptionsInOrder": false, "optimizer.filterPushdown": true}