DayF core  1.2.1.2
DayF (Decision at your Fingertips) is an AutoML freeware development framework that let developers works with Machine Learning models without any idea of AI, simply taking a csv dataset and the objective column
spark_optimizer.py
1 
5 
6 from gdayf.common.utils import get_model_fw
7 from gdayf.conf.loadconfig import LoadConfig
8 from gdayf.common.utils import decode_ordered_dict_to_dataframe
9 
10 class Optimizer(object):
11 
12 
17  def __init__(self, e_c):
18  self._ec = e_c
19  self._labels = self._ec.labels.get_config()['messages']['adviser']
20  self._config = LoadConfig().get_config()['optimizer']['AdviserStart_rules']['spark']
21 
22 
30  def optimize_models(self, armetadata, metric_value, objective, deepness, deep_impact):
31  model_list = list()
32  model = armetadata['model_parameters'][get_model_fw(armetadata)]
33  config = self._config
34 
35  if get_model_fw(armetadata) == 'spark' and metric_value != objective \
36  and armetadata['status'] != self._labels['failed_op']:
37  try:
38  scoring_metric = decode_ordered_dict_to_dataframe(armetadata['metrics']['scoring'])
39  except ValueError:
40  print("TRACE: Not scoring: " + model)
41  min_rows_limit = config['min_rows_limit']
42  min_rows_increment = config['min_rows_increment']
43  max_interactions_increment = config['max_interactions_increment']
44  interactions_increment = config['interactions_increment']
45  max_depth_increment = config['max_depth_increment']
46  ntrees_increment = config['ntrees_increment']
47  stepSize = config['stepSize']
48  aggregationDepth_increment = config['aggregationDepth_increment']
49  regParam = config['regParam']
50  elastic_variation = config['elastic_variation']
51  nv_smoothing = config['nv_smoothing']
52  nv_improvement = config['nv_improvement']
53  nv_divisor = config['nv_divisor']
54  clustering_increment = config['clustering_increment']
55  initstep_increment = config['initstep_increment']
56 
57  if model['model'] == 'LinearSVC':
58  if deepness == 2 and len(regParam) != 0:
59  for elastic in regParam:
60  new_armetadata = armetadata.copy_template()
61  model_aux = new_armetadata['model_parameters']['spark']
62  model_aux['parameters']['regParam']['value'] = elastic['value']
63  model_list.append(new_armetadata)
64 
65  try:
66  if model['parameters']['maxIter']['value'] \
67  >= scoring_metric['totalIterations'][0] and \
68  scoring_metric['totalIterations'][0] <= max_interactions_increment:
69  new_armetadata = armetadata.copy_template()
70  model_aux = new_armetadata['model_parameters']['spark']
71  model_aux['parameters']['maxIter']['value'] *= interactions_increment
72  model_list.append(new_armetadata)
73  except KeyError:
74  if model['parameters']['maxIter']['value'] \
75  <= max_interactions_increment:
76  new_armetadata = armetadata.copy_template()
77  model_aux = new_armetadata['model_parameters']['spark']
78  model_aux['parameters']['maxIter']['value'] *= interactions_increment
79  model_list.append(new_armetadata)
80 
81  new_armetadata = armetadata.copy_template()
82  model_aux = new_armetadata['model_parameters']['spark']
83  model_aux['parameters']['aggregationDepth']['value'] *= aggregationDepth_increment
84  model_list.append(new_armetadata)
85 
86  elif model['model'] == 'LogisticRegression' or model['model'] == 'LinearRegression':
87  if deepness == 2 and len(regParam) != 0:
88  for elastic in regParam:
89  new_armetadata = armetadata.copy_template()
90  model_aux = new_armetadata['model_parameters']['spark']
91  model_aux['parameters']['regParam']['value'] = elastic['value']
92  model_list.append(new_armetadata)
93 
94  if model['parameters']['elasticNetParam']['value'] \
95  * (1 + elastic_variation) <= 1.0:
96  new_armetadata = armetadata.copy_template()
97  model_aux = new_armetadata['model_parameters']['spark']
98  model_aux['parameters']['elasticNetParam']['value'] = \
99  model_aux['parameters']['elasticNetParam']['value'] * (1 + elastic_variation)
100  model_list.append(new_armetadata)
101  if model['parameters']['elasticNetParam']['value'] \
102  * (1 - elastic_variation) >= 0.0:
103  new_armetadata = armetadata.copy_template()
104  model_aux = new_armetadata['model_parameters']['spark']
105  model_aux['parameters']['elasticNetParam']['value'] = \
106  model_aux['parameters']['elasticNetParam']['value'] * (1 + elastic_variation)
107  model_list.append(new_armetadata)
108 
109  try:
110  if model['parameters']['maxIter']['value'] \
111  >= scoring_metric['totalIterations'][0] and \
112  scoring_metric['totalIterations'][0] <= max_interactions_increment:
113  new_armetadata = armetadata.copy_template()
114  model_aux = new_armetadata['model_parameters']['spark']
115  model_aux['parameters']['maxIter']['value'] *= interactions_increment
116  model_list.append(new_armetadata)
117  except KeyError:
118  if model['parameters']['maxIter']['value'] \
119  <= max_interactions_increment:
120  new_armetadata = armetadata.copy_template()
121  model_aux = new_armetadata['model_parameters']['spark']
122  model_aux['parameters']['maxIter']['value'] *= interactions_increment
123  model_list.append(new_armetadata)
124 
125  new_armetadata = armetadata.copy_template()
126  model_aux = new_armetadata['model_parameters']['spark']
127  model_aux['parameters']['aggregationDepth']['value'] *= aggregationDepth_increment
128  model_list.append(new_armetadata)
129 
130  elif model['model'] == 'DecisionTreeClassifier' or model['model'] == 'DecisionTreeRegressor':
131 
132  if model['parameters']['minInstancesPerNode']['value'] > (min_rows_limit / 2):
133  new_armetadata = armetadata.copy_template()
134  model_aux = new_armetadata['model_parameters']['spark']
135  model_aux['parameters']['minInstancesPerNode']['value'] = round(
136  model_aux['parameters']['minInstancesPerNode']['value']
137  / min_rows_increment, 0)
138  model_list.append(new_armetadata)
139 
140  if scoring_metric['max_depth'][0] >= model['parameters']['maxDepth']['value']:
141  new_armetadata = armetadata.copy_template()
142  model_aux = new_armetadata['model_parameters']['spark']
143  model_aux['parameters']['maxDepth']['value'] = \
144  model_aux['parameters']['maxDepth']['value'] * max_depth_increment
145  model_list.append(new_armetadata)
146 
147  elif model['model'] == 'GBTRegressor':
148  if deepness == 2 and len(stepSize) != 0 and len(eval(model['parameters']['lossType']['type'])) != 0:
149  for stepsize in stepSize:
150  for element in eval(model['parameters']['lossType']['type']):
151  new_armetadata = armetadata.copy_template()
152  model_aux = new_armetadata['model_parameters']['spark']
153  model_aux['parameters']['lossType']['value'] = element
154  model_aux['parameters']['stepSize']['value'] = stepsize['learn']
155  model_list.append(new_armetadata)
156  elif deepness == 2 and len(stepSize) != 0:
157  for stepsize in stepSize:
158  new_armetadata = armetadata.copy_template()
159  model_aux = new_armetadata['model_parameters']['spark']
160  model_aux['parameters']['stepSize']['value'] = stepsize['learn']
161  model_list.append(new_armetadata)
162  elif deepness == 2 and len(eval(model['parameters']['lossType']['type'])) != 0:
163  for element in eval(model['parameters']['lossType']['type']):
164  new_armetadata = armetadata.copy_template()
165  model_aux = new_armetadata['model_parameters']['spark']
166  model_aux['parameters']['impurity']['value'] = element
167  model_list.append(new_armetadata)
168 
169  if model['parameters']['minInstancesPerNode']['value'] > (min_rows_limit / 2):
170  new_armetadata = armetadata.copy_template()
171  model_aux = new_armetadata['model_parameters']['spark']
172  model_aux['parameters']['minInstancesPerNode']['value'] = round(
173  model_aux['parameters']['minInstancesPerNode']['value']
174  / min_rows_increment, 0)
175  model_list.append(new_armetadata)
176 
177  # 05/07/2018. Included platform base restriction maxDepth <=30
178  if scoring_metric['max_depth'][0] >= model['parameters']['maxDepth']['value'] \
179  and model['parameters']['maxDepth']['value'] != 30:
180  new_armetadata = armetadata.copy_template()
181  model_aux = new_armetadata['model_parameters']['spark']
182  if model_aux['parameters']['maxDepth']['value'] * max_depth_increment > 30:
183  model_aux['parameters']['maxDepth']['value'] = 30
184  else:
185  model_aux['parameters']['maxDepth']['value'] *= max_depth_increment
186 
187  model_list.append(new_armetadata)
188 
189  if scoring_metric['trees'][0] >= model['parameters']['maxIter']['value']:
190  new_armetadata = armetadata.copy_template()
191  model_aux = new_armetadata['model_parameters']['spark']
192  model_aux['parameters']['maxIter']['value'] *= ntrees_increment
193  model_list.append(new_armetadata)
194 
195  elif model['model'] == 'GBTClassifier':
196  if deepness == 2 and len(stepSize) != 0 and len(eval(model['parameters']['lossType']['type'])) != 0:
197  for stepsize in stepSize:
198  for element in eval(model['parameters']['lossType']['type']):
199  new_armetadata = armetadata.copy_template()
200  model_aux = new_armetadata['model_parameters']['spark']
201  model_aux['parameters']['lossType']['value'] = element
202  model_aux['parameters']['stepSize']['value'] = stepsize['learn']
203  model_list.append(new_armetadata)
204  elif deepness == 2 and len(stepSize) != 0:
205  for stepsize in stepSize:
206  new_armetadata = armetadata.copy_template()
207  model_aux = new_armetadata['model_parameters']['spark']
208  model_aux['parameters']['stepSize']['value'] = stepsize['learn']
209  model_list.append(new_armetadata)
210  elif deepness == 2 and len(eval(model['parameters']['impurity']['type'])) != 0:
211  for element in eval(model['parameters']['lossType']['type']):
212  new_armetadata = armetadata.copy_template()
213  model_aux = new_armetadata['model_parameters']['spark']
214  model_aux['parameters']['lossType']['value'] = element
215  model_list.append(new_armetadata)
216 
217  if model['parameters']['minInstancesPerNode']['value'] > (min_rows_limit / 2):
218  new_armetadata = armetadata.copy_template()
219  model_aux = new_armetadata['model_parameters']['spark']
220  model_aux['parameters']['minInstancesPerNode']['value'] = round(
221  model_aux['parameters']['minInstancesPerNode']['value']
222  / min_rows_increment, 0)
223  model_list.append(new_armetadata)
224 
225  if scoring_metric['max_depth'][0] >= model['parameters']['maxDepth']['value']:
226  new_armetadata = armetadata.copy_template()
227  model_aux = new_armetadata['model_parameters']['spark']
228  model_aux['parameters']['maxDepth']['value'] *= max_depth_increment
229  model_list.append(new_armetadata)
230 
231  if scoring_metric['trees'][0] >= model['parameters']['maxIter']['value']:
232  new_armetadata = armetadata.copy_template()
233  model_aux = new_armetadata['model_parameters']['spark']
234  model_aux['parameters']['maxIter']['value'] *= ntrees_increment
235  model_list.append(new_armetadata)
236 
237  elif model['model'] == 'RandomForestClassifier' or model['model'] == 'RandomForestRegressor':
238 
239  if deepness == 2 and len(eval(model['parameters']['featureSubsetStrategy']['type'])) != 0 \
240  and len(eval(model['parameters']['impurity']['type'])) != 0:
241  for featuresubsetstrategy in eval(model['parameters']['featureSubsetStrategy']['type']):
242  for element in eval(model['parameters']['impurity']['type']):
243  new_armetadata = armetadata.copy_template()
244  model_aux = new_armetadata['model_parameters']['spark']
245  model_aux['parameters']['impurity']['value'] = element
246  model_aux['parameters']['featureSubsetStrategy']['value'] = featuresubsetstrategy
247  model_list.append(new_armetadata)
248  elif deepness == 2 and len(eval(model['parameters']['featureSubsetStrategy']['type'])) != 0:
249  for featuresubsetstrategy in eval(model['parameters']['featureSubsetStrategy']['type']):
250  new_armetadata = armetadata.copy_template()
251  model_aux = new_armetadata['model_parameters']['spark']
252  model_aux['parameters']['featureSubsetStrategy']['value'] = featuresubsetstrategy
253  model_list.append(new_armetadata)
254  elif deepness == 2 and len(eval(model['parameters']['impurity']['type'])) != 0:
255  for element in model['parameters']['impurity']['type']:
256  new_armetadata = armetadata.copy_template()
257  model_aux = new_armetadata['model_parameters']['spark']
258  model_aux['parameters']['impurity']['value'] = element
259  model_list.append(new_armetadata)
260 
261  if model['parameters']['minInstancesPerNode']['value'] > (min_rows_limit / 2):
262  new_armetadata = armetadata.copy_template()
263  model_aux = new_armetadata['model_parameters']['spark']
264  model_aux['parameters']['minInstancesPerNode']['value'] = round(
265  model_aux['parameters']['minInstancesPerNode']['value']
266  / min_rows_increment, 0)
267  model_list.append(new_armetadata)
268 
269  if scoring_metric['max_depth'][0] >= model['parameters']['maxDepth']['value']:
270  new_armetadata = armetadata.copy_template()
271  model_aux = new_armetadata['model_parameters']['spark']
272  model_aux['parameters']['maxDepth']['value'] *= max_depth_increment
273  model_list.append(new_armetadata)
274 
275  if scoring_metric['trees'][0] >= model['parameters']['numTrees']['value']:
276  new_armetadata = armetadata.copy_template()
277  model_aux = new_armetadata['model_parameters']['spark']
278  model_aux['parameters']['numTrees']['value'] *= ntrees_increment
279  model_list.append(new_armetadata)
280 
281  elif model['model'] == 'GeneralizedLinearRegression':
282  if deepness == 2 and len(regParam) != 0:
283  for elastic in regParam:
284  new_armetadata = armetadata.copy_template()
285  model_aux = new_armetadata['model_parameters']['spark']
286  model_aux['parameters']['regParam']['value'] = elastic['value']
287  model_list.append(new_armetadata)
288  if deepness == 2:
289  if model['parameters']['family']['value'] in ['gaussian', 'gamma']:
290  linklist = ['log', 'inverse']
291  elif model['parameters']['family']['value'] in ['poisson']:
292  linklist = ['log', 'sqrt']
293  elif model['parameters']['family']['value'] in ['poisson', 'tweedie']:
294  linklist = []
295  for linkin in linklist:
296  new_armetadata = armetadata.copy_template()
297  model_aux = new_armetadata['model_parameters']['spark']
298  model_aux['parameters']['link']['value'] = linkin
299  model_list.append(new_armetadata)
300 
301  if model['parameters']['maxIter']['value'] \
302  <= max_interactions_increment:
303  new_armetadata = armetadata.copy_template()
304  model_aux = new_armetadata['model_parameters']['spark']
305  model_aux['parameters']['maxIter']['value'] *= interactions_increment
306  model_list.append(new_armetadata)
307 
308  elif model['model'] == 'NaiveBayes':
309  if deepness == 2 and len(nv_smoothing) != 0:
310  for elastic in nv_smoothing:
311  new_armetadata = armetadata.copy_template()
312  model_aux = new_armetadata['model_parameters']['spark']
313  model_aux['parameters']['smoothing']['value'] = elastic['value']
314  model_list.append(new_armetadata)
315 
316  for adjusting in ['improvement', 'decrement']:
317  new_armetadata = armetadata.copy_template()
318  model_aux = new_armetadata['model_parameters']['spark']
319  if adjusting == 'improvement':
320  model_aux['parameters']['smoothing']['value'] = model_aux['parameters']['smoothing'][
321  'value'] * (1 + nv_improvement)
322  else:
323  model_aux['parameters']['smoothing']['value'] = model_aux['parameters']['smoothing'][
324  'value'] * (1 - nv_divisor)
325  model_list.append(new_armetadata)
326 
327  elif model['model'] == 'BisectingKMeans':
328 
329  new_armetadata = armetadata.copy_template()
330  model_aux = new_armetadata['model_parameters']['spark']
331  model_aux['parameters']['maxIter']['value'] = \
332  int(model_aux['parameters']['maxIter']['value'] * clustering_increment)
333  model_list.append(new_armetadata)
334 
335  elif model['model'] == 'KMeans':
336 
337  if deepness == 2 and len(eval(model['parameters']['initMode']['type'])) != 0:
338  for element in eval(model['parameters']['initMode']['type']):
339  new_armetadata = armetadata.copy_template()
340  model_aux = new_armetadata['model_parameters']['spark']
341  model_aux['parameters']['initMode']['value'] = element
342  model_list.append(new_armetadata)
343 
344  new_armetadata = armetadata.copy_template()
345  model_aux = new_armetadata['model_parameters']['spark']
346  model_aux['parameters']['maxIter']['value'] = \
347  int(model_aux['parameters']['maxIter']['value'] * clustering_increment)
348  model_list.append(new_armetadata)
349 
350  else:
351  return None
352 
353  if len(model_list) == 0:
354  return None
355  else:
356  return model_list
gdayf.conf.loadconfig.LoadConfig
Class Getting the config file place on default location and load all parameters on an internal variab...
Definition: loadconfig.py:22
gdayf.optimizers.spark_optimizer.Optimizer._config
_config
Definition: spark_optimizer.py:20
gdayf.common.utils
Define all objects, functions and structs related to common utilities not associated to one concrete ...
Definition: utils.py:1
gdayf.optimizers.spark_optimizer.Optimizer._labels
_labels
Definition: spark_optimizer.py:19
gdayf.optimizers.spark_optimizer.Optimizer
Definition: spark_optimizer.py:10
gdayf.optimizers.spark_optimizer.Optimizer.__init__
def __init__(self, e_c)
Constructor Initialize all framework variables and starts or connect to spark cluster Aditionally sta...
Definition: spark_optimizer.py:17
gdayf.optimizers.spark_optimizer.Optimizer.optimize_models
def optimize_models(self, armetadata, metric_value, objective, deepness, deep_impact)
Method manging generation of possible optimized H2O models loadded dinamically on adviserclass params...
Definition: spark_optimizer.py:30
gdayf.conf.loadconfig
Define all objects, functions and structs related to load on system all configuration parameter from ...
Definition: loadconfig.py:1
gdayf.optimizers.spark_optimizer.Optimizer._ec
_ec
Definition: spark_optimizer.py:18
Generated by   doxygen 1.8.13