| DayF core 1.2.1.2 DayF (Decision at your Fingertips) is an AutoML freeware development framework that let developers works with Machine Learning models without any idea of AI, simply taking a csv dataset and the objective column

storagemetadata.py
1
4
5 '''
6 Copyright (C) e2its - All Rights Reserved
7 * Unauthorized copying of this file, via any medium is strictly prohibited
8 * Proprietary and confidential
9 *
10 * This file is part of gDayF project.
11 *
12 * Written by Jose L. Sanchez <e2its.es@gmail.com>, 2016-2019
13 '''
14
15 from gdayf.common.utils import hash_key
16 from collections import OrderedDict
17 from os import path
18 from gdayf.common.utils import get_model_fw
19 from copy import deepcopy
20
21
22
24 class StorageMetadata (list):
25
27 def __init__(self, e_c):
28 self._ec = e_c
29 self._config = self._ec.config.get_config()
30 #self._labels = self._ec.labels.get_config()['messages']['controller']
31 list.__init__(self)
32
33
41 def append(self, value, fstype='localfs', hash_type='MD5'):
42 try:
43 assert fstype in ['localfs', 'hdfs', 'mongoDB']
44 assert hash_type in ['MD5', 'SHA256']
45
46 fs = OrderedDict()
47 fs['type'] = fstype
48 fs['value'] = value
49 fs['hash_type'] = hash_type
50
51 if fstype == 'localfs' and path.exists(value) and not path.isdir(value):
52 fs['hash_value'] = hash_key(hash_type=hash_type, filename=fs['value'])
53 else:
54 fs['hash_value'] = None
55 super(StorageMetadata, self).append(fs)
56 except:
57 super(StorageMetadata, self).append(value)
58
59
63 def get_load_path(self, include=False):
64 return self.exclude_debug_fs(deepcopy(self._config['storage']['load_path']), include=include)
65
66
69 def get_log_path(self):
70 return deepcopy(self._config['storage']['log_path'])
71
72
76 def get_json_path(self, include=False):
77 return self.exclude_debug_fs(deepcopy(self._config['storage']['json_path']), include=include)
78
79
83 def get_prediction_path(self, include=False):
84 return self.exclude_debug_fs(deepcopy(self._config['storage']['prediction_path']), include=include)
85
86
89 def exclude_debug_fs(self, storage_metadata, include=False):
90 equals = list()
91 if not self._config['storage']['localfs_debug_mode'] and not include:
92 for each_storage in storage_metadata:
93 if each_storage['type'] == 'localfs':
94 equals.append(each_storage)
95 for deleter in equals:
96 storage_metadata.remove(deleter)
97
98 return storage_metadata
99
100
104 def generate_json_path(e_c, armetadata, json_type='json'):
105 config = e_c.config.get_config()
106 fw = get_model_fw(armetadata)
107
108 model_id = armetadata['model_parameters'][fw]['parameters']['model_id']['value']
109 compress = config['persistence']['compress_json']
110 json_storage = StorageMetadata(e_c)
111
112 command = 'json_storage.get_' + json_type + '_path()'
113 for each_storage_type in eval(command):
114 if each_storage_type['type'] in ['localfs', 'hdfs']:
115 primary_path = config['storage'][each_storage_type['type']]['value']
116 source_data = list()
117 source_data.append(primary_path)
118 source_data.append('/')
119 source_data.append(armetadata['user_id'])
120 source_data.append('/')
121 source_data.append(armetadata['workflow_id'])
122 source_data.append('/')
123 source_data.append(armetadata['model_id'])
124 source_data.append('/')
125 source_data.append(fw)
126 source_data.append('/')
127 source_data.append(armetadata['type'])
128 source_data.append('/')
129 source_data.append(str(armetadata['timestamp']))
130 source_data.append('/')
131
132 specific_data = list()
133 specific_data.append(each_storage_type['value'])
134 specific_data.append('/')
135 specific_data.append(model_id)
136 specific_data.append('.json')
137 if compress:
138 specific_data.append('.gz')
139
140 json_path = ''.join(source_data)
141 json_path += ''.join(specific_data)
142 json_storage.append(value=json_path, fstype=each_storage_type['type'],
143 hash_type=each_storage_type['hash_type'])
144
145 else:
146 if json_type == 'json':
147 source_data = list()
148 source_data.append('/')
149 source_data.append(armetadata['user_id'])
150 source_data.append('/')
151 source_data.append(armetadata['workflow_id'])
152 source_data.append('/')
153 source_data.append(armetadata['model_id'])
154 source_data.append('/')
155 source_data.append(model_id)
156 json_path = ''.join(source_data)
157 json_storage.append(value=json_path, fstype=each_storage_type['type'],
158 hash_type=each_storage_type['hash_type'])
159 else:
160 json_storage.append(value=each_storage_type['value'], fstype=each_storage_type['type'],
161 hash_type=each_storage_type['hash_type'])
162
163 command = json_type + '_path'
164 armetadata[command] = json_storage
gdayf.common.storagemetadata.StorageMetadata._config
_config
Definition: storagemetadata.py:29
gdayf.common.storagemetadata.StorageMetadata.get_log_path
def get_log_path(self)
method used to get relative log path from config.json
Definition: storagemetadata.py:69
gdayf.common.storagemetadata.StorageMetadata._ec
_ec
Definition: storagemetadata.py:28
gdayf.common.utils
Define all objects, functions and structs related to common utilities not associated to one concrete ...
Definition: utils.py:1
gdayf.common.storagemetadata.generate_json_path
def generate_json_path(e_c, armetadata, json_type='json')
Method to Generate json StorageMetadata for Armetadata.
Definition: storagemetadata.py:104
gdayf.common.storagemetadata.StorageMetadata
Class storage metadata format [{value: , fstype:[&#39;localfs&#39;, &#39;hdfs&#39;, &#39;mongoDB&#39;], hash_value : ""...
Definition: storagemetadata.py:24
gdayf.common.storagemetadata.StorageMetadata.exclude_debug_fs
def exclude_debug_fs(self, storage_metadata, include=False)
method used to exclude localfs in non-debug modes
Definition: storagemetadata.py:89
gdayf.common.storagemetadata.StorageMetadata.get_prediction_path
def get_prediction_path(self, include=False)
method used to get relative prediction path from config.json
Definition: storagemetadata.py:83
gdayf.common.storagemetadata.StorageMetadata.append
def append(self, value, fstype='localfs', hash_type='MD5')
class used to add storage locations to StorageMetadata.
Definition: storagemetadata.py:41
gdayf.common.storagemetadata.StorageMetadata.__init__
def __init__(self, e_c)
Constructor return empty list of StoragMetadata.
Definition: storagemetadata.py:27
gdayf.common.storagemetadata.StorageMetadata.get_load_path
def get_load_path(self, include=False)
method used to get relative load path from config.json
Definition: storagemetadata.py:63
gdayf.common.storagemetadata.StorageMetadata.get_json_path
def get_json_path(self, include=False)
method used to get relative json path from config.json
Definition: storagemetadata.py:76