Coverage for plugin/tasks.py : 47%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# ============LICENSE_START==========================================
2# ===================================================================
3# Copyright (c) 2018-2020 AT&T
4# Copyright (c) 2020 Pantheon.tech. All rights reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17# ============LICENSE_END============================================
19import shutil
20import errno
21import sys
22import pwd
23import grp
24import os
25import re
26import getpass
27import subprocess
28import json
29import base64
30import yaml
31try:
32 from urllib.request import Request, urlopen
33except ImportError:
34 from urllib2 import Request, urlopen
36from cloudify import ctx
37from cloudify import exceptions
38from cloudify.decorators import operation
39from cloudify.exceptions import OperationRetry
40from cloudify.exceptions import NonRecoverableError
41from cloudify_rest_client.exceptions import CloudifyClientError
44def debug_log_mask_credentials(_command_str):
45 debug_str = _command_str
46 if _command_str.find("@") != -1:
47 head, end = _command_str.rsplit('@', 1)
48 proto, auth = head.rsplit('//', 1)
49 uname, passwd = auth.rsplit(':', 1)
50 debug_str = _command_str.replace(passwd, "************")
51 ctx.logger.debug('command {0}.'.format(debug_str))
53def execute_command(_command):
54 debug_log_mask_credentials(_command)
56 subprocess_args = {
57 'args': _command.split(),
58 'stdout': subprocess.PIPE,
59 'stderr': subprocess.PIPE
60 }
62 debug_log_mask_credentials(str(subprocess_args))
63 try:
64 process = subprocess.Popen(**subprocess_args)
65 output, error = process.communicate()
66 except Exception as e:
67 ctx.logger.debug(str(e))
68 return False
70 debug_log_mask_credentials(_command)
71 ctx.logger.debug('output: {0} '.format(output))
72 ctx.logger.debug('error: {0} '.format(error))
73 ctx.logger.debug('process.returncode: {0} '.format(process.returncode))
75 if process.returncode:
76 ctx.logger.error('Error was returned while running helm command')
77 return False
79 return output
82def configure_admin_conf():
83 # Add the kubeadmin config to environment
84 agent_user = getpass.getuser()
85 uid = pwd.getpwnam(agent_user).pw_uid
86 gid = grp.getgrnam('docker').gr_gid
87 admin_file_dest = os.path.join(os.path.expanduser('~'), 'admin.conf')
89 execute_command(
90 'sudo cp {0} {1}'.format('/etc/kubernetes/admin.conf',
91 admin_file_dest))
92 execute_command('sudo chown {0}:{1} {2}'.format(uid, gid, admin_file_dest))
94 with open(os.path.join(os.path.expanduser('~'), '.bashrc'),
95 'a') as outfile:
96 outfile.write('export KUBECONFIG=$HOME/admin.conf')
97 os.environ['KUBECONFIG'] = admin_file_dest
100def get_current_helm_value(chart_name):
101 tiller_host = str(ctx.node.properties['tiller_ip']) + ':' + str(
102 ctx.node.properties['tiller_port'])
103 config_dir_root = str(ctx.node.properties['config_dir'])
104 config_dir = config_dir_root + str(ctx.deployment.id) + '/'
105 if str_to_bool(ctx.node.properties['tls_enable']):
106 getValueCommand = subprocess.Popen(
107 ["helm", "get", "values", "-a", chart_name, '--host', tiller_host,
108 '--tls', '--tls-ca-cert', config_dir + 'ca.cert.pem',
109 '--tls-cert',
110 config_dir + 'helm.cert.pem', '--tls-key',
111 config_dir + 'helm.key.pem'], stdout=subprocess.PIPE)
112 else:
113 getValueCommand = subprocess.Popen(
114 ["helm", "get", "values", "-a", chart_name, '--host', tiller_host],
115 stdout=subprocess.PIPE)
116 value = getValueCommand.communicate()[0]
117 valueMap = {}
118 valueMap = yaml.safe_load(value)
119 ctx.instance.runtime_properties['current-helm-value'] = valueMap
122def get_helm_history(chart_name):
123 tiller_host = str(ctx.node.properties['tiller_ip']) + ':' + str(
124 ctx.node.properties['tiller_port'])
125 config_dir_root = str(ctx.node.properties['config_dir'])
126 config_dir = config_dir_root + str(ctx.deployment.id) + '/'
127 if str_to_bool(ctx.node.properties['tls_enable']):
128 getHistoryCommand = subprocess.Popen(
129 ["helm", "history", chart_name, '--host', tiller_host, '--tls',
130 '--tls-ca-cert', config_dir + 'ca.cert.pem', '--tls-cert',
131 config_dir + 'helm.cert.pem', '--tls-key',
132 config_dir + 'helm.key.pem'], stdout=subprocess.PIPE)
133 else:
134 getHistoryCommand = subprocess.Popen(
135 ["helm", "history", chart_name, '--host', tiller_host],
136 stdout=subprocess.PIPE)
137 history = getHistoryCommand.communicate()[0]
138 history_start_output = [line.strip() for line in history.split('\n') if
139 line.strip()]
140 for index in range(len(history_start_output)):
141 history_start_output[index] = history_start_output[index].replace('\t',
142 ' ')
143 ctx.instance.runtime_properties['helm-history'] = history_start_output
146def tls():
147 if str_to_bool(ctx.node.properties['tls_enable']):
148 config_dir_root = str(ctx.node.properties['config_dir'])
149 config_dir = config_dir_root + str(ctx.deployment.id) + '/'
150 tls_command = ' --tls --tls-ca-cert ' + config_dir + 'ca.cert.pem ' \
151 '--tls-cert ' + \
152 config_dir + 'helm.cert.pem --tls-key ' + config_dir + \
153 'helm.key.pem '
154 ctx.logger.debug(tls_command)
155 return tls_command
156 else:
157 return ''
160def tiller_host():
161 tiller_host = ' --host ' + str(
162 ctx.node.properties['tiller_ip']) + ':' + str(
163 ctx.node.properties['tiller_port']) + ' '
164 ctx.logger.debug(tiller_host)
165 return tiller_host
168def str_to_bool(s):
169 s = str(s)
170 if s == 'True' or s == 'true':
171 return True
172 elif s == 'False' or s == 'false':
173 return False
174 else:
175 raise ValueError('Require [Tt]rue or [Ff]alse; got: {0}'.format(s))
178def get_config_json(config_json, config_path, config_opt_f, config_file_nm):
179 config_obj = {}
180 config_obj = json.loads(config_json)
181 config_file = config_path + config_file_nm + ".yaml"
182 gen_config_file(config_file, config_obj)
183 config_opt_f = config_opt_f + " -f " + config_file
184 return config_opt_f
187def pop_config_info(url, config_file, f_format, repo_user, repo_user_passwd):
188 if url.find("@") != -1:
189 head, end = url.rsplit('@', 1)
190 head, auth = head.rsplit('//', 1)
191 url = head + '//' + end
192 username, password = auth.rsplit(':', 1)
193 request = Request(url)
194 base64string = base64.encodestring(
195 '%s:%s' % (username, password)).replace('\n', '')
196 request.add_header("Authorization", "Basic %s" % base64string)
197 response = urlopen(request)
198 elif repo_user != '' and repo_user_passwd != '':
199 request = Request(url)
200 base64string = base64.b64encode('%s:%s' % (repo_user, repo_user_passwd))
201 request.add_header("Authorization", "Basic %s" % base64string)
202 response = urlopen(request)
203 else:
204 response = urlopen(url)
206 config_obj = {}
207 if f_format == 'json':
208 config_obj = json.load(response)
209 elif f_format == 'yaml':
210 config_obj = yaml.load(response)
211 else:
212 raise NonRecoverableError("Unable to get config input format.")
214 gen_config_file(config_file, config_obj)
217def gen_config_file(config_file, config_obj):
218 try:
219 with open(config_file, 'w') as outfile:
220 yaml.safe_dump(config_obj, outfile, default_flow_style=False)
221 except OSError as e:
222 if e.errno != errno.EEXIST:
223 raise
226def gen_config_str(config_file, config_opt_f):
227 try:
228 with open(config_file, 'w') as outfile:
229 yaml.safe_dump(config_opt_f, outfile, default_flow_style=False)
230 except OSError as e:
231 if e.errno != errno.EEXIST:
232 raise
235def get_rem_config(config_url, config_input_format, config_path, config_opt_f, config_file_nm, repo_user, repo_user_passwd):
236 ctx.logger.debug("config_url=" + config_url)
237 f_cnt = 0
238 # urls = config_url.split()
239 urls = [x.strip() for x in config_url.split(',')]
240 if len(urls) > 1:
241 for url in urls:
242 f_cnt = f_cnt + 1
243 config_file = config_path + config_file_nm + str(f_cnt) + ".yaml"
244 pop_config_info(url, config_file, config_input_format, repo_user, repo_user_passwd)
245 config_opt_f = config_opt_f + " -f " + config_file
246 else:
247 config_file = config_path + config_file_nm + ".yaml"
248 pop_config_info(config_url, config_file, config_input_format, repo_user, repo_user_passwd)
249 config_opt_f = config_opt_f + " -f " + config_file
251 return config_opt_f
254def get_config_str(config_file):
255 if os.path.isfile(config_file):
256 with open(config_file, 'r') as config_f:
257 return config_f.read().replace('\n', '')
258 return ''
261def opt(config_file):
262 opt_str = get_config_str(config_file)
263 if opt_str != '':
264 return opt_str.replace("'", "")
265 return opt_str
267def repo(repo_url, repo_user, repo_user_passwd):
268 if repo_user != '' and repo_user_passwd != '' and repo_url.find("@") == -1:
269 proto, ip = repo_url.rsplit('//', 1)
270 return proto + '//' + repo_user + ':' + repo_user_passwd + '@' + ip
271 else:
272 return repo_url
275@operation
276def config(**kwargs):
277 # create helm value file on K8s master
278 configJson = str(ctx.node.properties['config'])
279 configUrl = str(ctx.node.properties['config_url'])
280 configUrlInputFormat = str(ctx.node.properties['config_format'])
281 runtime_config = str(ctx.node.properties['runtime_config']) # json
282 componentName = ctx.node.properties['component_name']
283 config_dir_root = str(ctx.node.properties['config_dir'])
284 stable_repo_url = str(ctx.node.properties['stable_repo_url'])
285 config_opt_set = str(ctx.node.properties['config_set'])
286 repo_user = str(ctx.node.properties['repo_user'])
287 repo_user_passwd = str(ctx.node.properties['repo_user_password'])
288 ctx.logger.debug("debug " + configJson + runtime_config)
289 # load input config
290 config_dir = config_dir_root + str(ctx.deployment.id)
292 if not os.path.exists(config_dir):
293 try:
294 os.makedirs(config_dir)
295 except OSError as e:
296 if e.errno != errno.EEXIST:
297 raise
299 ctx.logger.debug('tls-enable type ' + str(
300 type(str_to_bool(ctx.node.properties['tls_enable']))))
302 # create TLS cert files
303 if str_to_bool(ctx.node.properties['tls_enable']):
304 ctx.logger.debug('tls enable')
305 ca_value = ctx.node.properties['ca']
306 cert_value = ctx.node.properties['cert']
307 key_value = ctx.node.properties['key']
308 ca = open(config_dir + '/ca.cert.pem', "w+")
309 ca.write(ca_value)
310 ca.close()
311 cert = open(config_dir + '/helm.cert.pem', "w+")
312 cert.write(cert_value)
313 cert.close()
314 key = open(config_dir + '/helm.key.pem', "w+")
315 key.write(key_value)
316 key.close()
317 else:
318 ctx.logger.debug('tls disable')
320 config_path = config_dir + '/' + componentName + '/'
321 ctx.logger.debug(config_path)
323 if os.path.exists(config_path):
324 shutil.rmtree(config_path)
326 try:
327 os.makedirs(config_path)
328 except OSError as e:
329 if e.errno != errno.EEXIST:
330 raise
332 config_opt_f = ""
333 if configJson == '' and configUrl == '':
334 ctx.logger.debug("Will use default HELM value")
335 elif configJson == '' and configUrl != '':
336 config_opt_f = get_rem_config(configUrl, configUrlInputFormat, config_path, config_opt_f, "rc", repo_user, repo_user_passwd)
337 elif configJson != '' and configUrl == '':
338 config_opt_f = get_config_json(configJson, config_path, config_opt_f, "lc")
339 else:
340 raise NonRecoverableError("Unable to get config input")
342 ctx.logger.debug("debug check runtime config")
343 if runtime_config == '':
344 ctx.logger.debug("there is no runtime config value")
345 else:
346 config_opt_f = get_config_json(runtime_config, config_path, config_opt_f, "rt")
348 if configUrl != '' or configJson != '' or runtime_config != '':
349 config_file = config_path + ".config_file"
350 gen_config_str(config_file, config_opt_f)
352 if config_opt_set != '':
353 config_file = config_path + ".config_set"
354 config_opt_set = " --set " + config_opt_set
355 gen_config_str(config_file, config_opt_set)
357 output = execute_command(
358 'helm init --client-only --stable-repo-url ' + repo(stable_repo_url, repo_user, repo_user_passwd))
359 if output == False:
360 raise NonRecoverableError("helm init failed")
363@operation
364def start(**kwargs):
365 # install the ONAP Helm chart
366 # get properties from node
367 repo_user = str(ctx.node.properties['repo_user'])
368 repo_user_passwd = str(ctx.node.properties['repo_user_password'])
369 chartRepo = ctx.node.properties['chart_repo_url']
370 componentName = ctx.node.properties['component_name']
371 chartVersion = str(ctx.node.properties['chart_version'])
372 config_dir_root = str(ctx.node.properties['config_dir'])
373 namespace = ctx.node.properties['namespace']
375 config_path = config_dir_root + str(
376 ctx.deployment.id) + '/' + componentName + '/'
377 chart = chartRepo + "/" + componentName + "-" + str(chartVersion) + ".tgz"
378 chartName = namespace + "-" + componentName
379 config_file = config_path + ".config_file"
380 config_set = config_path + ".config_set"
381 installCommand = 'helm install ' + repo(chart, repo_user, repo_user_passwd) + ' --name ' + chartName + \
382 ' --namespace ' + namespace + opt(config_file) + \
383 opt(config_set) + tiller_host() + tls()
385 output = execute_command(installCommand)
386 if output == False:
387 return ctx.operation.retry(
388 message='helm install failed, re-try after 5 second ',
389 retry_after=5)
391 get_current_helm_value(chartName)
392 get_helm_history(chartName)
395@operation
396def stop(**kwargs):
397 # delete the ONAP helm chart
398 # configure_admin_conf()
399 # get properties from node
400 namespace = ctx.node.properties['namespace']
401 component = ctx.node.properties['component_name']
402 chartName = namespace + "-" + component
403 config_dir_root = str(ctx.node.properties['config_dir'])
404 # Delete helm chart
405 command = 'helm delete --purge ' + chartName + tiller_host() + tls()
406 output = execute_command(command)
407 if output == False:
408 raise NonRecoverableError("helm delete failed")
409 config_path = config_dir_root + str(
410 ctx.deployment.id) + '/' + component
412 if os.path.exists(config_path):
413 shutil.rmtree(config_path)
416@operation
417def upgrade(**kwargs):
418 config_dir_root = str(ctx.node.properties['config_dir'])
419 componentName = ctx.node.properties['component_name']
420 namespace = ctx.node.properties['namespace']
421 repo_user = kwargs['repo_user']
422 repo_user_passwd = kwargs['repo_user_passwd']
423 configJson = kwargs['config']
424 chartRepo = kwargs['chart_repo']
425 chartVersion = kwargs['chart_version']
426 config_set = kwargs['config_set']
427 config_json = kwargs['config_json']
428 config_url = kwargs['config_url']
429 config_format = kwargs['config_format']
430 config_path = config_dir_root + str(
431 ctx.deployment.id) + '/' + componentName + '/'
433 # ctx.logger.debug('debug ' + str(configJson))
434 chartName = namespace + "-" + componentName
435 chart = chartRepo + "/" + componentName + "-" + chartVersion + ".tgz"
437 config_opt_f = ""
438 if config_json == '' and config_url == '':
439 ctx.logger.debug("Will use default HELM values")
440 elif config_json == '' and config_url != '':
441 config_opt_f = get_rem_config(config_url, config_format, config_path, config_opt_f, "ru", repo_user, repo_user_passwd)
442 elif config_json != '' and config_url == '':
443 config_opt_f = get_config_json(config_json, config_path, config_opt_f, "lu")
444 else:
445 raise NonRecoverableError("Unable to get upgrade config input")
447 config_upd = ""
448 if config_url != '' or config_json != '':
449 config_upd = config_path + ".config_upd"
450 gen_config_str(config_upd, config_opt_f)
452 config_upd_set = ""
453 if config_set != '':
454 config_upd_set = config_path + ".config_upd_set"
455 config_opt_set = " --set " + config_set
456 gen_config_str(config_upd_set, config_opt_set)
458 upgradeCommand = 'helm upgrade ' + chartName + ' ' + repo(chart, repo_user, repo_user_passwd) + opt(config_upd) + \
459 opt(config_upd_set) + tiller_host() + tls()
461 output = execute_command(upgradeCommand)
462 if output == False:
463 return ctx.operation.retry(
464 message='helm upgrade failed, re-try after 5 second ',
465 retry_after=5)
466 get_current_helm_value(chartName)
467 get_helm_history(chartName)
470@operation
471def rollback(**kwargs):
472 # rollback to some revision
473 componentName = ctx.node.properties['component_name']
474 namespace = ctx.node.properties['namespace']
475 revision = kwargs['revision']
476 # configure_admin_conf()
477 chartName = namespace + "-" + componentName
478 rollbackCommand = 'helm rollback ' + chartName + ' ' + revision + tiller_host() + tls()
479 output = execute_command(rollbackCommand)
480 if output == False:
481 return ctx.operation.retry(
482 message='helm rollback failed, re-try after 5 second ',
483 retry_after=5)
484 get_current_helm_value(chartName)
485 get_helm_history(chartName)
487@operation
488def status(**kwargs):
489 componentName = ctx.node.properties['component_name']
490 namespace = ctx.node.properties['namespace']
492 chartName = namespace + "-" + componentName
493 statusCommand = 'helm status ' + chartName + tiller_host() + tls()
494 output = execute_command(statusCommand)
495 if output == False:
496 return ctx.operation.retry(
497 message='helm status failed, re-try after 5 second ',
498 retry_after=5)
500 status_output = [line.strip() for line in output.split('\n') if
501 line.strip()]
502 for index in range(len(status_output)):
503 status_output[index] = status_output[index].replace('\t', ' ')
504 ctx.instance.runtime_properties['install-status'] = status_output