3372
+ − 1 #!/usr/bin/env python3
+ − 2
+ − 3 # SàT plugin to manage external applications
+ − 4 # Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org)
+ − 5
+ − 6 # This program is free software: you can redistribute it and/or modify
+ − 7 # it under the terms of the GNU Affero General Public License as published by
+ − 8 # the Free Software Foundation, either version 3 of the License, or
+ − 9 # (at your option) any later version.
+ − 10
+ − 11 # This program is distributed in the hope that it will be useful,
+ − 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
+ − 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ − 14 # GNU Affero General Public License for more details.
+ − 15
+ − 16 # You should have received a copy of the GNU Affero General Public License
+ − 17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
+ − 18
+ − 19 from pathlib import Path
+ − 20 from typing import Optional , List
+ − 21 from functools import partial , reduce
+ − 22 import tempfile
+ − 23 import secrets
+ − 24 import string
+ − 25 import shortuuid
+ − 26 from twisted.internet import defer
+ − 27 from twisted.python.procutils import which
+ − 28 from sat.core.i18n import _
+ − 29 from sat.core import exceptions
+ − 30 from sat.core.constants import Const as C
+ − 31 from sat.core.log import getLogger
+ − 32 from sat.tools.common import data_format
+ − 33 from sat.tools.common import async_process
+ − 34
+ − 35 log = getLogger ( __name__ )
+ − 36
+ − 37 try :
+ − 38 import yaml
+ − 39 except ImportError :
+ − 40 raise exceptions . MissingModule (
+ − 41 'Missing module PyYAML, please download/install it. You can use '
+ − 42 '"pip install pyyaml"'
+ − 43 )
+ − 44
+ − 45 try :
+ − 46 from yaml import CLoader as Loader , CDumper as Dumper
+ − 47 except ImportError :
+ − 48 log . warning (
+ − 49 "Can't use LibYAML binding (is libyaml installed?), pure Python version will be "
+ − 50 "used, but it is slower"
+ − 51 )
+ − 52 from yaml import Loader , Dumper
+ − 53
+ − 54 from yaml.constructor import ConstructorError
+ − 55
+ − 56
+ − 57 PLUGIN_INFO = {
+ − 58 C . PI_NAME : "Applications Manager" ,
+ − 59 C . PI_IMPORT_NAME : "APP_MANAGER" ,
+ − 60 C . PI_TYPE : C . PLUG_TYPE_MISC ,
+ − 61 C . PI_MODES : C . PLUG_MODE_BOTH ,
+ − 62 C . PI_MAIN : "AppManager" ,
+ − 63 C . PI_HANDLER : "no" ,
+ − 64 C . PI_DESCRIPTION : _ (
+ − 65 """Applications Manager
+ − 66
+ − 67 Manage external applications using packagers, OS virtualization/containers or other
+ − 68 software management tools.
+ − 69 """ ),
+ − 70 }
+ − 71
+ − 72 APP_FILE_PREFIX = "sat_app_"
+ − 73
+ − 74
+ − 75 class AppManager :
+ − 76 load = partial ( yaml . load , Loader = Loader )
+ − 77 dump = partial ( yaml . dump , Dumper = Dumper )
+ − 78
+ − 79 def __init__ ( self , host ):
+ − 80 log . info ( _ ( "plugin Applications Manager initialization" ))
+ − 81 self . host = host
+ − 82 self . _managers = {}
+ − 83 self . _apps = {}
+ − 84 self . _started = {}
+ − 85 # instance id to app data map
+ − 86 self . _instances = {}
+ − 87 host . bridge . addMethod (
+ − 88 "applicationsList" ,
+ − 89 ".plugin" ,
+ − 90 in_sign = "as" ,
+ − 91 out_sign = "as" ,
+ − 92 method = self . list_applications ,
+ − 93 )
+ − 94 host . bridge . addMethod (
+ − 95 "applicationStart" ,
+ − 96 ".plugin" ,
+ − 97 in_sign = "ss" ,
+ − 98 out_sign = "" ,
+ − 99 method = self . _start ,
+ − 100 async_ = True ,
+ − 101 )
+ − 102 host . bridge . addMethod (
+ − 103 "applicationStop" ,
+ − 104 ".plugin" ,
+ − 105 in_sign = "sss" ,
+ − 106 out_sign = "" ,
+ − 107 method = self . _stop ,
+ − 108 async_ = True ,
+ − 109 )
+ − 110 host . bridge . addMethod (
+ − 111 "applicationExposedGet" ,
+ − 112 ".plugin" ,
+ − 113 in_sign = "sss" ,
+ − 114 out_sign = "s" ,
+ − 115 method = self . _getExposed ,
+ − 116 async_ = True ,
+ − 117 )
+ − 118 yaml . add_constructor (
+ − 119 "!sat_conf" , self . _sat_conf_constr , Loader = Loader )
+ − 120 yaml . add_constructor (
+ − 121 "!sat_generate_pwd" , self . _sat_generate_pwd_constr , Loader = Loader )
+ − 122 yaml . add_constructor (
+ − 123 "!sat_param" , self . _sat_param_constr , Loader = Loader )
+ − 124
+ − 125 def unload ( self ):
+ − 126 log . debug ( "unloading applications manager" )
+ − 127 for instances in self . _started . values ():
+ − 128 for instance in instances :
+ − 129 data = instance [ 'data' ]
+ − 130 if not data [ 'single_instance' ]:
+ − 131 log . debug (
+ − 132 f "cleaning temporary directory at { data [ '_instance_dir_path' ] } " )
+ − 133 data [ '_instance_dir_obj' ] . cleanup ()
+ − 134
+ − 135 def _sat_conf_constr ( self , loader , node ):
+ − 136 """Get a value from SàT configuration
+ − 137
+ − 138 A list is expected with either "name" of a config parameter, a one or more of
+ − 139 those parameters:
+ − 140 - section
+ − 141 - name
+ − 142 - default value
+ − 143 - filter
+ − 144 filter can be:
+ − 145 - "first": get the first item of the value
+ − 146 """
+ − 147 config_data = loader . construct_sequence ( node )
+ − 148 if len ( config_data ) == 1 :
+ − 149 section , name , default , filter_ = "" , config_data [ 0 ], None , None
+ − 150 if len ( config_data ) == 2 :
+ − 151 ( section , name ), default , filter_ = config_data , None , None
+ − 152 elif len ( config_data ) == 3 :
+ − 153 ( section , name , default ), filter_ = config_data , None
+ − 154 elif len ( config_data ) == 4 :
+ − 155 section , name , default , filter_ = config_data
+ − 156 else :
+ − 157 raise ValueError (
+ − 158 f "invalid !sat_conf value ( { config_data !r} ), a list of 1 to 4 items is "
+ − 159 "expected"
+ − 160 )
+ − 161
+ − 162 value = self . host . memory . getConfig ( section , name , default )
+ − 163 if name == "public_url" and not value or not value . startswith ( 'http' ):
+ − 164 if not value :
+ − 165 log . warning ( _ (
+ − 166 'No value found for "public_url", using "https://example.org" for '
+ − 167 'now, please set the proper value in sat.conf' ))
+ − 168 else :
+ − 169 log . warning ( _ (
+ − 170 'invalid value for "public_url" ( {value} ), it must start with '
+ − 171 '"http", ignoring it and using "https://example.org" instead' )
+ − 172 . format ( value = value ))
+ − 173 value = "https://example.org"
+ − 174
+ − 175 if filter_ is None :
+ − 176 pass
+ − 177 elif filter_ == 'first' :
+ − 178 value = value [ 0 ]
+ − 179 else :
+ − 180 raise ValueError ( f "unmanaged filter: { filter_ } " )
+ − 181
+ − 182 return value
+ − 183
+ − 184 def _sat_generate_pwd_constr ( self , loader , node ):
+ − 185 alphabet = string . ascii_letters + string . digits
+ − 186 return '' . join ( secrets . choice ( alphabet ) for i in range ( 30 ))
+ − 187
+ − 188 def _sat_param_constr ( self , loader , node ):
+ − 189 """Get a parameter specified when starting the application
+ − 190
+ − 191 The value can be either the name of the parameter to get, or a list as
+ − 192 [name, default_value]
+ − 193 """
+ − 194 try :
+ − 195 name , default = loader . construct_sequence ( node )
+ − 196 except ConstructorError :
+ − 197 name , default = loader . construct_scalar ( node ), None
+ − 198 return self . _params . get ( name , default )
+ − 199
+ − 200 def register ( self , manager ):
+ − 201 name = manager . name
+ − 202 if name in self . _managers :
+ − 203 raise exceptions . ConflictError (
+ − 204 f "There is already a manager with the name { name } " )
+ − 205 self . _managers [ manager . name ] = manager
+ − 206 if hasattr ( manager , "discover_path" ):
+ − 207 self . discover ( manager . discover_path , manager )
+ − 208
+ − 209 def getManager ( self , app_data : dict ) -> object :
+ − 210 """Get manager instance needed for this app
+ − 211
+ − 212 @raise exceptions.DataError: something is wrong with the type
+ − 213 @raise exceptions.NotFound: manager is not registered
+ − 214 """
+ − 215 try :
+ − 216 app_type = app_data [ "type" ]
+ − 217 except KeyError :
+ − 218 raise exceptions . DataError (
+ − 219 "app file doesn't have the mandatory \" type \" key"
+ − 220 )
+ − 221 if not isinstance ( app_type , str ):
+ − 222 raise exceptions . DataError (
+ − 223 f "invalid app data type: { app_type !r} "
+ − 224 )
+ − 225 app_type = app_type . strip ()
+ − 226 try :
+ − 227 return self . _managers [ app_type ]
+ − 228 except KeyError :
+ − 229 raise exceptions . NotFound (
+ − 230 f "No manager found to manage app of type { app_type !r} " )
+ − 231
+ − 232 def getAppData (
+ − 233 self ,
+ − 234 id_type : Optional [ str ],
+ − 235 identifier : str
+ − 236 ) -> dict :
+ − 237 """Retrieve instance's app_data from identifier
+ − 238
+ − 239 @param id_type: type of the identifier, can be:
+ − 240 - "name": identifier is a canonical application name
+ − 241 the first found instance of this application is returned
+ − 242 - "instance": identifier is an instance id
+ − 243 @param identifier: identifier according to id_type
+ − 244 @return: instance application data
+ − 245 @raise exceptions.NotFound: no instance with this id can be found
+ − 246 @raise ValueError: id_type is invalid
+ − 247 """
+ − 248 if not id_type :
+ − 249 id_type = 'name'
+ − 250 if id_type == 'name' :
+ − 251 identifier = identifier . lower () . strip ()
+ − 252 try :
+ − 253 return next ( iter ( self . _started [ identifier ]))
+ − 254 except ( KeyError , StopIteration ):
+ − 255 raise exceptions . NotFound (
+ − 256 f "No instance of { identifier !r} is currently running"
+ − 257 )
+ − 258 elif id_type == 'instance' :
+ − 259 instance_id = identifier
+ − 260 try :
+ − 261 return self . _instances [ instance_id ]
+ − 262 except KeyError :
+ − 263 raise exceptions . NotFound (
+ − 264 f "There is no application instance running with id { instance_id !r} "
+ − 265 )
+ − 266 else :
+ − 267 raise ValueError ( f "invalid id_type: { id_type !r} " )
+ − 268
+ − 269 def discover (
+ − 270 self ,
+ − 271 dir_path : Path ,
+ − 272 manager : Optional = None
+ − 273 ) -> None :
+ − 274 for file_path in dir_path . glob ( f " { APP_FILE_PREFIX } *.yaml" ):
+ − 275 if manager is None :
+ − 276 try :
+ − 277 app_data = self . parse ( file_path )
+ − 278 manager = self . getManager ( app_data )
+ − 279 except ( exceptions . DataError , exceptions . NotFound ) as e :
+ − 280 log . warning (
+ − 281 f "Can't parse { file_path } , skipping: { e } " )
+ − 282 app_name = file_path . stem [ len ( APP_FILE_PREFIX ):] . strip () . lower ()
+ − 283 if not app_name :
+ − 284 log . warning (
+ − 285 f "invalid app file name at { file_path } " )
+ − 286 continue
+ − 287 app_dict = self . _apps . setdefault ( app_name , {})
+ − 288 manager_set = app_dict . setdefault ( manager , set ())
+ − 289 manager_set . add ( file_path )
+ − 290 log . debug (
+ − 291 f " { app_name !r} { manager . name } application found"
+ − 292 )
+ − 293
+ − 294 def parse ( self , file_path : Path , params : Optional [ dict ] = None ) -> dict :
+ − 295 """Parse SàT application file
+ − 296
+ − 297 @param params: parameters for running this instance
+ − 298 @raise exceptions.DataError: something is wrong in the file
+ − 299 """
+ − 300 if params is None :
+ − 301 params = {}
+ − 302 with file_path . open () as f :
+ − 303 # we set parameters to be used only with this instance
+ − 304 # no async method must used between this assignation and `load`
+ − 305 self . _params = params
+ − 306 app_data = self . load ( f )
+ − 307 self . _params = None
+ − 308 if "name" not in app_data :
+ − 309 # note that we don't use lower() here as we want human readable name and
+ − 310 # uppercase may be set on purpose
+ − 311 app_data [ 'name' ] = file_path . stem [ len ( APP_FILE_PREFIX ):] . strip ()
+ − 312 single_instance = app_data . setdefault ( "single_instance" , True )
+ − 313 if not isinstance ( single_instance , bool ):
+ − 314 raise ValueError (
+ − 315 f '"single_instance" must be a boolean, but it is { type ( single_instance ) } '
+ − 316 )
+ − 317 return app_data
+ − 318
+ − 319 def list_applications ( self , filters : Optional [ List [ str ]]) -> List [ str ]:
+ − 320 """List available application
+ − 321
+ − 322 @param filters: only show applications matching those filters.
+ − 323 using None will list all known applications
+ − 324 a filter can be:
+ − 325 - available: applications available locally
+ − 326 - running: only show launched applications
+ − 327 """
+ − 328 if not filters :
+ − 329 return list ( self . apps )
+ − 330 found = set ()
+ − 331 for filter_ in filters :
+ − 332 if filter_ == "available" :
+ − 333 found . update ( self . _apps )
+ − 334 elif filter_ == "running" :
+ − 335 found . update ( self . _started )
+ − 336 else :
+ − 337 raise ValueError ( f "Unknown filter: { filter_ } " )
+ − 338 return list ( found )
+ − 339
+ − 340 def _start ( self , app_name , extra ):
+ − 341 extra = data_format . deserialise ( extra )
+ − 342 return defer . ensureDeferred ( self . start ( str ( app_name ), extra ))
+ − 343
+ − 344 async def start (
+ − 345 self ,
+ − 346 app_name : str ,
+ − 347 extra : Optional [ dict ] = None ,
+ − 348 ) -> None :
+ − 349 # FIXME: for now we use the first app manager available for the requested app_name
+ − 350 # TODO: implement running multiple instance of the same app if some metadata
+ − 351 # to be defined in app_data allows explicitly it.
+ − 352 app_name = app_name . lower () . strip ()
+ − 353 try :
+ − 354 app_file_path = next ( iter ( next ( iter ( self . _apps [ app_name ] . values ()))))
+ − 355 except KeyError :
+ − 356 raise exceptions . NotFound (
+ − 357 f "No application found with the name { app_name !r} "
+ − 358 )
+ − 359 started_data = self . _started . setdefault ( app_name , [])
+ − 360 app_data = self . parse ( app_file_path , extra )
+ − 361 app_data [ '_file_path' ] = app_file_path
+ − 362 app_data [ '_name_canonical' ] = app_name
+ − 363 single_instance = app_data [ 'single_instance' ]
+ − 364 if single_instance :
+ − 365 if started_data :
+ − 366 log . info ( f " { app_name !r} is already started" )
+ − 367 return
+ − 368 else :
+ − 369 cache_path = self . host . memory . getCachePath (
+ − 370 PLUGIN_INFO [ C . PI_IMPORT_NAME ], app_name
+ − 371 )
+ − 372 cache_path . mkdir ( 0o700 , parents = True , exist_ok = True )
+ − 373 app_data [ '_instance_dir_path' ] = cache_path
+ − 374 else :
+ − 375 dest_dir_obj = tempfile . TemporaryDirectory ( prefix = "sat_app_" )
+ − 376 app_data [ '_instance_dir_obj' ] = dest_dir_obj
+ − 377 app_data [ '_instance_dir_path' ] = Path ( dest_dir_obj . name )
+ − 378 instance_id = app_data [ '_instance_id' ] = shortuuid . uuid ()
+ − 379 manager = self . getManager ( app_data )
+ − 380 app_data [ '_manager' ] = manager
+ − 381 started_data . append ( app_data )
+ − 382 self . _instances [ instance_id ] = app_data
+ − 383
+ − 384 try :
+ − 385 start = manager . start
+ − 386 except AttributeError :
+ − 387 raise exceptions . InternalError (
+ − 388 f " { manager . name } doesn't have the mandatory \" start \" method"
+ − 389 )
+ − 390 else :
+ − 391 await start ( app_data )
+ − 392 log . info ( f " { app_name !r} started" )
+ − 393
+ − 394 def _stop ( self , identifier , id_type , extra ):
+ − 395 extra = data_format . deserialise ( extra )
+ − 396 return defer . ensureDeferred (
+ − 397 self . stop ( str ( identifier ), str ( id_type ) or None , extra ))
+ − 398
+ − 399 async def stop (
+ − 400 self ,
+ − 401 identifier : str ,
+ − 402 id_type : Optional [ str ] = None ,
+ − 403 extra : Optional [ dict ] = None ,
+ − 404 ) -> None :
+ − 405 if extra is None :
+ − 406 extra = {}
+ − 407
+ − 408 app_data = self . getAppData ( id_type , identifier )
+ − 409
+ − 410 log . info ( f "stopping { app_data [ 'name' ] !r} " )
+ − 411
+ − 412 app_name = app_data [ '_name_canonical' ]
+ − 413 instance_id = app_data [ '_instance_id' ]
+ − 414 manager = app_data [ '_manager' ]
+ − 415
+ − 416 try :
+ − 417 stop = manager . stop
+ − 418 except AttributeError :
+ − 419 raise exceptions . InternalError (
+ − 420 f " { manager . name } doesn't have the mandatory \" stop \" method"
+ − 421 )
+ − 422 else :
+ − 423 try :
+ − 424 await stop ( app_data )
+ − 425 except Exception as e :
+ − 426 log . warning (
+ − 427 f "Instance { instance_id } of application { app_name } can't be stopped "
+ − 428 f "properly: { e } "
+ − 429 )
+ − 430 return
+ − 431
+ − 432 try :
+ − 433 del self . _instances [ instance_id ]
+ − 434 except KeyError :
+ − 435 log . error (
+ − 436 f "INTERNAL ERROR: { instance_id !r} is not present in self._instances" )
+ − 437
+ − 438 try :
+ − 439 self . _started [ app_name ] . remove ( app_data )
+ − 440 except ValueError :
+ − 441 log . error (
+ − 442 "INTERNAL ERROR: there is no app data in self._started with id "
+ − 443 f " { instance_id !r} "
+ − 444 )
+ − 445
+ − 446 log . info ( f " { app_name !r} stopped" )
+ − 447
+ − 448 def _getExposed ( self , identifier , id_type , extra ):
+ − 449 extra = data_format . deserialise ( extra )
+ − 450 d = defer . ensureDeferred ( self . getExposed ( identifier , id_type , extra ))
+ − 451 d . addCallback ( lambda d : data_format . serialise ( d ))
+ − 452 return d
+ − 453
+ − 454 def getValueFromPath ( self , app_data : dict , path : List [ str ]) -> any :
+ − 455 """Retrieve a value set in the data from it path
+ − 456
+ − 457 @param path: list of key to use in app_data to retrieve the value
+ − 458 @return: found value
+ − 459 @raise NotFound: the value can't be found
+ − 460 """
+ − 461
+ − 462 async def getExposed (
+ − 463 self ,
+ − 464 identifier : str ,
+ − 465 id_type : Optional [ str ] = None ,
+ − 466 extra : Optional [ dict ] = None ,
+ − 467 ) -> dict :
+ − 468 """Get data exposed by the application
+ − 469
+ − 470 The manager's "computeExpose" method will be called if it exists. It can be used
+ − 471 to handle manager specific conventions.
+ − 472 """
+ − 473 app_data = self . getAppData ( id_type , identifier )
+ − 474 if app_data . get ( '_exposed_computed' , False ):
+ − 475 return app_data [ 'expose' ]
+ − 476 if extra is None :
+ − 477 extra = {}
+ − 478 expose = app_data . setdefault ( "expose" , {})
+ − 479 if "passwords" in expose :
+ − 480 passwords = expose [ 'passwords' ]
+ − 481 for name , value in list ( passwords . items ()):
+ − 482 if isinstance ( value , list ):
+ − 483 # if we have a list, is the sequence of keys leading to the value
+ − 484 # to expose. We use "reduce" to retrieve the desired value
+ − 485 try :
+ − 486 passwords [ name ] = reduce ( lambda l , k : l [ k ], value , app_data )
+ − 487 except Exception as e :
+ − 488 log . warning (
+ − 489 f "Can't retrieve exposed value for password { name !r} : { e } " )
+ − 490 del passwords [ name ]
+ − 491
+ − 492 url_prefix = expose . get ( "url_prefix" )
+ − 493 if isinstance ( url_prefix , list ):
+ − 494 try :
+ − 495 expose [ "url_prefix" ] = reduce ( lambda l , k : l [ k ], url_prefix , app_data )
+ − 496 except Exception as e :
+ − 497 log . warning (
+ − 498 f "Can't retrieve exposed value for url_prefix: { e } " )
+ − 499 del expose [ "url_prefix" ]
+ − 500
+ − 501 try :
+ − 502 computeExpose = app_data [ '_manager' ] . computeExpose
+ − 503 except AttributeError :
+ − 504 pass
+ − 505 else :
+ − 506 await computeExpose ( app_data )
+ − 507
+ − 508 app_data [ '_exposed_computed' ] = True
+ − 509 return expose
+ − 510
+ − 511 async def _doPrepare (
+ − 512 self ,
+ − 513 app_data : dict ,
+ − 514 ) -> None :
+ − 515 name = app_data [ 'name' ]
+ − 516 dest_path = app_data [ '_instance_dir_path' ]
+ − 517 if next ( dest_path . iterdir (), None ) != None :
+ − 518 log . debug ( f "There is already a prepared dir at { dest_path } , nothing to do" )
+ − 519 return
+ − 520 try :
+ − 521 prepare = app_data [ 'prepare' ] . copy ()
+ − 522 except KeyError :
+ − 523 prepare = {}
+ − 524
+ − 525 if not prepare :
+ − 526 log . debug ( "Nothing to prepare for {name!r} " )
+ − 527 return
+ − 528
+ − 529 for action , value in list ( prepare . items ()):
+ − 530 log . debug ( f "[ { name } ] [prepare] running { action !r} action" )
+ − 531 if action == "git" :
+ − 532 try :
+ − 533 git_path = which ( 'git' )[ 0 ]
+ − 534 except IndexError :
+ − 535 raise exceptions . NotFound (
+ − 536 "Can't find \" git \" executable, {name} can't be started without it"
+ − 537 )
+ − 538 await async_process . run ( git_path , "clone" , value , str ( dest_path ))
+ − 539 log . debug ( f " { value !r} git repository cloned at { dest_path } " )
+ − 540 else :
+ − 541 raise NotImplementedError (
+ − 542 f " { action !r} is not managed, can't start { name } "
+ − 543 )
+ − 544 del prepare [ action ]
+ − 545
+ − 546 if prepare :
+ − 547 raise exceptions . InternalError ( '"prepare" should be empty' )
+ − 548
+ − 549 async def startCommon ( self , app_data : dict ) -> None :
+ − 550 """Method running common action when starting a manager
+ − 551
+ − 552 It should be called by managers in "start" method.
+ − 553 """
+ − 554 log . info ( f "starting { app_data [ 'name' ] !r} " )
+ − 555 await self . _doPrepare ( app_data )
