baenv

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Manage Ballistica execution environment.
  4
  5This module is used to set up and/or check the global Python environment
  6before running a Ballistica app. This includes things such as paths,
  7logging, and app-dirs. Because these things are global in nature, this
  8should be done before any Ballistica modules are imported.
  9
 10This module can also be exec'ed directly to set up a default environment
 11and then run the app.
 12
 13Ballistica can be used without explicitly configuring the environment in
 14order to integrate it in arbitrary Python environments, but this may
 15cause some features to be disabled or behave differently than expected.
 16"""
 17from __future__ import annotations
 18
 19import os
 20import sys
 21import time
 22import logging
 23from pathlib import Path
 24from dataclasses import dataclass
 25from typing import TYPE_CHECKING
 26import __main__
 27
 28if TYPE_CHECKING:
 29    from typing import Any
 30
 31    from efro.logging import LogHandler
 32
 33# IMPORTANT - It is likely (and in some cases expected) that this
 34# module's code will be exec'ed multiple times. This is because it is
 35# the job of this module to set up Python paths for an engine run, and
 36# that may involve modifying sys.path in such a way that this module
 37# resolves to a different path afterwards (for example from
 38# /abs/path/to/ba_data/scripts/babase.py to ba_data/scripts/babase.py).
 39# This can result in the next import of baenv loading us from our 'new'
 40# location, which may or may not actually be the same file on disk as
 41# the last load. Either way, however, multiple execs will happen in some
 42# form.
 43#
 44# To handle that situation gracefully, we need to do a few things:
 45#
 46# - First, we need to store any mutable global state in the __main__
 47#   module; not in ourself. This way, alternate versions of ourself will
 48#   still know if we already ran configure/etc.
 49#
 50# - Second, we should avoid the use of isinstance and similar calls for
 51#   our types. An EnvConfig we create would technically be a different
 52#   type than an EnvConfig created by an alternate baenv.
 53
 54# Build number and version of the ballistica binary we expect to be
 55# using.
 56TARGET_BALLISTICA_BUILD = 22324
 57TARGET_BALLISTICA_VERSION = '1.7.39'
 58
 59
 60@dataclass
 61class EnvConfig:
 62    """Final config values we provide to the engine."""
 63
 64    #: Where app config/state data lives.
 65    config_dir: str
 66
 67    #: Directory containing ba_data and any other platform-specific data.
 68    data_dir: str
 69
 70    #: Where the app's built-in Python stuff lives.
 71    app_python_dir: str | None
 72
 73    #: Where the app's built-in Python stuff lives in the default case.
 74    standard_app_python_dir: str
 75
 76    #: Where the app's bundled third party Python stuff lives.
 77    site_python_dir: str | None
 78
 79    #: Custom Python provided by the user (mods).
 80    user_python_dir: str | None
 81
 82    #: We have a mechanism allowing app scripts to be overridden by
 83    #: placing a specially named directory in a user-scripts dir. This is
 84    #: true if that is enabled.
 85    is_user_app_python_dir: bool
 86
 87    #: Our fancy app log handler. This handles feeding logs, stdout, and
 88    #: stderr into the engine so they show up on in-app consoles, etc.
 89    log_handler: LogHandler | None
 90
 91    #: Initial data from the config.json file in the config dir. The
 92    #: config file is parsed by
 93    initial_app_config: Any
 94
 95    #: Timestamp when we first started doing stuff.
 96    launch_time: float
 97
 98
 99@dataclass
100class _EnvGlobals:
101    """Globals related to baenv's operation.
102
103    We store this in __main__ instead of in our own module because it is
104    likely that multiple versions of our module will be spun up and we
105    want a single set of globals (see notes at top of our module code).
106    """
107
108    config: EnvConfig | None = None
109    called_configure: bool = False
110    paths_set_failed: bool = False
111    modular_main_called: bool = False
112
113    @classmethod
114    def get(cls) -> _EnvGlobals:
115        """Create/return our singleton."""
116        name = '_baenv_globals'
117        envglobals: _EnvGlobals | None = getattr(__main__, name, None)
118        if envglobals is None:
119            envglobals = _EnvGlobals()
120            setattr(__main__, name, envglobals)
121        return envglobals
122
123
124def did_paths_set_fail() -> bool:
125    """Did we try to set paths and fail?"""
126    return _EnvGlobals.get().paths_set_failed
127
128
129def config_exists() -> bool:
130    """Has a config been created?"""
131
132    return _EnvGlobals.get().config is not None
133
134
135def get_config() -> EnvConfig:
136    """Return the active config, creating a default if none exists."""
137    envglobals = _EnvGlobals.get()
138
139    # If configure() has not been explicitly called, set up a
140    # minimally-intrusive default config. We want Ballistica to default
141    # to being a good citizen when imported into alien environments and
142    # not blow away logging or otherwise muck with stuff. All official
143    # paths to run Ballistica apps should be explicitly calling
144    # configure() first to get a full featured setup.
145    if not envglobals.called_configure:
146        configure(setup_logging=False)
147
148    config = envglobals.config
149    if config is None:
150        raise RuntimeError(
151            'baenv.configure() has been called but no config exists;'
152            ' perhaps it errored?'
153        )
154    return config
155
156
157def configure(
158    *,
159    config_dir: str | None = None,
160    data_dir: str | None = None,
161    user_python_dir: str | None = None,
162    app_python_dir: str | None = None,
163    site_python_dir: str | None = None,
164    contains_python_dist: bool = False,
165    setup_logging: bool = True,
166) -> None:
167    """Set up the environment for running a Ballistica app.
168
169    This includes things such as Python path wrangling and app directory
170    creation. This must be called before any actual Ballistica modules
171    are imported; the environment is locked in as soon as that happens.
172    """
173
174    # Measure when we start doing this stuff. We plug this in to show
175    # relative times in our log timestamp displays and also pass this to
176    # the engine to do the same there.
177    launch_time = time.time()
178
179    envglobals = _EnvGlobals.get()
180
181    # Keep track of whether we've been *called*, not whether a config
182    # has been created. Otherwise its possible to get multiple
183    # overlapping configure calls going.
184    if envglobals.called_configure:
185        raise RuntimeError(
186            'baenv.configure() has already been called;'
187            ' it can only be called once.'
188        )
189    envglobals.called_configure = True
190
191    # The very first thing we do is setup Python paths (while also
192    # calculating some engine paths). This code needs to be bulletproof
193    # since we have no logging yet at this point. We used to set up
194    # logging first, but this way logging stuff will get loaded from its
195    # proper final path (otherwise we might wind up using two different
196    # versions of efro.logging in a single engine run).
197    (
198        user_python_dir,
199        app_python_dir,
200        site_python_dir,
201        data_dir,
202        config_dir,
203        standard_app_python_dir,
204        is_user_app_python_dir,
205    ) = _setup_paths(
206        user_python_dir,
207        app_python_dir,
208        site_python_dir,
209        data_dir,
210        config_dir,
211    )
212
213    # Set up our log-handler and pipe Python's stdout/stderr into it.
214    # Later, once the engine comes up, the handler will feed its logs
215    # (including cached history) to the os-specific output location.
216    # This means anything printed or logged at this point forward should
217    # be visible on all platforms.
218    log_handler = _create_log_handler(launch_time) if setup_logging else None
219
220    # Load the raw app-config dict.
221    app_config = _read_app_config(os.path.join(config_dir, 'config.json'))
222
223    # Set logging levels to stored values or defaults.
224    if setup_logging:
225        _set_log_levels(app_config)
226
227    # We want to always be run in UTF-8 mode; complain if we're not.
228    if sys.flags.utf8_mode != 1:
229        logging.warning(
230            "Python's UTF-8 mode is not set. Running Ballistica without"
231            ' it may lead to errors.'
232        )
233
234    # Attempt to create dirs that we'll write stuff to.
235    _setup_dirs(config_dir, user_python_dir)
236
237    # Get ssl working if needed so we can use https and all that.
238    _setup_certs(contains_python_dist)
239
240    # This is now the active config.
241    envglobals.config = EnvConfig(
242        config_dir=config_dir,
243        data_dir=data_dir,
244        user_python_dir=user_python_dir,
245        app_python_dir=app_python_dir,
246        standard_app_python_dir=standard_app_python_dir,
247        site_python_dir=site_python_dir,
248        log_handler=log_handler,
249        is_user_app_python_dir=is_user_app_python_dir,
250        initial_app_config=app_config,
251        launch_time=launch_time,
252    )
253
254
255def _read_app_config(config_file_path: str) -> dict:
256    """Read the app config."""
257    import json
258
259    config: dict | Any
260    config_contents = ''
261    try:
262        if os.path.exists(config_file_path):
263            with open(config_file_path, encoding='utf-8') as infile:
264                config_contents = infile.read()
265            config = json.loads(config_contents)
266            if not isinstance(config, dict):
267                raise RuntimeError('Got non-dict for config root.')
268        else:
269            config = {}
270
271    except Exception:
272        logging.exception(
273            "Error reading config file '%s'.\n"
274            "Backing up broken config to'%s.broken'.",
275            config_file_path,
276            config_file_path,
277        )
278
279        try:
280            import shutil
281
282            shutil.copyfile(config_file_path, config_file_path + '.broken')
283        except Exception:
284            logging.exception('Error copying broken config.')
285        config = {}
286
287    return config
288
289
290def _calc_data_dir(data_dir: str | None) -> str:
291    if data_dir is None:
292        # To calc default data_dir, we assume this module was imported
293        # from that dir's ba_data/python subdir.
294        assert Path(__file__).parts[-3:-1] == ('ba_data', 'python')
295        data_dir_path = Path(__file__).parents[2]
296
297        # Prefer tidy relative paths like './ba_data' if possible so
298        # that things like stack traces are easier to read. For best
299        # results, platforms where CWD doesn't matter can chdir to where
300        # ba_data lives before calling configure().
301        #
302        # NOTE: If there's ever a case where the user is chdir'ing at
303        # runtime we might want an option to use only abs paths here.
304        cwd_path = Path.cwd()
305        data_dir = str(
306            data_dir_path.relative_to(cwd_path)
307            if data_dir_path.is_relative_to(cwd_path)
308            else data_dir_path
309        )
310    return data_dir
311
312
313def _create_log_handler(launch_time: float) -> LogHandler:
314    from efro.logging import setup_logging, LogLevel
315
316    log_handler = setup_logging(
317        log_path=None,
318        level=LogLevel.INFO,
319        log_stdout_stderr=True,
320        cache_size_limit=1024 * 1024,
321        launch_time=launch_time,
322    )
323    return log_handler
324
325
326def _set_log_levels(app_config: dict) -> None:
327
328    from bacommon.logging import get_base_logger_control_config_client
329    from bacommon.loggercontrol import LoggerControlConfig
330
331    try:
332        config = app_config.get('Log Levels', None)
333
334        if config is None:
335            get_base_logger_control_config_client().apply()
336            return
337
338        # Make sure data is expected types/values since this is user
339        # editable.
340        valid_levels = {
341            logging.NOTSET,
342            logging.DEBUG,
343            logging.INFO,
344            logging.WARNING,
345            logging.ERROR,
346            logging.CRITICAL,
347        }
348        for logname, loglevel in config.items():
349            if (
350                not isinstance(logname, str)
351                or not logname
352                or not isinstance(loglevel, int)
353                or not loglevel in valid_levels
354            ):
355                raise ValueError("Invalid 'Log Levels' data read from config.")
356
357        get_base_logger_control_config_client().apply_diff(
358            LoggerControlConfig(levels=config)
359        ).apply()
360
361    except Exception:
362        logging.exception('Error setting log levels.')
363
364
365def _setup_certs(contains_python_dist: bool) -> None:
366    # In situations where we're bringing our own Python, let's also
367    # provide our own root certs so ssl works. We can consider
368    # overriding this in particular embedded cases if we can verify that
369    # system certs are working. We also allow forcing this via an env
370    # var if the user desires.
371    if (
372        contains_python_dist
373        or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'
374    ):
375        import certifi
376
377        # Let both OpenSSL and requests (if present) know to use this.
378        os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = (
379            certifi.where()
380        )
381
382
383def _setup_paths(
384    user_python_dir: str | None,
385    app_python_dir: str | None,
386    site_python_dir: str | None,
387    data_dir: str | None,
388    config_dir: str | None,
389) -> tuple[str | None, str | None, str | None, str, str, str, bool]:
390    # First a few paths we can ALWAYS calculate since they don't affect
391    # Python imports:
392
393    envglobals = _EnvGlobals.get()
394
395    data_dir = _calc_data_dir(data_dir)
396
397    # Default config-dir is simply ~/.ballisticakit
398    if config_dir is None:
399        config_dir = str(Path(Path.home(), '.ballisticakit'))
400
401    # Standard app-python-dir is simply ba_data/python under data-dir.
402    standard_app_python_dir = str(Path(data_dir, 'ba_data', 'python'))
403
404    # Whether the final app-dir we're returning is a custom user-owned one.
405    is_user_app_python_dir = False
406
407    # If _babase has already been imported, there's not much we can do
408    # at this point aside from complain and inform for next time.
409    if '_babase' in sys.modules:
410        app_python_dir = user_python_dir = site_python_dir = None
411
412        # We don't actually complain yet here; we simply take note that
413        # we weren't able to set paths. Then we complain if/when the app
414        # is started. This way, non-app uses of babase won't be filled
415        # with unnecessary warnings.
416        envglobals.paths_set_failed = True
417
418    else:
419        # Ok; _babase hasn't been imported yet, so we can muck with
420        # Python paths.
421
422        if app_python_dir is None:
423            app_python_dir = standard_app_python_dir
424
425        # Likewise site-python-dir defaults to ba_data/python-site-packages.
426        if site_python_dir is None:
427            site_python_dir = str(
428                Path(data_dir, 'ba_data', 'python-site-packages')
429            )
430
431        # By default, user-python-dir is simply 'mods' under config-dir.
432        if user_python_dir is None:
433            user_python_dir = str(Path(config_dir, 'mods'))
434
435        # Wherever our user_python_dir is, if we find a sys/FOO_BAR dir
436        # under it where FOO matches our version and BAR matches our
437        # build number, use that as our app_python_dir. This allows
438        # modding built-in stuff on platforms where there is no write
439        # access to said built-in stuff.
440        check_dir = Path(
441            user_python_dir,
442            'sys',
443            f'{TARGET_BALLISTICA_VERSION}_{TARGET_BALLISTICA_BUILD}',
444        )
445        try:
446            if check_dir.is_dir():
447                app_python_dir = str(check_dir)
448                is_user_app_python_dir = True
449        except PermissionError:
450            logging.warning(
451                "PermissionError checking user-app-python-dir path '%s'.",
452                check_dir,
453            )
454
455        # Ok, now apply these to sys.path.
456
457        # First off, strip out any instances of the path containing this
458        # module. We will *probably* be re-adding the same path in a
459        # moment so this keeps things cleaner. Though hmm should we
460        # leave it in there in cases where we *don't* re-add the same
461        # path?...
462        our_parent_path = Path(__file__).parent.resolve()
463        oldpaths: list[str] = [
464            p for p in sys.path if Path(p).resolve() != our_parent_path
465        ]
466
467        # Let's place mods first (so users can override whatever they
468        # want) followed by our app scripts and lastly our bundled site
469        # stuff.
470
471        # One could make the argument that at least our bundled app &
472        # site stuff should be placed at the end so actual local site
473        # stuff could override it. That could be a good thing or a bad
474        # thing. Maybe we could add an option for that, but for now I'm
475        # prioritizing our stuff to give as consistent an environment as
476        # possible.
477        ourpaths = [user_python_dir, app_python_dir, site_python_dir]
478
479        # Special case: our modular builds will have a 'python-dylib'
480        # dir alongside the 'python' scripts dir which contains our
481        # binary Python modules. If we see that, add it to the path also.
482        # Not sure if we'd ever have a need to customize this path.
483        dylibdir = f'{app_python_dir}-dylib'
484        if os.path.exists(dylibdir):
485            ourpaths.append(dylibdir)
486
487        sys.path = ourpaths + oldpaths
488
489    return (
490        user_python_dir,
491        app_python_dir,
492        site_python_dir,
493        data_dir,
494        config_dir,
495        standard_app_python_dir,
496        is_user_app_python_dir,
497    )
498
499
500def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
501    create_dirs: list[tuple[str, str | None]] = [
502        ('config', config_dir),
503        ('user_python', user_python_dir),
504    ]
505    for cdirname, cdir in create_dirs:
506        if cdir is not None:
507            try:
508                os.makedirs(cdir, exist_ok=True)
509            except Exception:
510                # Not the end of the world if we can't make these dirs.
511                logging.warning(
512                    "Unable to create %s dir at '%s'.", cdirname, cdir
513                )
514
515
516def extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
517    """Given a list of args and an arg name, returns a value.
518
519    The arg flag and value are removed from the arg list. We also check
520    to make sure the path exists.
521
522    raises CleanErrors on any problems.
523    """
524    from efro.error import CleanError
525
526    count = sum(args.count(n) for n in names)
527    if not count:
528        return None
529
530    if count > 1:
531        raise CleanError(f'Arg {names} passed multiple times.')
532
533    for name in names:
534        if name not in args:
535            continue
536        argindex = args.index(name)
537        if argindex + 1 >= len(args):
538            raise CleanError(f'No value passed after {name} arg.')
539
540        val = args[argindex + 1]
541        del args[argindex : argindex + 2]
542
543        if is_dir and not os.path.isdir(val):
544            namepretty = names[0].removeprefix('--')
545            raise CleanError(
546                f"Provided {namepretty} path '{val}' is not a directory."
547            )
548        return val
549
550    raise RuntimeError(f'Expected arg name not found from {names}')
551
552
553def _modular_main() -> None:
554    from efro.error import CleanError
555
556    # Fundamentally, running a Ballistica app consists of the following:
557    # import baenv; baenv.configure(); import babase; babase.app.run()
558    #
559    # First baenv sets up things like Python paths the way the engine
560    # needs them, and then we import and run the engine.
561    #
562    # Below we're doing a slightly fancier version of that. Namely, we
563    # do some processing of command line args to allow overriding of
564    # paths or running explicit commands or whatever else. Our goal is
565    # that this modular form of the app should be basically
566    # indistinguishable from the monolithic form when used from the
567    # command line.
568
569    try:
570        # Take note that we're running via modular-main. The native
571        # layer can key off this to know whether it should apply
572        # sys.argv or not.
573        _EnvGlobals.get().modular_main_called = True
574
575        # Deal with a few key things here ourself before even running
576        # configure.
577
578        # The extract_arg stuff below modifies this so we work with a
579        # copy.
580        args = sys.argv.copy()
581
582        # NOTE: We need to keep these arg long/short arg versions synced
583        # to those in core_config.cc. That code parses these same args
584        # (even if it doesn't handle them in our case) and will complain
585        # if unrecognized args come through.
586
587        # Our -c arg basically mirrors Python's -c arg. If we get that,
588        # simply exec it and return; no engine stuff.
589        command = extract_arg(args, ['--command', '-c'], is_dir=False)
590        if command is not None:
591            exec(command)  # pylint: disable=exec-used
592            return
593
594        config_dir = extract_arg(args, ['--config-dir', '-C'], is_dir=True)
595        data_dir = extract_arg(args, ['--data-dir', '-d'], is_dir=True)
596        mods_dir = extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
597
598        # We run configure() BEFORE importing babase. (part of its job
599        # is to wrangle paths which can affect where babase and
600        # everything else gets loaded from).
601        configure(
602            config_dir=config_dir,
603            data_dir=data_dir,
604            user_python_dir=mods_dir,
605        )
606
607        import babase
608
609        # The engine will have parsed and processed all other args as
610        # part of the above import. If there were errors or args such as
611        # --help which should lead to us immediately returning, do so.
612        code = babase.get_immediate_return_code()
613        if code is not None:
614            sys.exit(code)
615
616        # Aaaand we're off!
617        babase.app.run()
618
619    # Code wanting us to die with a clean error message instead of an
620    # ugly stack trace can raise one of these.
621    except CleanError as clean_exc:
622        clean_exc.pretty_print()
623        sys.exit(1)
624
625
626# Exec'ing this module directly will do a standard app run.
627if __name__ == '__main__':
628    _modular_main()