baenv

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Manage ballistica execution environment.
  4
  5This module is used to set up and/or check the global Python environment
  6before running a ballistica app. This includes things such as paths,
  7logging, and app-dirs. Because these things are global in nature, this
  8should be done before any ballistica modules are imported.
  9
 10This module can also be exec'ed directly to set up a default environment
 11and then run the app.
 12
 13Ballistica can be used without explicitly configuring the environment in
 14order to integrate it in arbitrary Python environments, but this may
 15cause some features to be disabled or behave differently than expected.
 16"""
 17from __future__ import annotations
 18
 19import os
 20import sys
 21import time
 22import logging
 23from pathlib import Path
 24from dataclasses import dataclass
 25from typing import TYPE_CHECKING
 26import __main__
 27
 28if TYPE_CHECKING:
 29    from typing import Any
 30
 31    from efro.logging import LogHandler
 32
 33# IMPORTANT - It is likely (and in some cases expected) that this
 34# module's code will be exec'ed multiple times. This is because it is
 35# the job of this module to set up Python paths for an engine run, and
 36# that may involve modifying sys.path in such a way that this module
 37# resolves to a different path afterwards (for example from
 38# /abs/path/to/ba_data/scripts/babase.py to ba_data/scripts/babase.py).
 39# This can result in the next import of baenv loading us from our 'new'
 40# location, which may or may not actually be the same file on disk as
 41# the last load. Either way, however, multiple execs will happen in some
 42# form.
 43#
 44# To handle that situation gracefully, we need to do a few things:
 45#
 46# - First, we need to store any mutable global state in the __main__
 47#   module; not in ourself. This way, alternate versions of ourself will
 48#   still know if we already ran configure/etc.
 49#
 50# - Second, we should avoid the use of isinstance and similar calls for
 51#   our types. An EnvConfig we create would technically be a different
 52#   type than an EnvConfig created by an alternate baenv.
 53
 54# Build number and version of the ballistica binary we expect to be
 55# using.
 56TARGET_BALLISTICA_BUILD = 22110
 57TARGET_BALLISTICA_VERSION = '1.7.37'
 58
 59
 60@dataclass
 61class EnvConfig:
 62    """Final config values we provide to the engine."""
 63
 64    # Where app config/state data lives.
 65    config_dir: str
 66
 67    # Directory containing ba_data and any other platform-specific data.
 68    data_dir: str
 69
 70    # Where the app's built-in Python stuff lives.
 71    app_python_dir: str | None
 72
 73    # Where the app's built-in Python stuff lives in the default case.
 74    standard_app_python_dir: str
 75
 76    # Where the app's bundled third party Python stuff lives.
 77    site_python_dir: str | None
 78
 79    # Custom Python provided by the user (mods).
 80    user_python_dir: str | None
 81
 82    # We have a mechanism allowing app scripts to be overridden by
 83    # placing a specially named directory in a user-scripts dir.
 84    # This is true if that is enabled.
 85    is_user_app_python_dir: bool
 86
 87    # Our fancy app log handler. This handles feeding logs, stdout, and
 88    # stderr into the engine so they show up on in-app consoles, etc.
 89    log_handler: LogHandler | None
 90
 91    # Initial data from the config.json file in the config dir. The
 92    # config file is parsed by
 93    initial_app_config: Any
 94
 95    # Timestamp when we first started doing stuff.
 96    launch_time: float
 97
 98
 99@dataclass
100class _EnvGlobals:
101    """Globals related to baenv's operation.
102
103    We store this in __main__ instead of in our own module because it
104    is likely that multiple versions of our module will be spun up
105    and we want a single set of globals (see notes at top of our module
106    code).
107    """
108
109    config: EnvConfig | None = None
110    called_configure: bool = False
111    paths_set_failed: bool = False
112    modular_main_called: bool = False
113
114    @classmethod
115    def get(cls) -> _EnvGlobals:
116        """Create/return our singleton."""
117        name = '_baenv_globals'
118        envglobals: _EnvGlobals | None = getattr(__main__, name, None)
119        if envglobals is None:
120            envglobals = _EnvGlobals()
121            setattr(__main__, name, envglobals)
122        return envglobals
123
124
125def did_paths_set_fail() -> bool:
126    """Did we try to set paths and fail?"""
127    return _EnvGlobals.get().paths_set_failed
128
129
130def config_exists() -> bool:
131    """Has a config been created?"""
132
133    return _EnvGlobals.get().config is not None
134
135
136def get_config() -> EnvConfig:
137    """Return the active config, creating a default if none exists."""
138    envglobals = _EnvGlobals.get()
139
140    # If configure() has not been explicitly called, set up a
141    # minimally-intrusive default config. We want Ballistica to default
142    # to being a good citizen when imported into alien environments and
143    # not blow away logging or otherwise muck with stuff. All official
144    # paths to run Ballistica apps should be explicitly calling
145    # configure() first to get a full featured setup.
146    if not envglobals.called_configure:
147        configure(setup_logging=False)
148
149    config = envglobals.config
150    if config is None:
151        raise RuntimeError(
152            'baenv.configure() has been called but no config exists;'
153            ' perhaps it errored?'
154        )
155    return config
156
157
158def configure(
159    *,
160    config_dir: str | None = None,
161    data_dir: str | None = None,
162    user_python_dir: str | None = None,
163    app_python_dir: str | None = None,
164    site_python_dir: str | None = None,
165    contains_python_dist: bool = False,
166    setup_logging: bool = True,
167) -> None:
168    """Set up the environment for running a Ballistica app.
169
170    This includes things such as Python path wrangling and app directory
171    creation. This must be called before any actual Ballistica modules
172    are imported; the environment is locked in as soon as that happens.
173    """
174
175    # Measure when we start doing this stuff. We plug this in to show
176    # relative times in our log timestamp displays and also pass this to
177    # the engine to do the same there.
178    launch_time = time.time()
179
180    envglobals = _EnvGlobals.get()
181
182    # Keep track of whether we've been *called*, not whether a config
183    # has been created. Otherwise its possible to get multiple
184    # overlapping configure calls going.
185    if envglobals.called_configure:
186        raise RuntimeError(
187            'baenv.configure() has already been called;'
188            ' it can only be called once.'
189        )
190    envglobals.called_configure = True
191
192    # The very first thing we do is setup Python paths (while also
193    # calculating some engine paths). This code needs to be bulletproof
194    # since we have no logging yet at this point. We used to set up
195    # logging first, but this way logging stuff will get loaded from its
196    # proper final path (otherwise we might wind up using two different
197    # versions of efro.logging in a single engine run).
198    (
199        user_python_dir,
200        app_python_dir,
201        site_python_dir,
202        data_dir,
203        config_dir,
204        standard_app_python_dir,
205        is_user_app_python_dir,
206    ) = _setup_paths(
207        user_python_dir,
208        app_python_dir,
209        site_python_dir,
210        data_dir,
211        config_dir,
212    )
213
214    # Set up our log-handler and pipe Python's stdout/stderr into it.
215    # Later, once the engine comes up, the handler will feed its logs
216    # (including cached history) to the os-specific output location.
217    # This means anything printed or logged at this point forward should
218    # be visible on all platforms.
219    log_handler = _create_log_handler(launch_time) if setup_logging else None
220
221    # Load the raw app-config dict.
222    app_config = _read_app_config(os.path.join(config_dir, 'config.json'))
223
224    # Set logging levels to stored values or defaults.
225    if setup_logging:
226        _set_log_levels(app_config)
227
228    # We want to always be run in UTF-8 mode; complain if we're not.
229    if sys.flags.utf8_mode != 1:
230        logging.warning(
231            "Python's UTF-8 mode is not set. Running Ballistica without"
232            ' it may lead to errors.'
233        )
234
235    # Attempt to create dirs that we'll write stuff to.
236    _setup_dirs(config_dir, user_python_dir)
237
238    # Get ssl working if needed so we can use https and all that.
239    _setup_certs(contains_python_dist)
240
241    # This is now the active config.
242    envglobals.config = EnvConfig(
243        config_dir=config_dir,
244        data_dir=data_dir,
245        user_python_dir=user_python_dir,
246        app_python_dir=app_python_dir,
247        standard_app_python_dir=standard_app_python_dir,
248        site_python_dir=site_python_dir,
249        log_handler=log_handler,
250        is_user_app_python_dir=is_user_app_python_dir,
251        initial_app_config=app_config,
252        launch_time=launch_time,
253    )
254
255
256def _read_app_config(config_file_path: str) -> dict:
257    """Read the app config."""
258    import json
259
260    config: dict | Any
261    config_contents = ''
262    try:
263        if os.path.exists(config_file_path):
264            with open(config_file_path, encoding='utf-8') as infile:
265                config_contents = infile.read()
266            config = json.loads(config_contents)
267            if not isinstance(config, dict):
268                raise RuntimeError('Got non-dict for config root.')
269        else:
270            config = {}
271
272    except Exception:
273        logging.exception(
274            "Error reading config file '%s'.\n"
275            "Backing up broken config to'%s.broken'.",
276            config_file_path,
277            config_file_path,
278        )
279
280        try:
281            import shutil
282
283            shutil.copyfile(config_file_path, config_file_path + '.broken')
284        except Exception:
285            logging.exception('Error copying broken config.')
286        config = {}
287
288    return config
289
290
291def _calc_data_dir(data_dir: str | None) -> str:
292    if data_dir is None:
293        # To calc default data_dir, we assume this module was imported
294        # from that dir's ba_data/python subdir.
295        assert Path(__file__).parts[-3:-1] == ('ba_data', 'python')
296        data_dir_path = Path(__file__).parents[2]
297
298        # Prefer tidy relative paths like './ba_data' if possible so
299        # that things like stack traces are easier to read. For best
300        # results, platforms where CWD doesn't matter can chdir to where
301        # ba_data lives before calling configure().
302        #
303        # NOTE: If there's ever a case where the user is chdir'ing at
304        # runtime we might want an option to use only abs paths here.
305        cwd_path = Path.cwd()
306        data_dir = str(
307            data_dir_path.relative_to(cwd_path)
308            if data_dir_path.is_relative_to(cwd_path)
309            else data_dir_path
310        )
311    return data_dir
312
313
314def _create_log_handler(launch_time: float) -> LogHandler:
315    from efro.logging import setup_logging, LogLevel
316
317    log_handler = setup_logging(
318        log_path=None,
319        level=LogLevel.INFO,
320        log_stdout_stderr=True,
321        cache_size_limit=1024 * 1024,
322        launch_time=launch_time,
323    )
324    return log_handler
325
326
327def _set_log_levels(app_config: dict) -> None:
328
329    from bacommon.logging import get_base_logger_control_config_client
330    from bacommon.loggercontrol import LoggerControlConfig
331
332    try:
333        config = app_config.get('Log Levels', None)
334
335        if config is None:
336            get_base_logger_control_config_client().apply()
337            return
338
339        # Make sure data is expected types/values since this is user
340        # editable.
341        valid_levels = {
342            logging.NOTSET,
343            logging.DEBUG,
344            logging.INFO,
345            logging.WARNING,
346            logging.ERROR,
347            logging.CRITICAL,
348        }
349        for logname, loglevel in config.items():
350            if (
351                not isinstance(logname, str)
352                or not logname
353                or not isinstance(loglevel, int)
354                or not loglevel in valid_levels
355            ):
356                raise ValueError("Invalid 'Log Levels' data read from config.")
357
358        get_base_logger_control_config_client().apply_diff(
359            LoggerControlConfig(levels=config)
360        ).apply()
361
362    except Exception:
363        logging.exception('Error setting log levels.')
364
365
366def _setup_certs(contains_python_dist: bool) -> None:
367    # In situations where we're bringing our own Python, let's also
368    # provide our own root certs so ssl works. We can consider
369    # overriding this in particular embedded cases if we can verify that
370    # system certs are working. We also allow forcing this via an env
371    # var if the user desires.
372    if (
373        contains_python_dist
374        or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'
375    ):
376        import certifi
377
378        # Let both OpenSSL and requests (if present) know to use this.
379        os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = (
380            certifi.where()
381        )
382
383
384def _setup_paths(
385    user_python_dir: str | None,
386    app_python_dir: str | None,
387    site_python_dir: str | None,
388    data_dir: str | None,
389    config_dir: str | None,
390) -> tuple[str | None, str | None, str | None, str, str, str, bool]:
391    # First a few paths we can ALWAYS calculate since they don't affect
392    # Python imports:
393
394    envglobals = _EnvGlobals.get()
395
396    data_dir = _calc_data_dir(data_dir)
397
398    # Default config-dir is simply ~/.ballisticakit
399    if config_dir is None:
400        config_dir = str(Path(Path.home(), '.ballisticakit'))
401
402    # Standard app-python-dir is simply ba_data/python under data-dir.
403    standard_app_python_dir = str(Path(data_dir, 'ba_data', 'python'))
404
405    # Whether the final app-dir we're returning is a custom user-owned one.
406    is_user_app_python_dir = False
407
408    # If _babase has already been imported, there's not much we can do
409    # at this point aside from complain and inform for next time.
410    if '_babase' in sys.modules:
411        app_python_dir = user_python_dir = site_python_dir = None
412
413        # We don't actually complain yet here; we simply take note that
414        # we weren't able to set paths. Then we complain if/when the app
415        # is started. This way, non-app uses of babase won't be filled
416        # with unnecessary warnings.
417        envglobals.paths_set_failed = True
418
419    else:
420        # Ok; _babase hasn't been imported yet, so we can muck with
421        # Python paths.
422
423        if app_python_dir is None:
424            app_python_dir = standard_app_python_dir
425
426        # Likewise site-python-dir defaults to ba_data/python-site-packages.
427        if site_python_dir is None:
428            site_python_dir = str(
429                Path(data_dir, 'ba_data', 'python-site-packages')
430            )
431
432        # By default, user-python-dir is simply 'mods' under config-dir.
433        if user_python_dir is None:
434            user_python_dir = str(Path(config_dir, 'mods'))
435
436        # Wherever our user_python_dir is, if we find a sys/FOO_BAR dir
437        # under it where FOO matches our version and BAR matches our
438        # build number, use that as our app_python_dir. This allows
439        # modding built-in stuff on platforms where there is no write
440        # access to said built-in stuff.
441        check_dir = Path(
442            user_python_dir,
443            'sys',
444            f'{TARGET_BALLISTICA_VERSION}_{TARGET_BALLISTICA_BUILD}',
445        )
446        try:
447            if check_dir.is_dir():
448                app_python_dir = str(check_dir)
449                is_user_app_python_dir = True
450        except PermissionError:
451            logging.warning(
452                "PermissionError checking user-app-python-dir path '%s'.",
453                check_dir,
454            )
455
456        # Ok, now apply these to sys.path.
457
458        # First off, strip out any instances of the path containing this
459        # module. We will *probably* be re-adding the same path in a
460        # moment so this keeps things cleaner. Though hmm should we
461        # leave it in there in cases where we *don't* re-add the same
462        # path?...
463        our_parent_path = Path(__file__).parent.resolve()
464        oldpaths: list[str] = [
465            p for p in sys.path if Path(p).resolve() != our_parent_path
466        ]
467
468        # Let's place mods first (so users can override whatever they
469        # want) followed by our app scripts and lastly our bundled site
470        # stuff.
471
472        # One could make the argument that at least our bundled app &
473        # site stuff should be placed at the end so actual local site
474        # stuff could override it. That could be a good thing or a bad
475        # thing. Maybe we could add an option for that, but for now I'm
476        # prioritizing our stuff to give as consistent an environment as
477        # possible.
478        ourpaths = [user_python_dir, app_python_dir, site_python_dir]
479
480        # Special case: our modular builds will have a 'python-dylib'
481        # dir alongside the 'python' scripts dir which contains our
482        # binary Python modules. If we see that, add it to the path also.
483        # Not sure if we'd ever have a need to customize this path.
484        dylibdir = f'{app_python_dir}-dylib'
485        if os.path.exists(dylibdir):
486            ourpaths.append(dylibdir)
487
488        sys.path = ourpaths + oldpaths
489
490    return (
491        user_python_dir,
492        app_python_dir,
493        site_python_dir,
494        data_dir,
495        config_dir,
496        standard_app_python_dir,
497        is_user_app_python_dir,
498    )
499
500
501def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
502    create_dirs: list[tuple[str, str | None]] = [
503        ('config', config_dir),
504        ('user_python', user_python_dir),
505    ]
506    for cdirname, cdir in create_dirs:
507        if cdir is not None:
508            try:
509                os.makedirs(cdir, exist_ok=True)
510            except Exception:
511                # Not the end of the world if we can't make these dirs.
512                logging.warning(
513                    "Unable to create %s dir at '%s'.", cdirname, cdir
514                )
515
516
517def extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
518    """Given a list of args and an arg name, returns a value.
519
520    The arg flag and value are removed from the arg list. We also check
521    to make sure the path exists.
522
523    raises CleanErrors on any problems.
524    """
525    from efro.error import CleanError
526
527    count = sum(args.count(n) for n in names)
528    if not count:
529        return None
530
531    if count > 1:
532        raise CleanError(f'Arg {names} passed multiple times.')
533
534    for name in names:
535        if name not in args:
536            continue
537        argindex = args.index(name)
538        if argindex + 1 >= len(args):
539            raise CleanError(f'No value passed after {name} arg.')
540
541        val = args[argindex + 1]
542        del args[argindex : argindex + 2]
543
544        if is_dir and not os.path.isdir(val):
545            namepretty = names[0].removeprefix('--')
546            raise CleanError(
547                f"Provided {namepretty} path '{val}' is not a directory."
548            )
549        return val
550
551    raise RuntimeError(f'Expected arg name not found from {names}')
552
553
554def _modular_main() -> None:
555    from efro.error import CleanError
556
557    # Fundamentally, running a Ballistica app consists of the following:
558    # import baenv; baenv.configure(); import babase; babase.app.run()
559    #
560    # First baenv sets up things like Python paths the way the engine
561    # needs them, and then we import and run the engine.
562    #
563    # Below we're doing a slightly fancier version of that. Namely, we
564    # do some processing of command line args to allow overriding of
565    # paths or running explicit commands or whatever else. Our goal is
566    # that this modular form of the app should be basically
567    # indistinguishable from the monolithic form when used from the
568    # command line.
569
570    try:
571        # Take note that we're running via modular-main. The native
572        # layer can key off this to know whether it should apply
573        # sys.argv or not.
574        _EnvGlobals.get().modular_main_called = True
575
576        # Deal with a few key things here ourself before even running
577        # configure.
578
579        # The extract_arg stuff below modifies this so we work with a
580        # copy.
581        args = sys.argv.copy()
582
583        # NOTE: We need to keep these arg long/short arg versions synced
584        # to those in core_config.cc. That code parses these same args
585        # (even if it doesn't handle them in our case) and will complain
586        # if unrecognized args come through.
587
588        # Our -c arg basically mirrors Python's -c arg. If we get that,
589        # simply exec it and return; no engine stuff.
590        command = extract_arg(args, ['--command', '-c'], is_dir=False)
591        if command is not None:
592            exec(command)  # pylint: disable=exec-used
593            return
594
595        config_dir = extract_arg(args, ['--config-dir', '-C'], is_dir=True)
596        data_dir = extract_arg(args, ['--data-dir', '-d'], is_dir=True)
597        mods_dir = extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
598
599        # We run configure() BEFORE importing babase. (part of its job
600        # is to wrangle paths which can affect where babase and
601        # everything else gets loaded from).
602        configure(
603            config_dir=config_dir,
604            data_dir=data_dir,
605            user_python_dir=mods_dir,
606        )
607
608        import babase
609
610        # The engine will have parsed and processed all other args as
611        # part of the above import. If there were errors or args such as
612        # --help which should lead to us immediately returning, do so.
613        code = babase.get_immediate_return_code()
614        if code is not None:
615            sys.exit(code)
616
617        # Aaaand we're off!
618        babase.app.run()
619
620    # Code wanting us to die with a clean error message instead of an
621    # ugly stack trace can raise one of these.
622    except CleanError as clean_exc:
623        clean_exc.pretty_print()
624        sys.exit(1)
625
626
627# Exec'ing this module directly will do a standard app run.
628if __name__ == '__main__':
629    _modular_main()