baenv

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Manage ballistica execution environment.
  4
  5This module is used to set up and/or check the global Python environment
  6before running a ballistica app. This includes things such as paths,
  7logging, and app-dirs. Because these things are global in nature, this
  8should be done before any ballistica modules are imported.
  9
 10This module can also be exec'ed directly to set up a default environment
 11and then run the app.
 12
 13Ballistica can be used without explicitly configuring the environment in
 14order to integrate it in arbitrary Python environments, but this may
 15cause some features to be disabled or behave differently than expected.
 16"""
 17from __future__ import annotations
 18
 19import os
 20import sys
 21import logging
 22from pathlib import Path
 23from dataclasses import dataclass
 24from typing import TYPE_CHECKING
 25import __main__
 26
 27if TYPE_CHECKING:
 28    from typing import Any
 29
 30    from efro.log import LogHandler
 31
 32# IMPORTANT - It is likely (and in some cases expected) that this
 33# module's code will be exec'ed multiple times. This is because it is
 34# the job of this module to set up Python paths for an engine run, and
 35# that may involve modifying sys.path in such a way that this module
 36# resolves to a different path afterwards (for example from
 37# /abs/path/to/ba_data/scripts/babase.py to ba_data/scripts/babase.py).
 38# This can result in the next import of baenv loading us from our 'new'
 39# location, which may or may not actually be the same file on disk as
 40# the last load. Either way, however, multiple execs will happen in some
 41# form.
 42#
 43# To handle that situation gracefully, we need to do a few things:
 44#
 45# - First, we need to store any mutable global state in the __main__
 46#   module; not in ourself. This way, alternate versions of ourself will
 47#   still know if we already ran configure/etc.
 48#
 49# - Second, we should avoid the use of isinstance and similar calls for
 50#   our types. An EnvConfig we create would technically be a different
 51#   type than an EnvConfig created by an alternate baenv.
 52
 53# Build number and version of the ballistica binary we expect to be
 54# using.
 55TARGET_BALLISTICA_BUILD = 21827
 56TARGET_BALLISTICA_VERSION = '1.7.35'
 57
 58
 59@dataclass
 60class EnvConfig:
 61    """Final config values we provide to the engine."""
 62
 63    # Where app config/state data lives.
 64    config_dir: str
 65
 66    # Directory containing ba_data and any other platform-specific data.
 67    data_dir: str
 68
 69    # Where the app's built-in Python stuff lives.
 70    app_python_dir: str | None
 71
 72    # Where the app's built-in Python stuff lives in the default case.
 73    standard_app_python_dir: str
 74
 75    # Where the app's bundled third party Python stuff lives.
 76    site_python_dir: str | None
 77
 78    # Custom Python provided by the user (mods).
 79    user_python_dir: str | None
 80
 81    # We have a mechanism allowing app scripts to be overridden by
 82    # placing a specially named directory in a user-scripts dir.
 83    # This is true if that is enabled.
 84    is_user_app_python_dir: bool
 85
 86    # Our fancy app log handler. This handles feeding logs, stdout, and
 87    # stderr into the engine so they show up on in-app consoles, etc.
 88    log_handler: LogHandler | None
 89
 90    # Initial data from the ballisticakit-config.json file. This is
 91    # passed mostly as an optimization to avoid reading the same config
 92    # file twice, since config data is first needed in baenv and next in
 93    # the engine. It will be cleared after passing it to the app's
 94    # config management subsystem and should not be accessed by any
 95    # other code.
 96    initial_app_config: Any
 97
 98
 99@dataclass
100class _EnvGlobals:
101    """Globals related to baenv's operation.
102
103    We store this in __main__ instead of in our own module because it
104    is likely that multiple versions of our module will be spun up
105    and we want a single set of globals (see notes at top of our module
106    code).
107    """
108
109    config: EnvConfig | None = None
110    called_configure: bool = False
111    paths_set_failed: bool = False
112    modular_main_called: bool = False
113
114    @classmethod
115    def get(cls) -> _EnvGlobals:
116        """Create/return our singleton."""
117        name = '_baenv_globals'
118        envglobals: _EnvGlobals | None = getattr(__main__, name, None)
119        if envglobals is None:
120            envglobals = _EnvGlobals()
121            setattr(__main__, name, envglobals)
122        return envglobals
123
124
125def did_paths_set_fail() -> bool:
126    """Did we try to set paths and fail?"""
127    return _EnvGlobals.get().paths_set_failed
128
129
130def config_exists() -> bool:
131    """Has a config been created?"""
132
133    return _EnvGlobals.get().config is not None
134
135
136def get_config() -> EnvConfig:
137    """Return the active config, creating a default if none exists."""
138    envglobals = _EnvGlobals.get()
139
140    # If configure() has not been explicitly called, set up a
141    # minimally-intrusive default config. We want Ballistica to default
142    # to being a good citizen when imported into alien environments and
143    # not blow away logging or otherwise muck with stuff. All official
144    # paths to run Ballistica apps should be explicitly calling
145    # configure() first to get a full featured setup.
146    if not envglobals.called_configure:
147        configure(setup_logging=False)
148
149    config = envglobals.config
150    if config is None:
151        raise RuntimeError(
152            'baenv.configure() has been called but no config exists;'
153            ' perhaps it errored?'
154        )
155    return config
156
157
158def configure(
159    config_dir: str | None = None,
160    data_dir: str | None = None,
161    user_python_dir: str | None = None,
162    app_python_dir: str | None = None,
163    site_python_dir: str | None = None,
164    contains_python_dist: bool = False,
165    setup_logging: bool = True,
166) -> None:
167    """Set up the environment for running a Ballistica app.
168
169    This includes things such as Python path wrangling and app directory
170    creation. This must be called before any actual Ballistica modules
171    are imported; the environment is locked in as soon as that happens.
172    """
173
174    envglobals = _EnvGlobals.get()
175
176    # Keep track of whether we've been *called*, not whether a config
177    # has been created. Otherwise its possible to get multiple
178    # overlapping configure calls going.
179    if envglobals.called_configure:
180        raise RuntimeError(
181            'baenv.configure() has already been called;'
182            ' it can only be called once.'
183        )
184    envglobals.called_configure = True
185
186    # The very first thing we do is setup Python paths (while also
187    # calculating some engine paths). This code needs to be bulletproof
188    # since we have no logging yet at this point. We used to set up
189    # logging first, but this way logging stuff will get loaded from its
190    # proper final path (otherwise we might wind up using two different
191    # versions of efro.logging in a single engine run).
192    (
193        user_python_dir,
194        app_python_dir,
195        site_python_dir,
196        data_dir,
197        config_dir,
198        standard_app_python_dir,
199        is_user_app_python_dir,
200    ) = _setup_paths(
201        user_python_dir,
202        app_python_dir,
203        site_python_dir,
204        data_dir,
205        config_dir,
206    )
207
208    # The second thing we do is set up our logging system and pipe
209    # Python's stdout/stderr into it. At this point we can at least
210    # debug problems on systems where native stdout/stderr is not easily
211    # accessible such as Android.
212    log_handler = _setup_logging() if setup_logging else None
213
214    # We want to always be run in UTF-8 mode; complain if we're not.
215    if sys.flags.utf8_mode != 1:
216        logging.warning(
217            "Python's UTF-8 mode is not set. Running Ballistica without"
218            ' it may lead to errors.'
219        )
220
221    # Attempt to create dirs that we'll write stuff to.
222    _setup_dirs(config_dir, user_python_dir)
223
224    # Get ssl working if needed so we can use https and all that.
225    _setup_certs(contains_python_dist)
226
227    # This is now the active config.
228    envglobals.config = EnvConfig(
229        config_dir=config_dir,
230        data_dir=data_dir,
231        user_python_dir=user_python_dir,
232        app_python_dir=app_python_dir,
233        standard_app_python_dir=standard_app_python_dir,
234        site_python_dir=site_python_dir,
235        log_handler=log_handler,
236        is_user_app_python_dir=is_user_app_python_dir,
237        initial_app_config=None,
238    )
239
240
241def _calc_data_dir(data_dir: str | None) -> str:
242    if data_dir is None:
243        # To calc default data_dir, we assume this module was imported
244        # from that dir's ba_data/python subdir.
245        assert Path(__file__).parts[-3:-1] == ('ba_data', 'python')
246        data_dir_path = Path(__file__).parents[2]
247
248        # Prefer tidy relative paths like './ba_data' if possible so
249        # that things like stack traces are easier to read. For best
250        # results, platforms where CWD doesn't matter can chdir to where
251        # ba_data lives before calling configure().
252        #
253        # NOTE: If there's ever a case where the user is chdir'ing at
254        # runtime we might want an option to use only abs paths here.
255        cwd_path = Path.cwd()
256        data_dir = str(
257            data_dir_path.relative_to(cwd_path)
258            if data_dir_path.is_relative_to(cwd_path)
259            else data_dir_path
260        )
261    return data_dir
262
263
264def _setup_logging() -> LogHandler:
265    from efro.log import setup_logging, LogLevel
266
267    # TODO: should set this up with individual loggers under a top level
268    # 'ba' logger, and at that point we can kill off the
269    # suppress_non_root_debug option since we'll only ever need to set
270    # 'ba' to DEBUG at most.
271    log_handler = setup_logging(
272        log_path=None,
273        level=LogLevel.DEBUG,
274        suppress_non_root_debug=True,
275        log_stdout_stderr=True,
276        cache_size_limit=1024 * 1024,
277    )
278    return log_handler
279
280
281def _setup_certs(contains_python_dist: bool) -> None:
282    # In situations where we're bringing our own Python, let's also
283    # provide our own root certs so ssl works. We can consider
284    # overriding this in particular embedded cases if we can verify that
285    # system certs are working. We also allow forcing this via an env
286    # var if the user desires.
287    if (
288        contains_python_dist
289        or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'
290    ):
291        import certifi
292
293        # Let both OpenSSL and requests (if present) know to use this.
294        os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = (
295            certifi.where()
296        )
297
298
299def _setup_paths(
300    user_python_dir: str | None,
301    app_python_dir: str | None,
302    site_python_dir: str | None,
303    data_dir: str | None,
304    config_dir: str | None,
305) -> tuple[str | None, str | None, str | None, str, str, str, bool]:
306    # First a few paths we can ALWAYS calculate since they don't affect
307    # Python imports:
308
309    envglobals = _EnvGlobals.get()
310
311    data_dir = _calc_data_dir(data_dir)
312
313    # Default config-dir is simply ~/.ballisticakit
314    if config_dir is None:
315        config_dir = str(Path(Path.home(), '.ballisticakit'))
316
317    # Standard app-python-dir is simply ba_data/python under data-dir.
318    standard_app_python_dir = str(Path(data_dir, 'ba_data', 'python'))
319
320    # Whether the final app-dir we're returning is a custom user-owned one.
321    is_user_app_python_dir = False
322
323    # If _babase has already been imported, there's not much we can do
324    # at this point aside from complain and inform for next time.
325    if '_babase' in sys.modules:
326        app_python_dir = user_python_dir = site_python_dir = None
327
328        # We don't actually complain yet here; we simply take note that
329        # we weren't able to set paths. Then we complain if/when the app
330        # is started. This way, non-app uses of babase won't be filled
331        # with unnecessary warnings.
332        envglobals.paths_set_failed = True
333
334    else:
335        # Ok; _babase hasn't been imported yet, so we can muck with
336        # Python paths.
337
338        if app_python_dir is None:
339            app_python_dir = standard_app_python_dir
340
341        # Likewise site-python-dir defaults to ba_data/python-site-packages.
342        if site_python_dir is None:
343            site_python_dir = str(
344                Path(data_dir, 'ba_data', 'python-site-packages')
345            )
346
347        # By default, user-python-dir is simply 'mods' under config-dir.
348        if user_python_dir is None:
349            user_python_dir = str(Path(config_dir, 'mods'))
350
351        # Wherever our user_python_dir is, if we find a sys/FOO dir
352        # under it where FOO matches our version, use that as our
353        # app_python_dir. This allows modding built-in stuff on
354        # platforms where there is no write access to said built-in
355        # stuff.
356        check_dir = Path(user_python_dir, 'sys', TARGET_BALLISTICA_VERSION)
357        try:
358            if check_dir.is_dir():
359                app_python_dir = str(check_dir)
360                is_user_app_python_dir = True
361        except PermissionError:
362            logging.warning(
363                "PermissionError checking user-app-python-dir path '%s'.",
364                check_dir,
365            )
366
367        # Ok, now apply these to sys.path.
368
369        # First off, strip out any instances of the path containing this
370        # module. We will *probably* be re-adding the same path in a
371        # moment so this keeps things cleaner. Though hmm should we
372        # leave it in there in cases where we *don't* re-add the same
373        # path?...
374        our_parent_path = Path(__file__).parent.resolve()
375        oldpaths: list[str] = [
376            p for p in sys.path if Path(p).resolve() != our_parent_path
377        ]
378
379        # Let's place mods first (so users can override whatever they
380        # want) followed by our app scripts and lastly our bundled site
381        # stuff.
382
383        # One could make the argument that at least our bundled app &
384        # site stuff should be placed at the end so actual local site
385        # stuff could override it. That could be a good thing or a bad
386        # thing. Maybe we could add an option for that, but for now I'm
387        # prioritizing our stuff to give as consistent an environment as
388        # possible.
389        ourpaths = [user_python_dir, app_python_dir, site_python_dir]
390
391        # Special case: our modular builds will have a 'python-dylib'
392        # dir alongside the 'python' scripts dir which contains our
393        # binary Python modules. If we see that, add it to the path also.
394        # Not sure if we'd ever have a need to customize this path.
395        dylibdir = f'{app_python_dir}-dylib'
396        if os.path.exists(dylibdir):
397            ourpaths.append(dylibdir)
398
399        sys.path = ourpaths + oldpaths
400
401    return (
402        user_python_dir,
403        app_python_dir,
404        site_python_dir,
405        data_dir,
406        config_dir,
407        standard_app_python_dir,
408        is_user_app_python_dir,
409    )
410
411
412def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
413    create_dirs: list[tuple[str, str | None]] = [
414        ('config', config_dir),
415        ('user_python', user_python_dir),
416    ]
417    for cdirname, cdir in create_dirs:
418        if cdir is not None:
419            try:
420                os.makedirs(cdir, exist_ok=True)
421            except Exception:
422                # Not the end of the world if we can't make these dirs.
423                logging.warning(
424                    "Unable to create %s dir at '%s'.", cdirname, cdir
425                )
426
427
428def extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
429    """Given a list of args and an arg name, returns a value.
430
431    The arg flag and value are removed from the arg list. We also check
432    to make sure the path exists.
433
434    raises CleanErrors on any problems.
435    """
436    from efro.error import CleanError
437
438    count = sum(args.count(n) for n in names)
439    if not count:
440        return None
441
442    if count > 1:
443        raise CleanError(f'Arg {names} passed multiple times.')
444
445    for name in names:
446        if name not in args:
447            continue
448        argindex = args.index(name)
449        if argindex + 1 >= len(args):
450            raise CleanError(f'No value passed after {name} arg.')
451
452        val = args[argindex + 1]
453        del args[argindex : argindex + 2]
454
455        if is_dir and not os.path.isdir(val):
456            namepretty = names[0].removeprefix('--')
457            raise CleanError(
458                f"Provided {namepretty} path '{val}' is not a directory."
459            )
460        return val
461
462    raise RuntimeError(f'Expected arg name not found from {names}')
463
464
465def _modular_main() -> None:
466    from efro.error import CleanError
467
468    # Fundamentally, running a Ballistica app consists of the following:
469    # import baenv; baenv.configure(); import babase; babase.app.run()
470    #
471    # First baenv sets up things like Python paths the way the engine
472    # needs them, and then we import and run the engine.
473    #
474    # Below we're doing a slightly fancier version of that. Namely, we
475    # do some processing of command line args to allow overriding of
476    # paths or running explicit commands or whatever else. Our goal is
477    # that this modular form of the app should be basically
478    # indistinguishable from the monolithic form when used from the
479    # command line.
480
481    try:
482        # Take note that we're running via modular-main. The native
483        # layer can key off this to know whether it should apply
484        # sys.argv or not.
485        _EnvGlobals.get().modular_main_called = True
486
487        # Deal with a few key things here ourself before even running
488        # configure.
489
490        # The extract_arg stuff below modifies this so we work with a
491        # copy.
492        args = sys.argv.copy()
493
494        # NOTE: We need to keep these arg long/short arg versions synced
495        # to those in core_config.cc. That code parses these same args
496        # (even if it doesn't handle them in our case) and will complain
497        # if unrecognized args come through.
498
499        # Our -c arg basically mirrors Python's -c arg. If we get that,
500        # simply exec it and return; no engine stuff.
501        command = extract_arg(args, ['--command', '-c'], is_dir=False)
502        if command is not None:
503            exec(command)  # pylint: disable=exec-used
504            return
505
506        config_dir = extract_arg(args, ['--config-dir', '-C'], is_dir=True)
507        data_dir = extract_arg(args, ['--data-dir', '-d'], is_dir=True)
508        mods_dir = extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
509
510        # We run configure() BEFORE importing babase. (part of its job
511        # is to wrangle paths which can affect where babase and
512        # everything else gets loaded from).
513        configure(
514            config_dir=config_dir,
515            data_dir=data_dir,
516            user_python_dir=mods_dir,
517        )
518
519        import babase
520
521        # The engine will have parsed and processed all other args as
522        # part of the above import. If there were errors or args such as
523        # --help which should lead to us immediately returning, do so.
524        code = babase.get_immediate_return_code()
525        if code is not None:
526            sys.exit(code)
527
528        # Aaaand we're off!
529        babase.app.run()
530
531    # Code wanting us to die with a clean error message instead of an
532    # ugly stack trace can raise one of these.
533    except CleanError as clean_exc:
534        clean_exc.pretty_print()
535        sys.exit(1)
536
537
538# Exec'ing this module directly will do a standard app run.
539if __name__ == '__main__':
540    _modular_main()