baenv

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Manage ballistica execution environment.
  4
  5This module is used to set up and/or check the global Python environment
  6before running a ballistica app. This includes things such as paths,
  7logging, and app-dirs. Because these things are global in nature, this
  8should be done before any ballistica modules are imported.
  9
 10This module can also be exec'ed directly to set up a default environment
 11and then run the app.
 12
 13Ballistica can be used without explicitly configuring the environment in
 14order to integrate it in arbitrary Python environments, but this may
 15cause some features to be disabled or behave differently than expected.
 16"""
 17from __future__ import annotations
 18
 19import os
 20import sys
 21import logging
 22from pathlib import Path
 23from dataclasses import dataclass
 24from typing import TYPE_CHECKING
 25import __main__
 26
 27if TYPE_CHECKING:
 28    from typing import Any
 29
 30    from efro.log import LogHandler
 31
 32# IMPORTANT - It is likely (and in some cases expected) that this
 33# module's code will be exec'ed multiple times. This is because it is
 34# the job of this module to set up Python paths for an engine run, and
 35# that may involve modifying sys.path in such a way that this module
 36# resolves to a different path afterwards (for example from
 37# /abs/path/to/ba_data/scripts/babase.py to ba_data/scripts/babase.py).
 38# This can result in the next import of baenv loading us from our 'new'
 39# location, which may or may not actually be the same file on disk as
 40# the last load. Either way, however, multiple execs will happen in some
 41# form.
 42#
 43# So we need to do a few things to handle that situation gracefully.
 44#
 45# - First, we need to store any mutable global state in the __main__
 46#   module; not in ourself. This way, alternate versions of ourself will
 47#   still know if we already ran configure/etc.
 48#
 49# - Second, we should avoid the use of isinstance and similar calls for
 50#   our types. An EnvConfig we create would technically be a different
 51#   type than that created by an alternate baenv.
 52
 53# Build number and version of the ballistica binary we expect to be
 54# using.
 55TARGET_BALLISTICA_BUILD = 21397
 56TARGET_BALLISTICA_VERSION = '1.7.28'
 57
 58
 59@dataclass
 60class EnvConfig:
 61    """Final config values we provide to the engine."""
 62
 63    # Where app config/state data lives.
 64    config_dir: str
 65
 66    # Directory containing ba_data and any other platform-specific data.
 67    data_dir: str
 68
 69    # Where the app's built-in Python stuff lives.
 70    app_python_dir: str | None
 71
 72    # Where the app's built-in Python stuff lives in the default case.
 73    standard_app_python_dir: str
 74
 75    # Where the app's bundled third party Python stuff lives.
 76    site_python_dir: str | None
 77
 78    # Custom Python provided by the user (mods).
 79    user_python_dir: str | None
 80
 81    # We have a mechanism allowing app scripts to be overridden by
 82    # placing a specially named directory in a user-scripts dir.
 83    # This is true if that is enabled.
 84    is_user_app_python_dir: bool
 85
 86    # Our fancy app log handler. This handles feeding logs, stdout, and
 87    # stderr into the engine so they show up on in-app consoles, etc.
 88    log_handler: LogHandler | None
 89
 90    # Initial data from the ballisticakit-config.json file. This is
 91    # passed mostly as an optimization to avoid reading the same config
 92    # file twice, since config data is first needed in baenv and next in
 93    # the engine. It will be cleared after passing it to the app's
 94    # config management subsystem and should not be accessed by any
 95    # other code.
 96    initial_app_config: Any
 97
 98
 99@dataclass
100class _EnvGlobals:
101    """Globals related to baenv's operation.
102
103    We store this in __main__ instead of in our own module because it
104    is likely that multiple versions of our module will be spun up
105    and we want a single set of globals (see notes at top of our module
106    code).
107    """
108
109    config: EnvConfig | None = None
110    called_configure: bool = False
111    paths_set_failed: bool = False
112    modular_main_called: bool = False
113
114    @classmethod
115    def get(cls) -> _EnvGlobals:
116        """Create/return our singleton."""
117        name = '_baenv_globals'
118        envglobals: _EnvGlobals | None = getattr(__main__, name, None)
119        if envglobals is None:
120            envglobals = _EnvGlobals()
121            setattr(__main__, name, envglobals)
122        return envglobals
123
124
125def did_paths_set_fail() -> bool:
126    """Did we try to set paths and fail?"""
127    return _EnvGlobals.get().paths_set_failed
128
129
130def config_exists() -> bool:
131    """Has a config been created?"""
132
133    return _EnvGlobals.get().config is not None
134
135
136def get_config() -> EnvConfig:
137    """Return the active config, creating a default if none exists."""
138    envglobals = _EnvGlobals.get()
139
140    # If configure() has not been explicitly called, set up a
141    # minimally-intrusive default config. We want Ballistica to default
142    # to being a good citizen when imported into alien environments and
143    # not blow away logging or otherwise muck with stuff. All official
144    # paths to run Ballistica apps should be explicitly calling
145    # configure() first to get a full featured setup.
146    if not envglobals.called_configure:
147        configure(setup_logging=False)
148
149    config = envglobals.config
150    if config is None:
151        raise RuntimeError(
152            'baenv.configure() has been called but no config exists;'
153            ' perhaps it errored?'
154        )
155    return config
156
157
158def configure(
159    config_dir: str | None = None,
160    data_dir: str | None = None,
161    user_python_dir: str | None = None,
162    app_python_dir: str | None = None,
163    site_python_dir: str | None = None,
164    contains_python_dist: bool = False,
165    setup_logging: bool = True,
166) -> None:
167    """Set up the environment for running a Ballistica app.
168
169    This includes things such as Python path wrangling and app directory
170    creation. This must be called before any actual Ballistica modules
171    are imported; the environment is locked in as soon as that happens.
172    """
173
174    envglobals = _EnvGlobals.get()
175
176    # Keep track of whether we've been *called*, not whether a config
177    # has been created. Otherwise its possible to get multiple
178    # overlapping configure calls going.
179    if envglobals.called_configure:
180        raise RuntimeError(
181            'baenv.configure() has already been called;'
182            ' it can only be called once.'
183        )
184    envglobals.called_configure = True
185
186    # The very first thing we do is setup Python paths (while also
187    # calculating some engine paths). This code needs to be bulletproof
188    # since we have no logging yet at this point. We used to set up
189    # logging first, but this way logging stuff will get loaded from its
190    # proper final path (otherwise we might wind up using two different
191    # versions of efro.logging in a single engine run).
192    (
193        user_python_dir,
194        app_python_dir,
195        site_python_dir,
196        data_dir,
197        config_dir,
198        standard_app_python_dir,
199        is_user_app_python_dir,
200    ) = _setup_paths(
201        user_python_dir,
202        app_python_dir,
203        site_python_dir,
204        data_dir,
205        config_dir,
206    )
207
208    # The second thing we do is set up our logging system and pipe
209    # Python's stdout/stderr into it. At this point we can at least
210    # debug problems on systems where native stdout/stderr is not easily
211    # accessible such as Android.
212    log_handler = _setup_logging() if setup_logging else None
213
214    # We want to always be run in UTF-8 mode; complain if we're not.
215    if sys.flags.utf8_mode != 1:
216        logging.warning(
217            "Python's UTF-8 mode is not set. Running Ballistica without"
218            ' it may lead to errors.'
219        )
220
221    # Attempt to create dirs that we'll write stuff to.
222    _setup_dirs(config_dir, user_python_dir)
223
224    # Get ssl working if needed so we can use https and all that.
225    _setup_certs(contains_python_dist)
226
227    # This is now the active config.
228    envglobals.config = EnvConfig(
229        config_dir=config_dir,
230        data_dir=data_dir,
231        user_python_dir=user_python_dir,
232        app_python_dir=app_python_dir,
233        standard_app_python_dir=standard_app_python_dir,
234        site_python_dir=site_python_dir,
235        log_handler=log_handler,
236        is_user_app_python_dir=is_user_app_python_dir,
237        initial_app_config=None,
238    )
239
240
241def _calc_data_dir(data_dir: str | None) -> str:
242    if data_dir is None:
243        # To calc default data_dir, we assume this module was imported
244        # from that dir's ba_data/python subdir.
245        assert Path(__file__).parts[-3:-1] == ('ba_data', 'python')
246        data_dir_path = Path(__file__).parents[2]
247
248        # Prefer tidy relative paths like './ba_data' if possible so
249        # that things like stack traces are easier to read. For best
250        # results, platforms where CWD doesn't matter can chdir to where
251        # ba_data lives before calling configure().
252        #
253        # NOTE: If there's ever a case where the user is chdir'ing at
254        # runtime we might want an option to use only abs paths here.
255        cwd_path = Path.cwd()
256        data_dir = str(
257            data_dir_path.relative_to(cwd_path)
258            if data_dir_path.is_relative_to(cwd_path)
259            else data_dir_path
260        )
261    return data_dir
262
263
264def _setup_logging() -> LogHandler:
265    from efro.log import setup_logging, LogLevel
266
267    log_handler = setup_logging(
268        log_path=None,
269        level=LogLevel.DEBUG,
270        suppress_non_root_debug=True,
271        log_stdout_stderr=True,
272        cache_size_limit=1024 * 1024,
273    )
274    return log_handler
275
276
277def _setup_certs(contains_python_dist: bool) -> None:
278    # In situations where we're bringing our own Python, let's also
279    # provide our own root certs so ssl works. We can consider
280    # overriding this in particular embedded cases if we can verify that
281    # system certs are working. We also allow forcing this via an env
282    # var if the user desires.
283    if (
284        contains_python_dist
285        or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'
286    ):
287        import certifi
288
289        # Let both OpenSSL and requests (if present) know to use this.
290        os.environ['SSL_CERT_FILE'] = os.environ[
291            'REQUESTS_CA_BUNDLE'
292        ] = certifi.where()
293
294
295def _setup_paths(
296    user_python_dir: str | None,
297    app_python_dir: str | None,
298    site_python_dir: str | None,
299    data_dir: str | None,
300    config_dir: str | None,
301) -> tuple[str | None, str | None, str | None, str, str, str, bool]:
302    # First a few paths we can ALWAYS calculate since they don't affect
303    # Python imports:
304
305    envglobals = _EnvGlobals.get()
306
307    data_dir = _calc_data_dir(data_dir)
308
309    # Default config-dir is simply ~/.ballisticakit
310    if config_dir is None:
311        config_dir = str(Path(Path.home(), '.ballisticakit'))
312
313    # Standard app-python-dir is simply ba_data/python under data-dir.
314    standard_app_python_dir = str(Path(data_dir, 'ba_data', 'python'))
315
316    # Whether the final app-dir we're returning is a custom user-owned one.
317    is_user_app_python_dir = False
318
319    # If _babase has already been imported, there's not much we can do
320    # at this point aside from complain and inform for next time.
321    if '_babase' in sys.modules:
322        app_python_dir = user_python_dir = site_python_dir = None
323
324        # We don't actually complain yet here; we simply take note that
325        # we weren't able to set paths. Then we complain if/when the app
326        # is started. This way, non-app uses of babase won't be filled
327        # with unnecessary warnings.
328        envglobals.paths_set_failed = True
329
330    else:
331        # Ok; _babase hasn't been imported yet, so we can muck with
332        # Python paths.
333
334        if app_python_dir is None:
335            app_python_dir = standard_app_python_dir
336
337        # Likewise site-python-dir defaults to ba_data/python-site-packages.
338        if site_python_dir is None:
339            site_python_dir = str(
340                Path(data_dir, 'ba_data', 'python-site-packages')
341            )
342
343        # By default, user-python-dir is simply 'mods' under config-dir.
344        if user_python_dir is None:
345            user_python_dir = str(Path(config_dir, 'mods'))
346
347        # Wherever our user_python_dir is, if we find a sys/FOO dir
348        # under it where FOO matches our version, use that as our
349        # app_python_dir. This allows modding built-in stuff on
350        # platforms where there is no write access to said built-in
351        # stuff.
352        check_dir = Path(user_python_dir, 'sys', TARGET_BALLISTICA_VERSION)
353        if check_dir.is_dir():
354            app_python_dir = str(check_dir)
355            is_user_app_python_dir = True
356
357        # Ok, now apply these to sys.path.
358
359        # First off, strip out any instances of the path containing this
360        # module. We will *probably* be re-adding the same path in a
361        # moment so this keeps things cleaner. Though hmm should we
362        # leave it in there in cases where we *don't* re-add the same
363        # path?...
364        our_parent_path = Path(__file__).parent.resolve()
365        oldpaths: list[str] = [
366            p for p in sys.path if Path(p).resolve() != our_parent_path
367        ]
368
369        # Let's place mods first (so users can override whatever they
370        # want) followed by our app scripts and lastly our bundled site
371        # stuff.
372
373        # One could make the argument that at least our bundled app &
374        # site stuff should be placed at the end so actual local site
375        # stuff could override it. That could be a good thing or a bad
376        # thing. Maybe we could add an option for that, but for now I'm
377        # prioritizing our stuff to give as consistent an environment as
378        # possible.
379        ourpaths = [user_python_dir, app_python_dir, site_python_dir]
380
381        # Special case: our modular builds will have a 'python-dylib'
382        # dir alongside the 'python' scripts dir which contains our
383        # binary Python modules. If we see that, add it to the path also.
384        # Not sure if we'd ever have a need to customize this path.
385        dylibdir = f'{app_python_dir}-dylib'
386        if os.path.exists(dylibdir):
387            ourpaths.append(dylibdir)
388
389        sys.path = ourpaths + oldpaths
390
391    return (
392        user_python_dir,
393        app_python_dir,
394        site_python_dir,
395        data_dir,
396        config_dir,
397        standard_app_python_dir,
398        is_user_app_python_dir,
399    )
400
401
402def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
403    create_dirs: list[tuple[str, str | None]] = [
404        ('config', config_dir),
405        ('user_python', user_python_dir),
406    ]
407    for cdirname, cdir in create_dirs:
408        if cdir is not None:
409            try:
410                os.makedirs(cdir, exist_ok=True)
411            except Exception:
412                # Not the end of the world if we can't make these dirs.
413                logging.warning(
414                    "Unable to create %s dir at '%s'.", cdirname, cdir
415                )
416
417
418def extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
419    """Given a list of args and an arg name, returns a value.
420
421    The arg flag and value are removed from the arg list. We also check
422    to make sure the path exists.
423
424    raises CleanErrors on any problems.
425    """
426    from efro.error import CleanError
427
428    count = sum(args.count(n) for n in names)
429    if not count:
430        return None
431
432    if count > 1:
433        raise CleanError(f'Arg {names} passed multiple times.')
434
435    for name in names:
436        if name not in args:
437            continue
438        argindex = args.index(name)
439        if argindex + 1 >= len(args):
440            raise CleanError(f'No value passed after {name} arg.')
441
442        val = args[argindex + 1]
443        del args[argindex : argindex + 2]
444
445        if is_dir and not os.path.isdir(val):
446            namepretty = names[0].removeprefix('--')
447            raise CleanError(
448                f"Provided {namepretty} path '{val}' is not a directory."
449            )
450        return val
451
452    raise RuntimeError(f'Expected arg name not found from {names}')
453
454
455def _modular_main() -> None:
456    from efro.error import CleanError
457
458    # Fundamentally, running a Ballistica app consists of the following:
459    # import baenv; baenv.configure(); import babase; babase.app.run()
460    #
461    # First baenv sets up things like Python paths the way the engine
462    # needs them, and then we import and run the engine.
463    #
464    # Below we're doing a slightly fancier version of that. Namely, we
465    # do some processing of command line args to allow overriding of
466    # paths or running explicit commands or whatever else. Our goal is
467    # that this modular form of the app should be basically
468    # indistinguishable from the monolithic form when used from the
469    # command line.
470
471    try:
472        # Take note that we're running via modular-main. The native
473        # layer can key off this to know whether it should apply
474        # sys.argv or not.
475        _EnvGlobals.get().modular_main_called = True
476
477        # Deal with a few key things here ourself before even running
478        # configure.
479
480        # The extract_arg stuff below modifies this so we work with a
481        # copy.
482        args = sys.argv.copy()
483
484        # NOTE: We need to keep these arg long/short arg versions synced
485        # to those in core_config.cc. That code parses these same args
486        # (even if it doesn't handle them in our case) and will complain
487        # if unrecognized args come through.
488
489        # Our -c arg basically mirrors Python's -c arg. If we get that,
490        # simply exec it and return; no engine stuff.
491        command = extract_arg(args, ['--command', '-c'], is_dir=False)
492        if command is not None:
493            exec(command)  # pylint: disable=exec-used
494            return
495
496        config_dir = extract_arg(args, ['--config-dir', '-C'], is_dir=True)
497        data_dir = extract_arg(args, ['--data-dir', '-d'], is_dir=True)
498        mods_dir = extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
499
500        # We run configure() BEFORE importing babase. (part of its job
501        # is to wrangle paths which can affect where babase and
502        # everything else gets loaded from).
503        configure(
504            config_dir=config_dir,
505            data_dir=data_dir,
506            user_python_dir=mods_dir,
507        )
508
509        import babase
510
511        # The engine will have parsed and processed all other args as
512        # part of the above import. If there were errors or args such as
513        # --help which should lead to us immediately returning, do so.
514        code = babase.get_immediate_return_code()
515        if code is not None:
516            sys.exit(code)
517
518        # Aaaand we're off!
519        babase.app.run()
520
521    # Code wanting us to die with a clean error message instead of an
522    # ugly stack trace can raise one of these.
523    except CleanError as clean_exc:
524        clean_exc.pretty_print()
525        sys.exit(1)
526
527
528# Exec'ing this module directly will do a standard app run.
529if __name__ == '__main__':
530    _modular_main()