baenv

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Manage ballistica execution environment.
  4
  5This module is used to set up and/or check the global Python environment
  6before running a ballistica app. This includes things such as paths,
  7logging, and app-dirs. Because these things are global in nature, this
  8should be done before any ballistica modules are imported.
  9
 10This module can also be exec'ed directly to set up a default environment
 11and then run the app.
 12
 13Ballistica can be used without explicitly configuring the environment in
 14order to integrate it in arbitrary Python environments, but this may
 15cause some features to be disabled or behave differently than expected.
 16"""
 17from __future__ import annotations
 18
 19import os
 20import sys
 21import logging
 22from pathlib import Path
 23from dataclasses import dataclass
 24from typing import TYPE_CHECKING
 25import __main__
 26
 27if TYPE_CHECKING:
 28    from typing import Any
 29
 30    from efro.log import LogHandler
 31
 32# IMPORTANT - It is likely (and in some cases expected) that this
 33# module's code will be exec'ed multiple times. This is because it is
 34# the job of this module to set up Python paths for an engine run, and
 35# that may involve modifying sys.path in such a way that this module
 36# resolves to a different path afterwards (for example from
 37# /abs/path/to/ba_data/scripts/babase.py to ba_data/scripts/babase.py).
 38# This can result in the next import of baenv loading us from our 'new'
 39# location, which may or may not actually be the same file on disk as
 40# the last load. Either way, however, multiple execs will happen in some
 41# form.
 42#
 43# To handle that situation gracefully, we need to do a few things:
 44#
 45# - First, we need to store any mutable global state in the __main__
 46#   module; not in ourself. This way, alternate versions of ourself will
 47#   still know if we already ran configure/etc.
 48#
 49# - Second, we should avoid the use of isinstance and similar calls for
 50#   our types. An EnvConfig we create would technically be a different
 51#   type than an EnvConfig created by an alternate baenv.
 52
 53# Build number and version of the ballistica binary we expect to be
 54# using.
 55TARGET_BALLISTICA_BUILD = 22025
 56TARGET_BALLISTICA_VERSION = '1.7.37'
 57
 58
 59@dataclass
 60class EnvConfig:
 61    """Final config values we provide to the engine."""
 62
 63    # Where app config/state data lives.
 64    config_dir: str
 65
 66    # Directory containing ba_data and any other platform-specific data.
 67    data_dir: str
 68
 69    # Where the app's built-in Python stuff lives.
 70    app_python_dir: str | None
 71
 72    # Where the app's built-in Python stuff lives in the default case.
 73    standard_app_python_dir: str
 74
 75    # Where the app's bundled third party Python stuff lives.
 76    site_python_dir: str | None
 77
 78    # Custom Python provided by the user (mods).
 79    user_python_dir: str | None
 80
 81    # We have a mechanism allowing app scripts to be overridden by
 82    # placing a specially named directory in a user-scripts dir.
 83    # This is true if that is enabled.
 84    is_user_app_python_dir: bool
 85
 86    # Our fancy app log handler. This handles feeding logs, stdout, and
 87    # stderr into the engine so they show up on in-app consoles, etc.
 88    log_handler: LogHandler | None
 89
 90    # Initial data from the ballisticakit-config.json file. This is
 91    # passed mostly as an optimization to avoid reading the same config
 92    # file twice, since config data is first needed in baenv and next in
 93    # the engine. It will be cleared after passing it to the app's
 94    # config management subsystem and should not be accessed by any
 95    # other code.
 96    initial_app_config: Any
 97
 98
 99@dataclass
100class _EnvGlobals:
101    """Globals related to baenv's operation.
102
103    We store this in __main__ instead of in our own module because it
104    is likely that multiple versions of our module will be spun up
105    and we want a single set of globals (see notes at top of our module
106    code).
107    """
108
109    config: EnvConfig | None = None
110    called_configure: bool = False
111    paths_set_failed: bool = False
112    modular_main_called: bool = False
113
114    @classmethod
115    def get(cls) -> _EnvGlobals:
116        """Create/return our singleton."""
117        name = '_baenv_globals'
118        envglobals: _EnvGlobals | None = getattr(__main__, name, None)
119        if envglobals is None:
120            envglobals = _EnvGlobals()
121            setattr(__main__, name, envglobals)
122        return envglobals
123
124
125def did_paths_set_fail() -> bool:
126    """Did we try to set paths and fail?"""
127    return _EnvGlobals.get().paths_set_failed
128
129
130def config_exists() -> bool:
131    """Has a config been created?"""
132
133    return _EnvGlobals.get().config is not None
134
135
136def get_config() -> EnvConfig:
137    """Return the active config, creating a default if none exists."""
138    envglobals = _EnvGlobals.get()
139
140    # If configure() has not been explicitly called, set up a
141    # minimally-intrusive default config. We want Ballistica to default
142    # to being a good citizen when imported into alien environments and
143    # not blow away logging or otherwise muck with stuff. All official
144    # paths to run Ballistica apps should be explicitly calling
145    # configure() first to get a full featured setup.
146    if not envglobals.called_configure:
147        configure(setup_logging=False)
148
149    config = envglobals.config
150    if config is None:
151        raise RuntimeError(
152            'baenv.configure() has been called but no config exists;'
153            ' perhaps it errored?'
154        )
155    return config
156
157
158def configure(
159    *,
160    config_dir: str | None = None,
161    data_dir: str | None = None,
162    user_python_dir: str | None = None,
163    app_python_dir: str | None = None,
164    site_python_dir: str | None = None,
165    contains_python_dist: bool = False,
166    setup_logging: bool = True,
167) -> None:
168    """Set up the environment for running a Ballistica app.
169
170    This includes things such as Python path wrangling and app directory
171    creation. This must be called before any actual Ballistica modules
172    are imported; the environment is locked in as soon as that happens.
173    """
174
175    envglobals = _EnvGlobals.get()
176
177    # Keep track of whether we've been *called*, not whether a config
178    # has been created. Otherwise its possible to get multiple
179    # overlapping configure calls going.
180    if envglobals.called_configure:
181        raise RuntimeError(
182            'baenv.configure() has already been called;'
183            ' it can only be called once.'
184        )
185    envglobals.called_configure = True
186
187    # The very first thing we do is setup Python paths (while also
188    # calculating some engine paths). This code needs to be bulletproof
189    # since we have no logging yet at this point. We used to set up
190    # logging first, but this way logging stuff will get loaded from its
191    # proper final path (otherwise we might wind up using two different
192    # versions of efro.logging in a single engine run).
193    (
194        user_python_dir,
195        app_python_dir,
196        site_python_dir,
197        data_dir,
198        config_dir,
199        standard_app_python_dir,
200        is_user_app_python_dir,
201    ) = _setup_paths(
202        user_python_dir,
203        app_python_dir,
204        site_python_dir,
205        data_dir,
206        config_dir,
207    )
208
209    # The second thing we do is set up our logging system and pipe
210    # Python's stdout/stderr into it. At this point we can at least
211    # debug problems on systems where native stdout/stderr is not easily
212    # accessible such as Android.
213    log_handler = _setup_logging() if setup_logging else None
214
215    # We want to always be run in UTF-8 mode; complain if we're not.
216    if sys.flags.utf8_mode != 1:
217        logging.warning(
218            "Python's UTF-8 mode is not set. Running Ballistica without"
219            ' it may lead to errors.'
220        )
221
222    # Attempt to create dirs that we'll write stuff to.
223    _setup_dirs(config_dir, user_python_dir)
224
225    # Get ssl working if needed so we can use https and all that.
226    _setup_certs(contains_python_dist)
227
228    # This is now the active config.
229    envglobals.config = EnvConfig(
230        config_dir=config_dir,
231        data_dir=data_dir,
232        user_python_dir=user_python_dir,
233        app_python_dir=app_python_dir,
234        standard_app_python_dir=standard_app_python_dir,
235        site_python_dir=site_python_dir,
236        log_handler=log_handler,
237        is_user_app_python_dir=is_user_app_python_dir,
238        initial_app_config=None,
239    )
240
241
242def _calc_data_dir(data_dir: str | None) -> str:
243    if data_dir is None:
244        # To calc default data_dir, we assume this module was imported
245        # from that dir's ba_data/python subdir.
246        assert Path(__file__).parts[-3:-1] == ('ba_data', 'python')
247        data_dir_path = Path(__file__).parents[2]
248
249        # Prefer tidy relative paths like './ba_data' if possible so
250        # that things like stack traces are easier to read. For best
251        # results, platforms where CWD doesn't matter can chdir to where
252        # ba_data lives before calling configure().
253        #
254        # NOTE: If there's ever a case where the user is chdir'ing at
255        # runtime we might want an option to use only abs paths here.
256        cwd_path = Path.cwd()
257        data_dir = str(
258            data_dir_path.relative_to(cwd_path)
259            if data_dir_path.is_relative_to(cwd_path)
260            else data_dir_path
261        )
262    return data_dir
263
264
265def _setup_logging() -> LogHandler:
266    from efro.log import setup_logging, LogLevel
267
268    # TODO: should set this up with individual loggers under a top level
269    # 'ba' logger, and at that point we can kill off the
270    # suppress_non_root_debug option since we'll only ever need to set
271    # 'ba' to DEBUG at most.
272    log_handler = setup_logging(
273        log_path=None,
274        level=LogLevel.DEBUG,
275        suppress_non_root_debug=True,
276        log_stdout_stderr=True,
277        cache_size_limit=1024 * 1024,
278    )
279    return log_handler
280
281
282def _setup_certs(contains_python_dist: bool) -> None:
283    # In situations where we're bringing our own Python, let's also
284    # provide our own root certs so ssl works. We can consider
285    # overriding this in particular embedded cases if we can verify that
286    # system certs are working. We also allow forcing this via an env
287    # var if the user desires.
288    if (
289        contains_python_dist
290        or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'
291    ):
292        import certifi
293
294        # Let both OpenSSL and requests (if present) know to use this.
295        os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = (
296            certifi.where()
297        )
298
299
300def _setup_paths(
301    user_python_dir: str | None,
302    app_python_dir: str | None,
303    site_python_dir: str | None,
304    data_dir: str | None,
305    config_dir: str | None,
306) -> tuple[str | None, str | None, str | None, str, str, str, bool]:
307    # First a few paths we can ALWAYS calculate since they don't affect
308    # Python imports:
309
310    envglobals = _EnvGlobals.get()
311
312    data_dir = _calc_data_dir(data_dir)
313
314    # Default config-dir is simply ~/.ballisticakit
315    if config_dir is None:
316        config_dir = str(Path(Path.home(), '.ballisticakit'))
317
318    # Standard app-python-dir is simply ba_data/python under data-dir.
319    standard_app_python_dir = str(Path(data_dir, 'ba_data', 'python'))
320
321    # Whether the final app-dir we're returning is a custom user-owned one.
322    is_user_app_python_dir = False
323
324    # If _babase has already been imported, there's not much we can do
325    # at this point aside from complain and inform for next time.
326    if '_babase' in sys.modules:
327        app_python_dir = user_python_dir = site_python_dir = None
328
329        # We don't actually complain yet here; we simply take note that
330        # we weren't able to set paths. Then we complain if/when the app
331        # is started. This way, non-app uses of babase won't be filled
332        # with unnecessary warnings.
333        envglobals.paths_set_failed = True
334
335    else:
336        # Ok; _babase hasn't been imported yet, so we can muck with
337        # Python paths.
338
339        if app_python_dir is None:
340            app_python_dir = standard_app_python_dir
341
342        # Likewise site-python-dir defaults to ba_data/python-site-packages.
343        if site_python_dir is None:
344            site_python_dir = str(
345                Path(data_dir, 'ba_data', 'python-site-packages')
346            )
347
348        # By default, user-python-dir is simply 'mods' under config-dir.
349        if user_python_dir is None:
350            user_python_dir = str(Path(config_dir, 'mods'))
351
352        # Wherever our user_python_dir is, if we find a sys/FOO_BAR dir
353        # under it where FOO matches our version and BAR matches our
354        # build number, use that as our app_python_dir. This allows
355        # modding built-in stuff on platforms where there is no write
356        # access to said built-in stuff.
357        check_dir = Path(
358            user_python_dir,
359            'sys',
360            f'{TARGET_BALLISTICA_VERSION}_{TARGET_BALLISTICA_BUILD}',
361        )
362        try:
363            if check_dir.is_dir():
364                app_python_dir = str(check_dir)
365                is_user_app_python_dir = True
366        except PermissionError:
367            logging.warning(
368                "PermissionError checking user-app-python-dir path '%s'.",
369                check_dir,
370            )
371
372        # Ok, now apply these to sys.path.
373
374        # First off, strip out any instances of the path containing this
375        # module. We will *probably* be re-adding the same path in a
376        # moment so this keeps things cleaner. Though hmm should we
377        # leave it in there in cases where we *don't* re-add the same
378        # path?...
379        our_parent_path = Path(__file__).parent.resolve()
380        oldpaths: list[str] = [
381            p for p in sys.path if Path(p).resolve() != our_parent_path
382        ]
383
384        # Let's place mods first (so users can override whatever they
385        # want) followed by our app scripts and lastly our bundled site
386        # stuff.
387
388        # One could make the argument that at least our bundled app &
389        # site stuff should be placed at the end so actual local site
390        # stuff could override it. That could be a good thing or a bad
391        # thing. Maybe we could add an option for that, but for now I'm
392        # prioritizing our stuff to give as consistent an environment as
393        # possible.
394        ourpaths = [user_python_dir, app_python_dir, site_python_dir]
395
396        # Special case: our modular builds will have a 'python-dylib'
397        # dir alongside the 'python' scripts dir which contains our
398        # binary Python modules. If we see that, add it to the path also.
399        # Not sure if we'd ever have a need to customize this path.
400        dylibdir = f'{app_python_dir}-dylib'
401        if os.path.exists(dylibdir):
402            ourpaths.append(dylibdir)
403
404        sys.path = ourpaths + oldpaths
405
406    return (
407        user_python_dir,
408        app_python_dir,
409        site_python_dir,
410        data_dir,
411        config_dir,
412        standard_app_python_dir,
413        is_user_app_python_dir,
414    )
415
416
417def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
418    create_dirs: list[tuple[str, str | None]] = [
419        ('config', config_dir),
420        ('user_python', user_python_dir),
421    ]
422    for cdirname, cdir in create_dirs:
423        if cdir is not None:
424            try:
425                os.makedirs(cdir, exist_ok=True)
426            except Exception:
427                # Not the end of the world if we can't make these dirs.
428                logging.warning(
429                    "Unable to create %s dir at '%s'.", cdirname, cdir
430                )
431
432
433def extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
434    """Given a list of args and an arg name, returns a value.
435
436    The arg flag and value are removed from the arg list. We also check
437    to make sure the path exists.
438
439    raises CleanErrors on any problems.
440    """
441    from efro.error import CleanError
442
443    count = sum(args.count(n) for n in names)
444    if not count:
445        return None
446
447    if count > 1:
448        raise CleanError(f'Arg {names} passed multiple times.')
449
450    for name in names:
451        if name not in args:
452            continue
453        argindex = args.index(name)
454        if argindex + 1 >= len(args):
455            raise CleanError(f'No value passed after {name} arg.')
456
457        val = args[argindex + 1]
458        del args[argindex : argindex + 2]
459
460        if is_dir and not os.path.isdir(val):
461            namepretty = names[0].removeprefix('--')
462            raise CleanError(
463                f"Provided {namepretty} path '{val}' is not a directory."
464            )
465        return val
466
467    raise RuntimeError(f'Expected arg name not found from {names}')
468
469
470def _modular_main() -> None:
471    from efro.error import CleanError
472
473    # Fundamentally, running a Ballistica app consists of the following:
474    # import baenv; baenv.configure(); import babase; babase.app.run()
475    #
476    # First baenv sets up things like Python paths the way the engine
477    # needs them, and then we import and run the engine.
478    #
479    # Below we're doing a slightly fancier version of that. Namely, we
480    # do some processing of command line args to allow overriding of
481    # paths or running explicit commands or whatever else. Our goal is
482    # that this modular form of the app should be basically
483    # indistinguishable from the monolithic form when used from the
484    # command line.
485
486    try:
487        # Take note that we're running via modular-main. The native
488        # layer can key off this to know whether it should apply
489        # sys.argv or not.
490        _EnvGlobals.get().modular_main_called = True
491
492        # Deal with a few key things here ourself before even running
493        # configure.
494
495        # The extract_arg stuff below modifies this so we work with a
496        # copy.
497        args = sys.argv.copy()
498
499        # NOTE: We need to keep these arg long/short arg versions synced
500        # to those in core_config.cc. That code parses these same args
501        # (even if it doesn't handle them in our case) and will complain
502        # if unrecognized args come through.
503
504        # Our -c arg basically mirrors Python's -c arg. If we get that,
505        # simply exec it and return; no engine stuff.
506        command = extract_arg(args, ['--command', '-c'], is_dir=False)
507        if command is not None:
508            exec(command)  # pylint: disable=exec-used
509            return
510
511        config_dir = extract_arg(args, ['--config-dir', '-C'], is_dir=True)
512        data_dir = extract_arg(args, ['--data-dir', '-d'], is_dir=True)
513        mods_dir = extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
514
515        # We run configure() BEFORE importing babase. (part of its job
516        # is to wrangle paths which can affect where babase and
517        # everything else gets loaded from).
518        configure(
519            config_dir=config_dir,
520            data_dir=data_dir,
521            user_python_dir=mods_dir,
522        )
523
524        import babase
525
526        # The engine will have parsed and processed all other args as
527        # part of the above import. If there were errors or args such as
528        # --help which should lead to us immediately returning, do so.
529        code = babase.get_immediate_return_code()
530        if code is not None:
531            sys.exit(code)
532
533        # Aaaand we're off!
534        babase.app.run()
535
536    # Code wanting us to die with a clean error message instead of an
537    # ugly stack trace can raise one of these.
538    except CleanError as clean_exc:
539        clean_exc.pretty_print()
540        sys.exit(1)
541
542
543# Exec'ing this module directly will do a standard app run.
544if __name__ == '__main__':
545    _modular_main()