baenv

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Manage ballistica execution environment.
  4
  5This module is used to set up and/or check the global Python environment
  6before running a ballistica app. This includes things such as paths,
  7logging, and app-dirs. Because these things are global in nature, this
  8should be done before any ballistica modules are imported.
  9
 10This module can also be exec'ed directly to set up a default environment
 11and then run the app.
 12
 13Ballistica can be used without explicitly configuring the environment in
 14order to integrate it in arbitrary Python environments, but this may
 15cause some features to be disabled or behave differently than expected.
 16"""
 17from __future__ import annotations
 18
 19import os
 20import sys
 21import logging
 22from pathlib import Path
 23from dataclasses import dataclass
 24from typing import TYPE_CHECKING
 25import __main__
 26
 27if TYPE_CHECKING:
 28    from typing import Any
 29
 30    from efro.log import LogHandler
 31
 32# IMPORTANT - It is likely (and in some cases expected) that this
 33# module's code will be exec'ed multiple times. This is because it is
 34# the job of this module to set up Python paths for an engine run, and
 35# that may involve modifying sys.path in such a way that this module
 36# resolves to a different path afterwards (for example from
 37# /abs/path/to/ba_data/scripts/babase.py to ba_data/scripts/babase.py).
 38# This can result in the next import of baenv loading us from our 'new'
 39# location, which may or may not actually be the same file on disk as
 40# the last load. Either way, however, multiple execs will happen in some
 41# form.
 42#
 43# To handle that situation gracefully, we need to do a few things:
 44#
 45# - First, we need to store any mutable global state in the __main__
 46#   module; not in ourself. This way, alternate versions of ourself will
 47#   still know if we already ran configure/etc.
 48#
 49# - Second, we should avoid the use of isinstance and similar calls for
 50#   our types. An EnvConfig we create would technically be a different
 51#   type than an EnvConfig created by an alternate baenv.
 52
 53# Build number and version of the ballistica binary we expect to be
 54# using.
 55TARGET_BALLISTICA_BUILD = 21968
 56TARGET_BALLISTICA_VERSION = '1.7.37'
 57
 58
 59@dataclass
 60class EnvConfig:
 61    """Final config values we provide to the engine."""
 62
 63    # Where app config/state data lives.
 64    config_dir: str
 65
 66    # Directory containing ba_data and any other platform-specific data.
 67    data_dir: str
 68
 69    # Where the app's built-in Python stuff lives.
 70    app_python_dir: str | None
 71
 72    # Where the app's built-in Python stuff lives in the default case.
 73    standard_app_python_dir: str
 74
 75    # Where the app's bundled third party Python stuff lives.
 76    site_python_dir: str | None
 77
 78    # Custom Python provided by the user (mods).
 79    user_python_dir: str | None
 80
 81    # We have a mechanism allowing app scripts to be overridden by
 82    # placing a specially named directory in a user-scripts dir.
 83    # This is true if that is enabled.
 84    is_user_app_python_dir: bool
 85
 86    # Our fancy app log handler. This handles feeding logs, stdout, and
 87    # stderr into the engine so they show up on in-app consoles, etc.
 88    log_handler: LogHandler | None
 89
 90    # Initial data from the ballisticakit-config.json file. This is
 91    # passed mostly as an optimization to avoid reading the same config
 92    # file twice, since config data is first needed in baenv and next in
 93    # the engine. It will be cleared after passing it to the app's
 94    # config management subsystem and should not be accessed by any
 95    # other code.
 96    initial_app_config: Any
 97
 98
 99@dataclass
100class _EnvGlobals:
101    """Globals related to baenv's operation.
102
103    We store this in __main__ instead of in our own module because it
104    is likely that multiple versions of our module will be spun up
105    and we want a single set of globals (see notes at top of our module
106    code).
107    """
108
109    config: EnvConfig | None = None
110    called_configure: bool = False
111    paths_set_failed: bool = False
112    modular_main_called: bool = False
113
114    @classmethod
115    def get(cls) -> _EnvGlobals:
116        """Create/return our singleton."""
117        name = '_baenv_globals'
118        envglobals: _EnvGlobals | None = getattr(__main__, name, None)
119        if envglobals is None:
120            envglobals = _EnvGlobals()
121            setattr(__main__, name, envglobals)
122        return envglobals
123
124
125def did_paths_set_fail() -> bool:
126    """Did we try to set paths and fail?"""
127    return _EnvGlobals.get().paths_set_failed
128
129
130def config_exists() -> bool:
131    """Has a config been created?"""
132
133    return _EnvGlobals.get().config is not None
134
135
136def get_config() -> EnvConfig:
137    """Return the active config, creating a default if none exists."""
138    envglobals = _EnvGlobals.get()
139
140    # If configure() has not been explicitly called, set up a
141    # minimally-intrusive default config. We want Ballistica to default
142    # to being a good citizen when imported into alien environments and
143    # not blow away logging or otherwise muck with stuff. All official
144    # paths to run Ballistica apps should be explicitly calling
145    # configure() first to get a full featured setup.
146    if not envglobals.called_configure:
147        configure(setup_logging=False)
148
149    config = envglobals.config
150    if config is None:
151        raise RuntimeError(
152            'baenv.configure() has been called but no config exists;'
153            ' perhaps it errored?'
154        )
155    return config
156
157
158def configure(
159    config_dir: str | None = None,
160    data_dir: str | None = None,
161    user_python_dir: str | None = None,
162    app_python_dir: str | None = None,
163    site_python_dir: str | None = None,
164    contains_python_dist: bool = False,
165    setup_logging: bool = True,
166) -> None:
167    """Set up the environment for running a Ballistica app.
168
169    This includes things such as Python path wrangling and app directory
170    creation. This must be called before any actual Ballistica modules
171    are imported; the environment is locked in as soon as that happens.
172    """
173
174    envglobals = _EnvGlobals.get()
175
176    # Keep track of whether we've been *called*, not whether a config
177    # has been created. Otherwise its possible to get multiple
178    # overlapping configure calls going.
179    if envglobals.called_configure:
180        raise RuntimeError(
181            'baenv.configure() has already been called;'
182            ' it can only be called once.'
183        )
184    envglobals.called_configure = True
185
186    # The very first thing we do is setup Python paths (while also
187    # calculating some engine paths). This code needs to be bulletproof
188    # since we have no logging yet at this point. We used to set up
189    # logging first, but this way logging stuff will get loaded from its
190    # proper final path (otherwise we might wind up using two different
191    # versions of efro.logging in a single engine run).
192    (
193        user_python_dir,
194        app_python_dir,
195        site_python_dir,
196        data_dir,
197        config_dir,
198        standard_app_python_dir,
199        is_user_app_python_dir,
200    ) = _setup_paths(
201        user_python_dir,
202        app_python_dir,
203        site_python_dir,
204        data_dir,
205        config_dir,
206    )
207
208    # The second thing we do is set up our logging system and pipe
209    # Python's stdout/stderr into it. At this point we can at least
210    # debug problems on systems where native stdout/stderr is not easily
211    # accessible such as Android.
212    log_handler = _setup_logging() if setup_logging else None
213
214    # We want to always be run in UTF-8 mode; complain if we're not.
215    if sys.flags.utf8_mode != 1:
216        logging.warning(
217            "Python's UTF-8 mode is not set. Running Ballistica without"
218            ' it may lead to errors.'
219        )
220
221    # Attempt to create dirs that we'll write stuff to.
222    _setup_dirs(config_dir, user_python_dir)
223
224    # Get ssl working if needed so we can use https and all that.
225    _setup_certs(contains_python_dist)
226
227    # This is now the active config.
228    envglobals.config = EnvConfig(
229        config_dir=config_dir,
230        data_dir=data_dir,
231        user_python_dir=user_python_dir,
232        app_python_dir=app_python_dir,
233        standard_app_python_dir=standard_app_python_dir,
234        site_python_dir=site_python_dir,
235        log_handler=log_handler,
236        is_user_app_python_dir=is_user_app_python_dir,
237        initial_app_config=None,
238    )
239
240
241def _calc_data_dir(data_dir: str | None) -> str:
242    if data_dir is None:
243        # To calc default data_dir, we assume this module was imported
244        # from that dir's ba_data/python subdir.
245        assert Path(__file__).parts[-3:-1] == ('ba_data', 'python')
246        data_dir_path = Path(__file__).parents[2]
247
248        # Prefer tidy relative paths like './ba_data' if possible so
249        # that things like stack traces are easier to read. For best
250        # results, platforms where CWD doesn't matter can chdir to where
251        # ba_data lives before calling configure().
252        #
253        # NOTE: If there's ever a case where the user is chdir'ing at
254        # runtime we might want an option to use only abs paths here.
255        cwd_path = Path.cwd()
256        data_dir = str(
257            data_dir_path.relative_to(cwd_path)
258            if data_dir_path.is_relative_to(cwd_path)
259            else data_dir_path
260        )
261    return data_dir
262
263
264def _setup_logging() -> LogHandler:
265    from efro.log import setup_logging, LogLevel
266
267    # TODO: should set this up with individual loggers under a top level
268    # 'ba' logger, and at that point we can kill off the
269    # suppress_non_root_debug option since we'll only ever need to set
270    # 'ba' to DEBUG at most.
271    log_handler = setup_logging(
272        log_path=None,
273        level=LogLevel.DEBUG,
274        suppress_non_root_debug=True,
275        log_stdout_stderr=True,
276        cache_size_limit=1024 * 1024,
277    )
278    return log_handler
279
280
281def _setup_certs(contains_python_dist: bool) -> None:
282    # In situations where we're bringing our own Python, let's also
283    # provide our own root certs so ssl works. We can consider
284    # overriding this in particular embedded cases if we can verify that
285    # system certs are working. We also allow forcing this via an env
286    # var if the user desires.
287    if (
288        contains_python_dist
289        or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'
290    ):
291        import certifi
292
293        # Let both OpenSSL and requests (if present) know to use this.
294        os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = (
295            certifi.where()
296        )
297
298
299def _setup_paths(
300    user_python_dir: str | None,
301    app_python_dir: str | None,
302    site_python_dir: str | None,
303    data_dir: str | None,
304    config_dir: str | None,
305) -> tuple[str | None, str | None, str | None, str, str, str, bool]:
306    # First a few paths we can ALWAYS calculate since they don't affect
307    # Python imports:
308
309    envglobals = _EnvGlobals.get()
310
311    data_dir = _calc_data_dir(data_dir)
312
313    # Default config-dir is simply ~/.ballisticakit
314    if config_dir is None:
315        config_dir = str(Path(Path.home(), '.ballisticakit'))
316
317    # Standard app-python-dir is simply ba_data/python under data-dir.
318    standard_app_python_dir = str(Path(data_dir, 'ba_data', 'python'))
319
320    # Whether the final app-dir we're returning is a custom user-owned one.
321    is_user_app_python_dir = False
322
323    # If _babase has already been imported, there's not much we can do
324    # at this point aside from complain and inform for next time.
325    if '_babase' in sys.modules:
326        app_python_dir = user_python_dir = site_python_dir = None
327
328        # We don't actually complain yet here; we simply take note that
329        # we weren't able to set paths. Then we complain if/when the app
330        # is started. This way, non-app uses of babase won't be filled
331        # with unnecessary warnings.
332        envglobals.paths_set_failed = True
333
334    else:
335        # Ok; _babase hasn't been imported yet, so we can muck with
336        # Python paths.
337
338        if app_python_dir is None:
339            app_python_dir = standard_app_python_dir
340
341        # Likewise site-python-dir defaults to ba_data/python-site-packages.
342        if site_python_dir is None:
343            site_python_dir = str(
344                Path(data_dir, 'ba_data', 'python-site-packages')
345            )
346
347        # By default, user-python-dir is simply 'mods' under config-dir.
348        if user_python_dir is None:
349            user_python_dir = str(Path(config_dir, 'mods'))
350
351        # Wherever our user_python_dir is, if we find a sys/FOO_BAR dir
352        # under it where FOO matches our version and BAR matches our
353        # build number, use that as our app_python_dir. This allows
354        # modding built-in stuff on platforms where there is no write
355        # access to said built-in stuff.
356        check_dir = Path(
357            user_python_dir,
358            'sys',
359            f'{TARGET_BALLISTICA_VERSION}_{TARGET_BALLISTICA_BUILD}',
360        )
361        try:
362            if check_dir.is_dir():
363                app_python_dir = str(check_dir)
364                is_user_app_python_dir = True
365        except PermissionError:
366            logging.warning(
367                "PermissionError checking user-app-python-dir path '%s'.",
368                check_dir,
369            )
370
371        # Ok, now apply these to sys.path.
372
373        # First off, strip out any instances of the path containing this
374        # module. We will *probably* be re-adding the same path in a
375        # moment so this keeps things cleaner. Though hmm should we
376        # leave it in there in cases where we *don't* re-add the same
377        # path?...
378        our_parent_path = Path(__file__).parent.resolve()
379        oldpaths: list[str] = [
380            p for p in sys.path if Path(p).resolve() != our_parent_path
381        ]
382
383        # Let's place mods first (so users can override whatever they
384        # want) followed by our app scripts and lastly our bundled site
385        # stuff.
386
387        # One could make the argument that at least our bundled app &
388        # site stuff should be placed at the end so actual local site
389        # stuff could override it. That could be a good thing or a bad
390        # thing. Maybe we could add an option for that, but for now I'm
391        # prioritizing our stuff to give as consistent an environment as
392        # possible.
393        ourpaths = [user_python_dir, app_python_dir, site_python_dir]
394
395        # Special case: our modular builds will have a 'python-dylib'
396        # dir alongside the 'python' scripts dir which contains our
397        # binary Python modules. If we see that, add it to the path also.
398        # Not sure if we'd ever have a need to customize this path.
399        dylibdir = f'{app_python_dir}-dylib'
400        if os.path.exists(dylibdir):
401            ourpaths.append(dylibdir)
402
403        sys.path = ourpaths + oldpaths
404
405    return (
406        user_python_dir,
407        app_python_dir,
408        site_python_dir,
409        data_dir,
410        config_dir,
411        standard_app_python_dir,
412        is_user_app_python_dir,
413    )
414
415
416def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
417    create_dirs: list[tuple[str, str | None]] = [
418        ('config', config_dir),
419        ('user_python', user_python_dir),
420    ]
421    for cdirname, cdir in create_dirs:
422        if cdir is not None:
423            try:
424                os.makedirs(cdir, exist_ok=True)
425            except Exception:
426                # Not the end of the world if we can't make these dirs.
427                logging.warning(
428                    "Unable to create %s dir at '%s'.", cdirname, cdir
429                )
430
431
432def extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
433    """Given a list of args and an arg name, returns a value.
434
435    The arg flag and value are removed from the arg list. We also check
436    to make sure the path exists.
437
438    raises CleanErrors on any problems.
439    """
440    from efro.error import CleanError
441
442    count = sum(args.count(n) for n in names)
443    if not count:
444        return None
445
446    if count > 1:
447        raise CleanError(f'Arg {names} passed multiple times.')
448
449    for name in names:
450        if name not in args:
451            continue
452        argindex = args.index(name)
453        if argindex + 1 >= len(args):
454            raise CleanError(f'No value passed after {name} arg.')
455
456        val = args[argindex + 1]
457        del args[argindex : argindex + 2]
458
459        if is_dir and not os.path.isdir(val):
460            namepretty = names[0].removeprefix('--')
461            raise CleanError(
462                f"Provided {namepretty} path '{val}' is not a directory."
463            )
464        return val
465
466    raise RuntimeError(f'Expected arg name not found from {names}')
467
468
469def _modular_main() -> None:
470    from efro.error import CleanError
471
472    # Fundamentally, running a Ballistica app consists of the following:
473    # import baenv; baenv.configure(); import babase; babase.app.run()
474    #
475    # First baenv sets up things like Python paths the way the engine
476    # needs them, and then we import and run the engine.
477    #
478    # Below we're doing a slightly fancier version of that. Namely, we
479    # do some processing of command line args to allow overriding of
480    # paths or running explicit commands or whatever else. Our goal is
481    # that this modular form of the app should be basically
482    # indistinguishable from the monolithic form when used from the
483    # command line.
484
485    try:
486        # Take note that we're running via modular-main. The native
487        # layer can key off this to know whether it should apply
488        # sys.argv or not.
489        _EnvGlobals.get().modular_main_called = True
490
491        # Deal with a few key things here ourself before even running
492        # configure.
493
494        # The extract_arg stuff below modifies this so we work with a
495        # copy.
496        args = sys.argv.copy()
497
498        # NOTE: We need to keep these arg long/short arg versions synced
499        # to those in core_config.cc. That code parses these same args
500        # (even if it doesn't handle them in our case) and will complain
501        # if unrecognized args come through.
502
503        # Our -c arg basically mirrors Python's -c arg. If we get that,
504        # simply exec it and return; no engine stuff.
505        command = extract_arg(args, ['--command', '-c'], is_dir=False)
506        if command is not None:
507            exec(command)  # pylint: disable=exec-used
508            return
509
510        config_dir = extract_arg(args, ['--config-dir', '-C'], is_dir=True)
511        data_dir = extract_arg(args, ['--data-dir', '-d'], is_dir=True)
512        mods_dir = extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
513
514        # We run configure() BEFORE importing babase. (part of its job
515        # is to wrangle paths which can affect where babase and
516        # everything else gets loaded from).
517        configure(
518            config_dir=config_dir,
519            data_dir=data_dir,
520            user_python_dir=mods_dir,
521        )
522
523        import babase
524
525        # The engine will have parsed and processed all other args as
526        # part of the above import. If there were errors or args such as
527        # --help which should lead to us immediately returning, do so.
528        code = babase.get_immediate_return_code()
529        if code is not None:
530            sys.exit(code)
531
532        # Aaaand we're off!
533        babase.app.run()
534
535    # Code wanting us to die with a clean error message instead of an
536    # ugly stack trace can raise one of these.
537    except CleanError as clean_exc:
538        clean_exc.pretty_print()
539        sys.exit(1)
540
541
542# Exec'ing this module directly will do a standard app run.
543if __name__ == '__main__':
544    _modular_main()