bacommon.bacloud

Functionality related to the bacloud tool.

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Functionality related to the bacloud tool."""
  4
  5from __future__ import annotations
  6
  7from dataclasses import dataclass
  8from typing import TYPE_CHECKING, Annotated
  9
 10from efro.dataclassio import ioprepped, IOAttrs
 11
 12if TYPE_CHECKING:
 13    pass
 14
 15# Version is sent to the master-server with all commands. Can be incremented
 16# if we need to change behavior server-side to go along with client changes.
 17BACLOUD_VERSION = 13
 18
 19
 20def asset_file_cache_path(filehash: str) -> str:
 21    """Given a sha256 hex file hash, return a storage path."""
 22
 23    # We expect a 64 byte hex str with only lowercase letters and
 24    # numbers. Note to self: I considered base64 hashes to save space
 25    # but then remembered that lots of filesystems out there ignore case
 26    # so that would not end well.
 27    assert len(filehash) == 64
 28    assert filehash.islower()
 29    assert filehash.isalnum()
 30
 31    # Split into a few levels of directories to keep directory listings
 32    # and operations reasonable. This will give 256 top level dirs, each
 33    # with 256 subdirs. So if we have 65,536 files in our cache then
 34    # dirs will average 1 file each. That seems like a reasonable spread
 35    # I think.
 36    return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}'
 37
 38
 39@ioprepped
 40@dataclass
 41class RequestData:
 42    """Request sent to bacloud server."""
 43
 44    command: Annotated[str, IOAttrs('c')]
 45    token: Annotated[str | None, IOAttrs('t')]
 46    payload: Annotated[dict, IOAttrs('p')]
 47    tzoffset: Annotated[float, IOAttrs('z')]
 48    isatty: Annotated[bool, IOAttrs('y')]
 49
 50
 51@ioprepped
 52@dataclass
 53class ResponseData:
 54    """Response sent from the bacloud server to the client.
 55
 56    Attributes:
 57      message: If present, client should print this message before any other
 58        response processing (including error handling) occurs.
 59      message_end: end arg for message print() call.
 60      error: If present, client should abort with this error message.
 61      delay_seconds: How long to wait before proceeding with remaining
 62        response (can be useful when waiting for server progress in a loop).
 63      login: If present, a token that should be stored client-side and passed
 64        with subsequent commands.
 65      logout: If True, any existing client-side token should be discarded.
 66      dir_manifest: If present, client should generate a manifest of this dir.
 67        It should be added to end_command args as 'manifest'.
 68      uploads: If present, client should upload the requested files (arg1)
 69        individually to a server command (arg2) with provided args (arg3).
 70      uploads_inline: If present, a list of pathnames that should be gzipped
 71        and uploaded to an 'uploads_inline' bytes dict in end_command args.
 72        This should be limited to relatively small files.
 73      deletes: If present, file paths that should be deleted on the client.
 74      downloads: If present, describes files the client should individually
 75        request from the server if not already present on the client.
 76      downloads_inline: If present, pathnames mapped to gzipped data to
 77        be written to the client. This should only be used for relatively
 78        small files as they are all included inline as part of the response.
 79      dir_prune_empty: If present, all empty dirs under this one should be
 80        removed.
 81      open_url: If present, url to display to the user.
 82      input_prompt: If present, a line of input is read and placed into
 83        end_command args as 'input'. The first value is the prompt printed
 84        before reading and the second is whether it should be read as a
 85        password (without echoing to the terminal).
 86      end_message: If present, a message that should be printed after all other
 87        response processing is done.
 88      end_message_end: end arg for end_message print() call.
 89      end_command: If present, this command is run with these args at the end
 90        of response processing.
 91    """
 92
 93    @ioprepped
 94    @dataclass
 95    class Downloads:
 96        """Info about downloads included in a response."""
 97
 98        @ioprepped
 99        @dataclass
100        class Entry:
101            """Individual download."""
102
103            path: Annotated[str, IOAttrs('p')]
104            # Args include with this particular request (combined with
105            # baseargs).
106            args: Annotated[dict[str, str], IOAttrs('a')]
107            # TODO: could add a hash here if we want the client to
108            # verify hashes.
109
110        # If present, will be prepended to all entry paths via os.path.join.
111        basepath: Annotated[str | None, IOAttrs('p')]
112
113        # Server command that should be called for each download. The
114        # server command is expected to respond with a downloads_inline
115        # containing a single 'default' entry. In the future this may
116        # be expanded to a more streaming-friendly process.
117        cmd: Annotated[str, IOAttrs('c')]
118
119        # Args that should be included with all download requests.
120        baseargs: Annotated[dict[str, str], IOAttrs('a')]
121
122        # Everything that should be downloaded.
123        entries: Annotated[list[Entry], IOAttrs('e')]
124
125    message: Annotated[str | None, IOAttrs('m', store_default=False)] = None
126    message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n'
127    error: Annotated[str | None, IOAttrs('e', store_default=False)] = None
128    delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0
129    login: Annotated[str | None, IOAttrs('l', store_default=False)] = None
130    logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False
131    dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = (
132        None
133    )
134    uploads: Annotated[
135        tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False)
136    ] = None
137    uploads_inline: Annotated[
138        list[str] | None, IOAttrs('uinl', store_default=False)
139    ] = None
140    deletes: Annotated[
141        list[str] | None, IOAttrs('dlt', store_default=False)
142    ] = None
143    downloads: Annotated[
144        Downloads | None, IOAttrs('dl', store_default=False)
145    ] = None
146    downloads_inline: Annotated[
147        dict[str, bytes] | None, IOAttrs('dinl', store_default=False)
148    ] = None
149    dir_prune_empty: Annotated[
150        str | None, IOAttrs('dpe', store_default=False)
151    ] = None
152    open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None
153    input_prompt: Annotated[
154        tuple[str, bool] | None, IOAttrs('inp', store_default=False)
155    ] = None
156    end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = (
157        None
158    )
159    end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n'
160    end_command: Annotated[
161        tuple[str, dict] | None, IOAttrs('ec', store_default=False)
162    ] = None
BACLOUD_VERSION = 13
def asset_file_cache_path(filehash: str) -> str:
21def asset_file_cache_path(filehash: str) -> str:
22    """Given a sha256 hex file hash, return a storage path."""
23
24    # We expect a 64 byte hex str with only lowercase letters and
25    # numbers. Note to self: I considered base64 hashes to save space
26    # but then remembered that lots of filesystems out there ignore case
27    # so that would not end well.
28    assert len(filehash) == 64
29    assert filehash.islower()
30    assert filehash.isalnum()
31
32    # Split into a few levels of directories to keep directory listings
33    # and operations reasonable. This will give 256 top level dirs, each
34    # with 256 subdirs. So if we have 65,536 files in our cache then
35    # dirs will average 1 file each. That seems like a reasonable spread
36    # I think.
37    return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}'

Given a sha256 hex file hash, return a storage path.

@ioprepped
@dataclass
class RequestData:
40@ioprepped
41@dataclass
42class RequestData:
43    """Request sent to bacloud server."""
44
45    command: Annotated[str, IOAttrs('c')]
46    token: Annotated[str | None, IOAttrs('t')]
47    payload: Annotated[dict, IOAttrs('p')]
48    tzoffset: Annotated[float, IOAttrs('z')]
49    isatty: Annotated[bool, IOAttrs('y')]

Request sent to bacloud server.

RequestData( command: Annotated[str, <efro.dataclassio._base.IOAttrs object>], token: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>], payload: Annotated[dict, <efro.dataclassio._base.IOAttrs object>], tzoffset: Annotated[float, <efro.dataclassio._base.IOAttrs object>], isatty: Annotated[bool, <efro.dataclassio._base.IOAttrs object>])
command: Annotated[str, <efro.dataclassio._base.IOAttrs object at 0x105364320>]
token: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x105364fe0>]
payload: Annotated[dict, <efro.dataclassio._base.IOAttrs object at 0x105365130>]
tzoffset: Annotated[float, <efro.dataclassio._base.IOAttrs object at 0x1053647d0>]
isatty: Annotated[bool, <efro.dataclassio._base.IOAttrs object at 0x105366de0>]
@ioprepped
@dataclass
class ResponseData:
 52@ioprepped
 53@dataclass
 54class ResponseData:
 55    """Response sent from the bacloud server to the client.
 56
 57    Attributes:
 58      message: If present, client should print this message before any other
 59        response processing (including error handling) occurs.
 60      message_end: end arg for message print() call.
 61      error: If present, client should abort with this error message.
 62      delay_seconds: How long to wait before proceeding with remaining
 63        response (can be useful when waiting for server progress in a loop).
 64      login: If present, a token that should be stored client-side and passed
 65        with subsequent commands.
 66      logout: If True, any existing client-side token should be discarded.
 67      dir_manifest: If present, client should generate a manifest of this dir.
 68        It should be added to end_command args as 'manifest'.
 69      uploads: If present, client should upload the requested files (arg1)
 70        individually to a server command (arg2) with provided args (arg3).
 71      uploads_inline: If present, a list of pathnames that should be gzipped
 72        and uploaded to an 'uploads_inline' bytes dict in end_command args.
 73        This should be limited to relatively small files.
 74      deletes: If present, file paths that should be deleted on the client.
 75      downloads: If present, describes files the client should individually
 76        request from the server if not already present on the client.
 77      downloads_inline: If present, pathnames mapped to gzipped data to
 78        be written to the client. This should only be used for relatively
 79        small files as they are all included inline as part of the response.
 80      dir_prune_empty: If present, all empty dirs under this one should be
 81        removed.
 82      open_url: If present, url to display to the user.
 83      input_prompt: If present, a line of input is read and placed into
 84        end_command args as 'input'. The first value is the prompt printed
 85        before reading and the second is whether it should be read as a
 86        password (without echoing to the terminal).
 87      end_message: If present, a message that should be printed after all other
 88        response processing is done.
 89      end_message_end: end arg for end_message print() call.
 90      end_command: If present, this command is run with these args at the end
 91        of response processing.
 92    """
 93
 94    @ioprepped
 95    @dataclass
 96    class Downloads:
 97        """Info about downloads included in a response."""
 98
 99        @ioprepped
100        @dataclass
101        class Entry:
102            """Individual download."""
103
104            path: Annotated[str, IOAttrs('p')]
105            # Args include with this particular request (combined with
106            # baseargs).
107            args: Annotated[dict[str, str], IOAttrs('a')]
108            # TODO: could add a hash here if we want the client to
109            # verify hashes.
110
111        # If present, will be prepended to all entry paths via os.path.join.
112        basepath: Annotated[str | None, IOAttrs('p')]
113
114        # Server command that should be called for each download. The
115        # server command is expected to respond with a downloads_inline
116        # containing a single 'default' entry. In the future this may
117        # be expanded to a more streaming-friendly process.
118        cmd: Annotated[str, IOAttrs('c')]
119
120        # Args that should be included with all download requests.
121        baseargs: Annotated[dict[str, str], IOAttrs('a')]
122
123        # Everything that should be downloaded.
124        entries: Annotated[list[Entry], IOAttrs('e')]
125
126    message: Annotated[str | None, IOAttrs('m', store_default=False)] = None
127    message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n'
128    error: Annotated[str | None, IOAttrs('e', store_default=False)] = None
129    delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0
130    login: Annotated[str | None, IOAttrs('l', store_default=False)] = None
131    logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False
132    dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = (
133        None
134    )
135    uploads: Annotated[
136        tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False)
137    ] = None
138    uploads_inline: Annotated[
139        list[str] | None, IOAttrs('uinl', store_default=False)
140    ] = None
141    deletes: Annotated[
142        list[str] | None, IOAttrs('dlt', store_default=False)
143    ] = None
144    downloads: Annotated[
145        Downloads | None, IOAttrs('dl', store_default=False)
146    ] = None
147    downloads_inline: Annotated[
148        dict[str, bytes] | None, IOAttrs('dinl', store_default=False)
149    ] = None
150    dir_prune_empty: Annotated[
151        str | None, IOAttrs('dpe', store_default=False)
152    ] = None
153    open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None
154    input_prompt: Annotated[
155        tuple[str, bool] | None, IOAttrs('inp', store_default=False)
156    ] = None
157    end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = (
158        None
159    )
160    end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n'
161    end_command: Annotated[
162        tuple[str, dict] | None, IOAttrs('ec', store_default=False)
163    ] = None

Response sent from the bacloud server to the client.

Attributes: message: If present, client should print this message before any other response processing (including error handling) occurs. message_end: end arg for message print() call. error: If present, client should abort with this error message. delay_seconds: How long to wait before proceeding with remaining response (can be useful when waiting for server progress in a loop). login: If present, a token that should be stored client-side and passed with subsequent commands. logout: If True, any existing client-side token should be discarded. dir_manifest: If present, client should generate a manifest of this dir. It should be added to end_command args as 'manifest'. uploads: If present, client should upload the requested files (arg1) individually to a server command (arg2) with provided args (arg3). uploads_inline: If present, a list of pathnames that should be gzipped and uploaded to an 'uploads_inline' bytes dict in end_command args. This should be limited to relatively small files. deletes: If present, file paths that should be deleted on the client. downloads: If present, describes files the client should individually request from the server if not already present on the client. downloads_inline: If present, pathnames mapped to gzipped data to be written to the client. This should only be used for relatively small files as they are all included inline as part of the response. dir_prune_empty: If present, all empty dirs under this one should be removed. open_url: If present, url to display to the user. input_prompt: If present, a line of input is read and placed into end_command args as 'input'. The first value is the prompt printed before reading and the second is whether it should be read as a password (without echoing to the terminal). end_message: If present, a message that should be printed after all other response processing is done. end_message_end: end arg for end_message print() call. end_command: If present, this command is run with these args at the end of response processing.

ResponseData( message: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>] = None, message_end: Annotated[str, <efro.dataclassio._base.IOAttrs object>] = '\n', error: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>] = None, delay_seconds: Annotated[float, <efro.dataclassio._base.IOAttrs object>] = 0.0, login: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>] = None, logout: Annotated[bool, <efro.dataclassio._base.IOAttrs object>] = False, dir_manifest: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>] = None, uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio._base.IOAttrs object>] = None, uploads_inline: Annotated[list[str] | None, <efro.dataclassio._base.IOAttrs object>] = None, deletes: Annotated[list[str] | None, <efro.dataclassio._base.IOAttrs object>] = None, downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio._base.IOAttrs object>] = None, downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio._base.IOAttrs object>] = None, dir_prune_empty: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>] = None, open_url: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>] = None, input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio._base.IOAttrs object>] = None, end_message: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>] = None, end_message_end: Annotated[str, <efro.dataclassio._base.IOAttrs object>] = '\n', end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio._base.IOAttrs object>] = None)
message: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x10530c6e0>] = None
message_end: Annotated[str, <efro.dataclassio._base.IOAttrs object at 0x10530c320>] = '\n'
error: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x10530c1d0>] = None
delay_seconds: Annotated[float, <efro.dataclassio._base.IOAttrs object at 0x10551adb0>] = 0.0
login: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x10551b290>] = None
logout: Annotated[bool, <efro.dataclassio._base.IOAttrs object at 0x10551a900>] = False
dir_manifest: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x105518f20>] = None
uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio._base.IOAttrs object at 0x10551a360>] = None
uploads_inline: Annotated[list[str] | None, <efro.dataclassio._base.IOAttrs object at 0x105518290>] = None
deletes: Annotated[list[str] | None, <efro.dataclassio._base.IOAttrs object at 0x1055183e0>] = None
downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio._base.IOAttrs object at 0x105518620>] = None
downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio._base.IOAttrs object at 0x1055187d0>] = None
dir_prune_empty: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x105518950>] = None
open_url: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x105518e60>] = None
input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio._base.IOAttrs object at 0x105519010>] = None
end_message: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x105519160>] = None
end_message_end: Annotated[str, <efro.dataclassio._base.IOAttrs object at 0x1055193a0>] = '\n'
end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio._base.IOAttrs object at 0x105519400>] = None
@ioprepped
@dataclass
class ResponseData.Downloads:
 94    @ioprepped
 95    @dataclass
 96    class Downloads:
 97        """Info about downloads included in a response."""
 98
 99        @ioprepped
100        @dataclass
101        class Entry:
102            """Individual download."""
103
104            path: Annotated[str, IOAttrs('p')]
105            # Args include with this particular request (combined with
106            # baseargs).
107            args: Annotated[dict[str, str], IOAttrs('a')]
108            # TODO: could add a hash here if we want the client to
109            # verify hashes.
110
111        # If present, will be prepended to all entry paths via os.path.join.
112        basepath: Annotated[str | None, IOAttrs('p')]
113
114        # Server command that should be called for each download. The
115        # server command is expected to respond with a downloads_inline
116        # containing a single 'default' entry. In the future this may
117        # be expanded to a more streaming-friendly process.
118        cmd: Annotated[str, IOAttrs('c')]
119
120        # Args that should be included with all download requests.
121        baseargs: Annotated[dict[str, str], IOAttrs('a')]
122
123        # Everything that should be downloaded.
124        entries: Annotated[list[Entry], IOAttrs('e')]

Info about downloads included in a response.

ResponseData.Downloads( basepath: Annotated[str | None, <efro.dataclassio._base.IOAttrs object>], cmd: Annotated[str, <efro.dataclassio._base.IOAttrs object>], baseargs: Annotated[dict[str, str], <efro.dataclassio._base.IOAttrs object>], entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio._base.IOAttrs object>])
basepath: Annotated[str | None, <efro.dataclassio._base.IOAttrs object at 0x10553fb90>]
cmd: Annotated[str, <efro.dataclassio._base.IOAttrs object at 0x10553c890>]
baseargs: Annotated[dict[str, str], <efro.dataclassio._base.IOAttrs object at 0x10553cad0>]
entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio._base.IOAttrs object at 0x10553cbf0>]
@ioprepped
@dataclass
class ResponseData.Downloads.Entry:
 99        @ioprepped
100        @dataclass
101        class Entry:
102            """Individual download."""
103
104            path: Annotated[str, IOAttrs('p')]
105            # Args include with this particular request (combined with
106            # baseargs).
107            args: Annotated[dict[str, str], IOAttrs('a')]
108            # TODO: could add a hash here if we want the client to
109            # verify hashes.

Individual download.

ResponseData.Downloads.Entry( path: Annotated[str, <efro.dataclassio._base.IOAttrs object>], args: Annotated[dict[str, str], <efro.dataclassio._base.IOAttrs object>])
path: Annotated[str, <efro.dataclassio._base.IOAttrs object at 0x10553df40>]
args: Annotated[dict[str, str], <efro.dataclassio._base.IOAttrs object at 0x10553e180>]