bacommon.bacloud

Functionality related to the bacloud tool.

  1# Released under the MIT License. See LICENSE for details.
  2#
  3"""Functionality related to the bacloud tool."""
  4
  5from __future__ import annotations
  6
  7from dataclasses import dataclass
  8from typing import TYPE_CHECKING, Annotated
  9
 10from efro.dataclassio import ioprepped, IOAttrs
 11
 12if TYPE_CHECKING:
 13    pass
 14
 15# Version is sent to the master-server with all commands. Can be incremented
 16# if we need to change behavior server-side to go along with client changes.
 17BACLOUD_VERSION = 13
 18
 19
 20def asset_file_cache_path(filehash: str) -> str:
 21    """Given a sha256 hex file hash, return a storage path."""
 22
 23    # We expect a 64 byte hex str with only lowercase letters and
 24    # numbers. Note to self: I considered base64 hashes to save space
 25    # but then remembered that lots of filesystems out there ignore case
 26    # so that would not end well.
 27    assert len(filehash) == 64
 28    assert filehash.islower()
 29    assert filehash.isalnum()
 30
 31    # Split into a few levels of directories to keep directory listings
 32    # and operations reasonable. This will give 256 top level dirs, each
 33    # with 256 subdirs. So if we have 65,536 files in our cache then
 34    # dirs will average 1 file each. That seems like a reasonable spread
 35    # I think.
 36    return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}'
 37
 38
 39@ioprepped
 40@dataclass
 41class RequestData:
 42    """Request sent to bacloud server."""
 43
 44    command: Annotated[str, IOAttrs('c')]
 45    token: Annotated[str | None, IOAttrs('t')]
 46    payload: Annotated[dict, IOAttrs('p')]
 47    tzoffset: Annotated[float, IOAttrs('z')]
 48    isatty: Annotated[bool, IOAttrs('y')]
 49
 50
 51@ioprepped
 52@dataclass
 53class ResponseData:
 54    """Response sent from the bacloud server to the client."""
 55
 56    @ioprepped
 57    @dataclass
 58    class Downloads:
 59        """Info about downloads included in a response."""
 60
 61        @ioprepped
 62        @dataclass
 63        class Entry:
 64            """Individual download."""
 65
 66            path: Annotated[str, IOAttrs('p')]
 67
 68            #: Args include with this particular request (combined with
 69            #: baseargs).
 70            args: Annotated[dict[str, str], IOAttrs('a')]
 71
 72            # TODO: could add a hash here if we want the client to
 73            # verify hashes.
 74
 75        #: If present, will be prepended to all entry paths via os.path.join.
 76        basepath: Annotated[str | None, IOAttrs('p')]
 77
 78        #: Server command that should be called for each download. The
 79        #: server command is expected to respond with a downloads_inline
 80        #: containing a single 'default' entry. In the future this may
 81        #: be expanded to a more streaming-friendly process.
 82        cmd: Annotated[str, IOAttrs('c')]
 83
 84        #: Args that should be included with all download requests.
 85        baseargs: Annotated[dict[str, str], IOAttrs('a')]
 86
 87        #: Everything that should be downloaded.
 88        entries: Annotated[list[Entry], IOAttrs('e')]
 89
 90    #: If present, client should print this message before any other
 91    #: response processing (including error handling) occurs.
 92    message: Annotated[str | None, IOAttrs('m', store_default=False)] = None
 93
 94    #: End arg for message print() call.
 95    message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n'
 96
 97    #: If present, client should abort with this error message.
 98    error: Annotated[str | None, IOAttrs('e', store_default=False)] = None
 99
100    #: How long to wait before proceeding with remaining response (can
101    #: be useful when waiting for server progress in a loop).
102    delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0
103
104    #: If present, a token that should be stored client-side and passed
105    #: with subsequent commands.
106    login: Annotated[str | None, IOAttrs('l', store_default=False)] = None
107
108    #: If True, any existing client-side token should be discarded.
109    logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False
110
111    #: If present, client should generate a manifest of this dir.
112    #: It should be added to end_command args as 'manifest'.
113    dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = (
114        None
115    )
116
117    #: If present, client should upload the requested files (arg1)
118    #: individually to a server command (arg2) with provided args (arg3).
119    uploads: Annotated[
120        tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False)
121    ] = None
122
123    #: If present, a list of pathnames that should be gzipped
124    #: and uploaded to an 'uploads_inline' bytes dict in end_command args.
125    #: This should be limited to relatively small files.
126    uploads_inline: Annotated[
127        list[str] | None, IOAttrs('uinl', store_default=False)
128    ] = None
129
130    #: If present, file paths that should be deleted on the client.
131    deletes: Annotated[
132        list[str] | None, IOAttrs('dlt', store_default=False)
133    ] = None
134
135    #: If present, describes files the client should individually
136    #: request from the server if not already present on the client.
137    downloads: Annotated[
138        Downloads | None, IOAttrs('dl', store_default=False)
139    ] = None
140
141    #: If present, pathnames mapped to gzipped data to
142    #: be written to the client. This should only be used for relatively
143    #: small files as they are all included inline as part of the response.
144    downloads_inline: Annotated[
145        dict[str, bytes] | None, IOAttrs('dinl', store_default=False)
146    ] = None
147
148    #: If present, all empty dirs under this one should be removed.
149    dir_prune_empty: Annotated[
150        str | None, IOAttrs('dpe', store_default=False)
151    ] = None
152
153    #: If present, url to display to the user.
154    open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None
155
156    #: If present, a line of input is read and placed into
157    #: end_command args as 'input'. The first value is the prompt printed
158    #: before reading and the second is whether it should be read as a
159    #: password (without echoing to the terminal).
160    input_prompt: Annotated[
161        tuple[str, bool] | None, IOAttrs('inp', store_default=False)
162    ] = None
163
164    #: If present, a message that should be printed after all other
165    #: response processing is done.
166    end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = (
167        None
168    )
169
170    #: End arg for end_message print() call.
171    end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n'
172
173    #: If present, this command is run with these args at the end
174    #: of response processing.
175    end_command: Annotated[
176        tuple[str, dict] | None, IOAttrs('ec', store_default=False)
177    ] = None
BACLOUD_VERSION = 13
def asset_file_cache_path(filehash: str) -> str:
21def asset_file_cache_path(filehash: str) -> str:
22    """Given a sha256 hex file hash, return a storage path."""
23
24    # We expect a 64 byte hex str with only lowercase letters and
25    # numbers. Note to self: I considered base64 hashes to save space
26    # but then remembered that lots of filesystems out there ignore case
27    # so that would not end well.
28    assert len(filehash) == 64
29    assert filehash.islower()
30    assert filehash.isalnum()
31
32    # Split into a few levels of directories to keep directory listings
33    # and operations reasonable. This will give 256 top level dirs, each
34    # with 256 subdirs. So if we have 65,536 files in our cache then
35    # dirs will average 1 file each. That seems like a reasonable spread
36    # I think.
37    return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}'

Given a sha256 hex file hash, return a storage path.

@ioprepped
@dataclass
class RequestData:
40@ioprepped
41@dataclass
42class RequestData:
43    """Request sent to bacloud server."""
44
45    command: Annotated[str, IOAttrs('c')]
46    token: Annotated[str | None, IOAttrs('t')]
47    payload: Annotated[dict, IOAttrs('p')]
48    tzoffset: Annotated[float, IOAttrs('z')]
49    isatty: Annotated[bool, IOAttrs('y')]

Request sent to bacloud server.

RequestData( command: Annotated[str, <efro.dataclassio.IOAttrs object>], token: Annotated[str | None, <efro.dataclassio.IOAttrs object>], payload: Annotated[dict, <efro.dataclassio.IOAttrs object>], tzoffset: Annotated[float, <efro.dataclassio.IOAttrs object>], isatty: Annotated[bool, <efro.dataclassio.IOAttrs object>])
command: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1069f35c0>]
token: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f3560>]
payload: Annotated[dict, <efro.dataclassio.IOAttrs object at 0x1069f1f10>]
tzoffset: Annotated[float, <efro.dataclassio.IOAttrs object at 0x1069f2c00>]
isatty: Annotated[bool, <efro.dataclassio.IOAttrs object at 0x1069f1b50>]
@ioprepped
@dataclass
class ResponseData:
 52@ioprepped
 53@dataclass
 54class ResponseData:
 55    """Response sent from the bacloud server to the client."""
 56
 57    @ioprepped
 58    @dataclass
 59    class Downloads:
 60        """Info about downloads included in a response."""
 61
 62        @ioprepped
 63        @dataclass
 64        class Entry:
 65            """Individual download."""
 66
 67            path: Annotated[str, IOAttrs('p')]
 68
 69            #: Args include with this particular request (combined with
 70            #: baseargs).
 71            args: Annotated[dict[str, str], IOAttrs('a')]
 72
 73            # TODO: could add a hash here if we want the client to
 74            # verify hashes.
 75
 76        #: If present, will be prepended to all entry paths via os.path.join.
 77        basepath: Annotated[str | None, IOAttrs('p')]
 78
 79        #: Server command that should be called for each download. The
 80        #: server command is expected to respond with a downloads_inline
 81        #: containing a single 'default' entry. In the future this may
 82        #: be expanded to a more streaming-friendly process.
 83        cmd: Annotated[str, IOAttrs('c')]
 84
 85        #: Args that should be included with all download requests.
 86        baseargs: Annotated[dict[str, str], IOAttrs('a')]
 87
 88        #: Everything that should be downloaded.
 89        entries: Annotated[list[Entry], IOAttrs('e')]
 90
 91    #: If present, client should print this message before any other
 92    #: response processing (including error handling) occurs.
 93    message: Annotated[str | None, IOAttrs('m', store_default=False)] = None
 94
 95    #: End arg for message print() call.
 96    message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n'
 97
 98    #: If present, client should abort with this error message.
 99    error: Annotated[str | None, IOAttrs('e', store_default=False)] = None
100
101    #: How long to wait before proceeding with remaining response (can
102    #: be useful when waiting for server progress in a loop).
103    delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0
104
105    #: If present, a token that should be stored client-side and passed
106    #: with subsequent commands.
107    login: Annotated[str | None, IOAttrs('l', store_default=False)] = None
108
109    #: If True, any existing client-side token should be discarded.
110    logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False
111
112    #: If present, client should generate a manifest of this dir.
113    #: It should be added to end_command args as 'manifest'.
114    dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = (
115        None
116    )
117
118    #: If present, client should upload the requested files (arg1)
119    #: individually to a server command (arg2) with provided args (arg3).
120    uploads: Annotated[
121        tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False)
122    ] = None
123
124    #: If present, a list of pathnames that should be gzipped
125    #: and uploaded to an 'uploads_inline' bytes dict in end_command args.
126    #: This should be limited to relatively small files.
127    uploads_inline: Annotated[
128        list[str] | None, IOAttrs('uinl', store_default=False)
129    ] = None
130
131    #: If present, file paths that should be deleted on the client.
132    deletes: Annotated[
133        list[str] | None, IOAttrs('dlt', store_default=False)
134    ] = None
135
136    #: If present, describes files the client should individually
137    #: request from the server if not already present on the client.
138    downloads: Annotated[
139        Downloads | None, IOAttrs('dl', store_default=False)
140    ] = None
141
142    #: If present, pathnames mapped to gzipped data to
143    #: be written to the client. This should only be used for relatively
144    #: small files as they are all included inline as part of the response.
145    downloads_inline: Annotated[
146        dict[str, bytes] | None, IOAttrs('dinl', store_default=False)
147    ] = None
148
149    #: If present, all empty dirs under this one should be removed.
150    dir_prune_empty: Annotated[
151        str | None, IOAttrs('dpe', store_default=False)
152    ] = None
153
154    #: If present, url to display to the user.
155    open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None
156
157    #: If present, a line of input is read and placed into
158    #: end_command args as 'input'. The first value is the prompt printed
159    #: before reading and the second is whether it should be read as a
160    #: password (without echoing to the terminal).
161    input_prompt: Annotated[
162        tuple[str, bool] | None, IOAttrs('inp', store_default=False)
163    ] = None
164
165    #: If present, a message that should be printed after all other
166    #: response processing is done.
167    end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = (
168        None
169    )
170
171    #: End arg for end_message print() call.
172    end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n'
173
174    #: If present, this command is run with these args at the end
175    #: of response processing.
176    end_command: Annotated[
177        tuple[str, dict] | None, IOAttrs('ec', store_default=False)
178    ] = None

Response sent from the bacloud server to the client.

ResponseData( message: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, message_end: Annotated[str, <efro.dataclassio.IOAttrs object>] = '\n', error: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, delay_seconds: Annotated[float, <efro.dataclassio.IOAttrs object>] = 0.0, login: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, logout: Annotated[bool, <efro.dataclassio.IOAttrs object>] = False, dir_manifest: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio.IOAttrs object>] = None, uploads_inline: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object>] = None, deletes: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object>] = None, downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio.IOAttrs object>] = None, downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio.IOAttrs object>] = None, dir_prune_empty: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, open_url: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio.IOAttrs object>] = None, end_message: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, end_message_end: Annotated[str, <efro.dataclassio.IOAttrs object>] = '\n', end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio.IOAttrs object>] = None)
message: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f41d0>] = None
message_end: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1069f42f0>] = '\n'
error: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f4380>] = None
delay_seconds: Annotated[float, <efro.dataclassio.IOAttrs object at 0x1069f4530>] = 0.0
login: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f4590>] = None
logout: Annotated[bool, <efro.dataclassio.IOAttrs object at 0x1069f4770>] = False
dir_manifest: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f47d0>] = None
uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio.IOAttrs object at 0x1069f4950>] = None
uploads_inline: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object at 0x1069f4b00>] = None
deletes: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object at 0x1069f4c20>] = None
downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio.IOAttrs object at 0x1069f4d70>] = None
downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio.IOAttrs object at 0x1069f54f0>] = None
dir_prune_empty: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f5c70>] = None
open_url: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f5e50>] = None
input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio.IOAttrs object at 0x1069f5f70>] = None
end_message: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f60f0>] = None
end_message_end: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1069f6270>] = '\n'
end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio.IOAttrs object at 0x1069f62d0>] = None
@ioprepped
@dataclass
class ResponseData.Downloads:
57    @ioprepped
58    @dataclass
59    class Downloads:
60        """Info about downloads included in a response."""
61
62        @ioprepped
63        @dataclass
64        class Entry:
65            """Individual download."""
66
67            path: Annotated[str, IOAttrs('p')]
68
69            #: Args include with this particular request (combined with
70            #: baseargs).
71            args: Annotated[dict[str, str], IOAttrs('a')]
72
73            # TODO: could add a hash here if we want the client to
74            # verify hashes.
75
76        #: If present, will be prepended to all entry paths via os.path.join.
77        basepath: Annotated[str | None, IOAttrs('p')]
78
79        #: Server command that should be called for each download. The
80        #: server command is expected to respond with a downloads_inline
81        #: containing a single 'default' entry. In the future this may
82        #: be expanded to a more streaming-friendly process.
83        cmd: Annotated[str, IOAttrs('c')]
84
85        #: Args that should be included with all download requests.
86        baseargs: Annotated[dict[str, str], IOAttrs('a')]
87
88        #: Everything that should be downloaded.
89        entries: Annotated[list[Entry], IOAttrs('e')]

Info about downloads included in a response.

ResponseData.Downloads( basepath: Annotated[str | None, <efro.dataclassio.IOAttrs object>], cmd: Annotated[str, <efro.dataclassio.IOAttrs object>], baseargs: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object>], entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio.IOAttrs object>])
basepath: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x106405b50>]
cmd: Annotated[str, <efro.dataclassio.IOAttrs object at 0x106872390>]
baseargs: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object at 0x106872630>]
entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio.IOAttrs object at 0x106871940>]
@ioprepped
@dataclass
class ResponseData.Downloads.Entry:
62        @ioprepped
63        @dataclass
64        class Entry:
65            """Individual download."""
66
67            path: Annotated[str, IOAttrs('p')]
68
69            #: Args include with this particular request (combined with
70            #: baseargs).
71            args: Annotated[dict[str, str], IOAttrs('a')]
72
73            # TODO: could add a hash here if we want the client to
74            # verify hashes.

Individual download.

ResponseData.Downloads.Entry( path: Annotated[str, <efro.dataclassio.IOAttrs object>], args: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object>])
path: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1071975c0>]
args: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object at 0x1071953a0>]