bacommon.bacloud
Functionality related to the bacloud tool.
1# Released under the MIT License. See LICENSE for details. 2# 3"""Functionality related to the bacloud tool.""" 4 5from __future__ import annotations 6 7from dataclasses import dataclass 8from typing import TYPE_CHECKING, Annotated 9 10from efro.dataclassio import ioprepped, IOAttrs 11 12if TYPE_CHECKING: 13 pass 14 15# Version is sent to the master-server with all commands. Can be incremented 16# if we need to change behavior server-side to go along with client changes. 17BACLOUD_VERSION = 13 18 19 20def asset_file_cache_path(filehash: str) -> str: 21 """Given a sha256 hex file hash, return a storage path.""" 22 23 # We expect a 64 byte hex str with only lowercase letters and 24 # numbers. Note to self: I considered base64 hashes to save space 25 # but then remembered that lots of filesystems out there ignore case 26 # so that would not end well. 27 assert len(filehash) == 64 28 assert filehash.islower() 29 assert filehash.isalnum() 30 31 # Split into a few levels of directories to keep directory listings 32 # and operations reasonable. This will give 256 top level dirs, each 33 # with 256 subdirs. So if we have 65,536 files in our cache then 34 # dirs will average 1 file each. That seems like a reasonable spread 35 # I think. 36 return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}' 37 38 39@ioprepped 40@dataclass 41class RequestData: 42 """Request sent to bacloud server.""" 43 44 command: Annotated[str, IOAttrs('c')] 45 token: Annotated[str | None, IOAttrs('t')] 46 payload: Annotated[dict, IOAttrs('p')] 47 tzoffset: Annotated[float, IOAttrs('z')] 48 isatty: Annotated[bool, IOAttrs('y')] 49 50 51@ioprepped 52@dataclass 53class ResponseData: 54 """Response sent from the bacloud server to the client.""" 55 56 @ioprepped 57 @dataclass 58 class Downloads: 59 """Info about downloads included in a response.""" 60 61 @ioprepped 62 @dataclass 63 class Entry: 64 """Individual download.""" 65 66 path: Annotated[str, IOAttrs('p')] 67 68 #: Args include with this particular request (combined with 69 #: baseargs). 70 args: Annotated[dict[str, str], IOAttrs('a')] 71 72 # TODO: could add a hash here if we want the client to 73 # verify hashes. 74 75 #: If present, will be prepended to all entry paths via os.path.join. 76 basepath: Annotated[str | None, IOAttrs('p')] 77 78 #: Server command that should be called for each download. The 79 #: server command is expected to respond with a downloads_inline 80 #: containing a single 'default' entry. In the future this may 81 #: be expanded to a more streaming-friendly process. 82 cmd: Annotated[str, IOAttrs('c')] 83 84 #: Args that should be included with all download requests. 85 baseargs: Annotated[dict[str, str], IOAttrs('a')] 86 87 #: Everything that should be downloaded. 88 entries: Annotated[list[Entry], IOAttrs('e')] 89 90 #: If present, client should print this message before any other 91 #: response processing (including error handling) occurs. 92 message: Annotated[str | None, IOAttrs('m', store_default=False)] = None 93 94 #: End arg for message print() call. 95 message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n' 96 97 #: If present, client should abort with this error message. 98 error: Annotated[str | None, IOAttrs('e', store_default=False)] = None 99 100 #: How long to wait before proceeding with remaining response (can 101 #: be useful when waiting for server progress in a loop). 102 delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0 103 104 #: If present, a token that should be stored client-side and passed 105 #: with subsequent commands. 106 login: Annotated[str | None, IOAttrs('l', store_default=False)] = None 107 108 #: If True, any existing client-side token should be discarded. 109 logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False 110 111 #: If present, client should generate a manifest of this dir. 112 #: It should be added to end_command args as 'manifest'. 113 dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = ( 114 None 115 ) 116 117 #: If present, client should upload the requested files (arg1) 118 #: individually to a server command (arg2) with provided args (arg3). 119 uploads: Annotated[ 120 tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False) 121 ] = None 122 123 #: If present, a list of pathnames that should be gzipped 124 #: and uploaded to an 'uploads_inline' bytes dict in end_command args. 125 #: This should be limited to relatively small files. 126 uploads_inline: Annotated[ 127 list[str] | None, IOAttrs('uinl', store_default=False) 128 ] = None 129 130 #: If present, file paths that should be deleted on the client. 131 deletes: Annotated[ 132 list[str] | None, IOAttrs('dlt', store_default=False) 133 ] = None 134 135 #: If present, describes files the client should individually 136 #: request from the server if not already present on the client. 137 downloads: Annotated[ 138 Downloads | None, IOAttrs('dl', store_default=False) 139 ] = None 140 141 #: If present, pathnames mapped to gzipped data to 142 #: be written to the client. This should only be used for relatively 143 #: small files as they are all included inline as part of the response. 144 downloads_inline: Annotated[ 145 dict[str, bytes] | None, IOAttrs('dinl', store_default=False) 146 ] = None 147 148 #: If present, all empty dirs under this one should be removed. 149 dir_prune_empty: Annotated[ 150 str | None, IOAttrs('dpe', store_default=False) 151 ] = None 152 153 #: If present, url to display to the user. 154 open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None 155 156 #: If present, a line of input is read and placed into 157 #: end_command args as 'input'. The first value is the prompt printed 158 #: before reading and the second is whether it should be read as a 159 #: password (without echoing to the terminal). 160 input_prompt: Annotated[ 161 tuple[str, bool] | None, IOAttrs('inp', store_default=False) 162 ] = None 163 164 #: If present, a message that should be printed after all other 165 #: response processing is done. 166 end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = ( 167 None 168 ) 169 170 #: End arg for end_message print() call. 171 end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n' 172 173 #: If present, this command is run with these args at the end 174 #: of response processing. 175 end_command: Annotated[ 176 tuple[str, dict] | None, IOAttrs('ec', store_default=False) 177 ] = None
BACLOUD_VERSION =
13
def
asset_file_cache_path(filehash: str) -> str:
21def asset_file_cache_path(filehash: str) -> str: 22 """Given a sha256 hex file hash, return a storage path.""" 23 24 # We expect a 64 byte hex str with only lowercase letters and 25 # numbers. Note to self: I considered base64 hashes to save space 26 # but then remembered that lots of filesystems out there ignore case 27 # so that would not end well. 28 assert len(filehash) == 64 29 assert filehash.islower() 30 assert filehash.isalnum() 31 32 # Split into a few levels of directories to keep directory listings 33 # and operations reasonable. This will give 256 top level dirs, each 34 # with 256 subdirs. So if we have 65,536 files in our cache then 35 # dirs will average 1 file each. That seems like a reasonable spread 36 # I think. 37 return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}'
Given a sha256 hex file hash, return a storage path.
@ioprepped
@dataclass
class
RequestData:
40@ioprepped 41@dataclass 42class RequestData: 43 """Request sent to bacloud server.""" 44 45 command: Annotated[str, IOAttrs('c')] 46 token: Annotated[str | None, IOAttrs('t')] 47 payload: Annotated[dict, IOAttrs('p')] 48 tzoffset: Annotated[float, IOAttrs('z')] 49 isatty: Annotated[bool, IOAttrs('y')]
Request sent to bacloud server.
RequestData( command: Annotated[str, <efro.dataclassio.IOAttrs object>], token: Annotated[str | None, <efro.dataclassio.IOAttrs object>], payload: Annotated[dict, <efro.dataclassio.IOAttrs object>], tzoffset: Annotated[float, <efro.dataclassio.IOAttrs object>], isatty: Annotated[bool, <efro.dataclassio.IOAttrs object>])
command: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1069f35c0>]
token: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f3560>]
payload: Annotated[dict, <efro.dataclassio.IOAttrs object at 0x1069f1f10>]
tzoffset: Annotated[float, <efro.dataclassio.IOAttrs object at 0x1069f2c00>]
isatty: Annotated[bool, <efro.dataclassio.IOAttrs object at 0x1069f1b50>]
@ioprepped
@dataclass
class
ResponseData:
52@ioprepped 53@dataclass 54class ResponseData: 55 """Response sent from the bacloud server to the client.""" 56 57 @ioprepped 58 @dataclass 59 class Downloads: 60 """Info about downloads included in a response.""" 61 62 @ioprepped 63 @dataclass 64 class Entry: 65 """Individual download.""" 66 67 path: Annotated[str, IOAttrs('p')] 68 69 #: Args include with this particular request (combined with 70 #: baseargs). 71 args: Annotated[dict[str, str], IOAttrs('a')] 72 73 # TODO: could add a hash here if we want the client to 74 # verify hashes. 75 76 #: If present, will be prepended to all entry paths via os.path.join. 77 basepath: Annotated[str | None, IOAttrs('p')] 78 79 #: Server command that should be called for each download. The 80 #: server command is expected to respond with a downloads_inline 81 #: containing a single 'default' entry. In the future this may 82 #: be expanded to a more streaming-friendly process. 83 cmd: Annotated[str, IOAttrs('c')] 84 85 #: Args that should be included with all download requests. 86 baseargs: Annotated[dict[str, str], IOAttrs('a')] 87 88 #: Everything that should be downloaded. 89 entries: Annotated[list[Entry], IOAttrs('e')] 90 91 #: If present, client should print this message before any other 92 #: response processing (including error handling) occurs. 93 message: Annotated[str | None, IOAttrs('m', store_default=False)] = None 94 95 #: End arg for message print() call. 96 message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n' 97 98 #: If present, client should abort with this error message. 99 error: Annotated[str | None, IOAttrs('e', store_default=False)] = None 100 101 #: How long to wait before proceeding with remaining response (can 102 #: be useful when waiting for server progress in a loop). 103 delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0 104 105 #: If present, a token that should be stored client-side and passed 106 #: with subsequent commands. 107 login: Annotated[str | None, IOAttrs('l', store_default=False)] = None 108 109 #: If True, any existing client-side token should be discarded. 110 logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False 111 112 #: If present, client should generate a manifest of this dir. 113 #: It should be added to end_command args as 'manifest'. 114 dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = ( 115 None 116 ) 117 118 #: If present, client should upload the requested files (arg1) 119 #: individually to a server command (arg2) with provided args (arg3). 120 uploads: Annotated[ 121 tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False) 122 ] = None 123 124 #: If present, a list of pathnames that should be gzipped 125 #: and uploaded to an 'uploads_inline' bytes dict in end_command args. 126 #: This should be limited to relatively small files. 127 uploads_inline: Annotated[ 128 list[str] | None, IOAttrs('uinl', store_default=False) 129 ] = None 130 131 #: If present, file paths that should be deleted on the client. 132 deletes: Annotated[ 133 list[str] | None, IOAttrs('dlt', store_default=False) 134 ] = None 135 136 #: If present, describes files the client should individually 137 #: request from the server if not already present on the client. 138 downloads: Annotated[ 139 Downloads | None, IOAttrs('dl', store_default=False) 140 ] = None 141 142 #: If present, pathnames mapped to gzipped data to 143 #: be written to the client. This should only be used for relatively 144 #: small files as they are all included inline as part of the response. 145 downloads_inline: Annotated[ 146 dict[str, bytes] | None, IOAttrs('dinl', store_default=False) 147 ] = None 148 149 #: If present, all empty dirs under this one should be removed. 150 dir_prune_empty: Annotated[ 151 str | None, IOAttrs('dpe', store_default=False) 152 ] = None 153 154 #: If present, url to display to the user. 155 open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None 156 157 #: If present, a line of input is read and placed into 158 #: end_command args as 'input'. The first value is the prompt printed 159 #: before reading and the second is whether it should be read as a 160 #: password (without echoing to the terminal). 161 input_prompt: Annotated[ 162 tuple[str, bool] | None, IOAttrs('inp', store_default=False) 163 ] = None 164 165 #: If present, a message that should be printed after all other 166 #: response processing is done. 167 end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = ( 168 None 169 ) 170 171 #: End arg for end_message print() call. 172 end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n' 173 174 #: If present, this command is run with these args at the end 175 #: of response processing. 176 end_command: Annotated[ 177 tuple[str, dict] | None, IOAttrs('ec', store_default=False) 178 ] = None
Response sent from the bacloud server to the client.
ResponseData( message: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, message_end: Annotated[str, <efro.dataclassio.IOAttrs object>] = '\n', error: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, delay_seconds: Annotated[float, <efro.dataclassio.IOAttrs object>] = 0.0, login: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, logout: Annotated[bool, <efro.dataclassio.IOAttrs object>] = False, dir_manifest: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio.IOAttrs object>] = None, uploads_inline: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object>] = None, deletes: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object>] = None, downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio.IOAttrs object>] = None, downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio.IOAttrs object>] = None, dir_prune_empty: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, open_url: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio.IOAttrs object>] = None, end_message: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None, end_message_end: Annotated[str, <efro.dataclassio.IOAttrs object>] = '\n', end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio.IOAttrs object>] = None)
uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio.IOAttrs object at 0x1069f4950>] =
None
uploads_inline: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object at 0x1069f4b00>] =
None
downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio.IOAttrs object at 0x1069f4d70>] =
None
downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio.IOAttrs object at 0x1069f54f0>] =
None
input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio.IOAttrs object at 0x1069f5f70>] =
None
end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio.IOAttrs object at 0x1069f62d0>] =
None
@ioprepped
@dataclass
class
ResponseData.Downloads:
57 @ioprepped 58 @dataclass 59 class Downloads: 60 """Info about downloads included in a response.""" 61 62 @ioprepped 63 @dataclass 64 class Entry: 65 """Individual download.""" 66 67 path: Annotated[str, IOAttrs('p')] 68 69 #: Args include with this particular request (combined with 70 #: baseargs). 71 args: Annotated[dict[str, str], IOAttrs('a')] 72 73 # TODO: could add a hash here if we want the client to 74 # verify hashes. 75 76 #: If present, will be prepended to all entry paths via os.path.join. 77 basepath: Annotated[str | None, IOAttrs('p')] 78 79 #: Server command that should be called for each download. The 80 #: server command is expected to respond with a downloads_inline 81 #: containing a single 'default' entry. In the future this may 82 #: be expanded to a more streaming-friendly process. 83 cmd: Annotated[str, IOAttrs('c')] 84 85 #: Args that should be included with all download requests. 86 baseargs: Annotated[dict[str, str], IOAttrs('a')] 87 88 #: Everything that should be downloaded. 89 entries: Annotated[list[Entry], IOAttrs('e')]
Info about downloads included in a response.
ResponseData.Downloads( basepath: Annotated[str | None, <efro.dataclassio.IOAttrs object>], cmd: Annotated[str, <efro.dataclassio.IOAttrs object>], baseargs: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object>], entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio.IOAttrs object>])
basepath: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x106405b50>]
cmd: Annotated[str, <efro.dataclassio.IOAttrs object at 0x106872390>]
baseargs: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object at 0x106872630>]
entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio.IOAttrs object at 0x106871940>]
@ioprepped
@dataclass
class
ResponseData.Downloads.Entry:
62 @ioprepped 63 @dataclass 64 class Entry: 65 """Individual download.""" 66 67 path: Annotated[str, IOAttrs('p')] 68 69 #: Args include with this particular request (combined with 70 #: baseargs). 71 args: Annotated[dict[str, str], IOAttrs('a')] 72 73 # TODO: could add a hash here if we want the client to 74 # verify hashes.
Individual download.
ResponseData.Downloads.Entry( path: Annotated[str, <efro.dataclassio.IOAttrs object>], args: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object>])
path: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1071975c0>]
args: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object at 0x1071953a0>]