bacommon.bacloud
Functionality related to the bacloud tool.
1# Released under the MIT License. See LICENSE for details. 2# 3"""Functionality related to the bacloud tool.""" 4 5from __future__ import annotations 6 7from dataclasses import dataclass 8from typing import TYPE_CHECKING, Annotated 9 10from efro.dataclassio import ioprepped, IOAttrs 11 12if TYPE_CHECKING: 13 pass 14 15# Version is sent to the master-server with all commands. Can be incremented 16# if we need to change behavior server-side to go along with client changes. 17BACLOUD_VERSION = 13 18 19 20def asset_file_cache_path(filehash: str) -> str: 21 """Given a sha256 hex file hash, return a storage path.""" 22 23 # We expect a 64 byte hex str with only lowercase letters and 24 # numbers. Note to self: I considered base64 hashes to save space 25 # but then remembered that lots of filesystems out there ignore case 26 # so that would not end well. 27 assert len(filehash) == 64 28 assert filehash.islower() 29 assert filehash.isalnum() 30 31 # Split into a few levels of directories to keep directory listings 32 # and operations reasonable. This will give 256 top level dirs, each 33 # with 256 subdirs. So if we have 65,536 files in our cache then 34 # dirs will average 1 file each. That seems like a reasonable spread 35 # I think. 36 return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}' 37 38 39@ioprepped 40@dataclass 41class RequestData: 42 """Request sent to bacloud server.""" 43 44 command: Annotated[str, IOAttrs('c')] 45 token: Annotated[str | None, IOAttrs('t')] 46 payload: Annotated[dict, IOAttrs('p')] 47 tzoffset: Annotated[float, IOAttrs('z')] 48 isatty: Annotated[bool, IOAttrs('y')] 49 50 51@ioprepped 52@dataclass 53class ResponseData: 54 """Response sent from the bacloud server to the client. 55 56 Attributes: 57 message: If present, client should print this message before any other 58 response processing (including error handling) occurs. 59 message_end: end arg for message print() call. 60 error: If present, client should abort with this error message. 61 delay_seconds: How long to wait before proceeding with remaining 62 response (can be useful when waiting for server progress in a loop). 63 login: If present, a token that should be stored client-side and passed 64 with subsequent commands. 65 logout: If True, any existing client-side token should be discarded. 66 dir_manifest: If present, client should generate a manifest of this dir. 67 It should be added to end_command args as 'manifest'. 68 uploads: If present, client should upload the requested files (arg1) 69 individually to a server command (arg2) with provided args (arg3). 70 uploads_inline: If present, a list of pathnames that should be gzipped 71 and uploaded to an 'uploads_inline' bytes dict in end_command args. 72 This should be limited to relatively small files. 73 deletes: If present, file paths that should be deleted on the client. 74 downloads: If present, describes files the client should individually 75 request from the server if not already present on the client. 76 downloads_inline: If present, pathnames mapped to gzipped data to 77 be written to the client. This should only be used for relatively 78 small files as they are all included inline as part of the response. 79 dir_prune_empty: If present, all empty dirs under this one should be 80 removed. 81 open_url: If present, url to display to the user. 82 input_prompt: If present, a line of input is read and placed into 83 end_command args as 'input'. The first value is the prompt printed 84 before reading and the second is whether it should be read as a 85 password (without echoing to the terminal). 86 end_message: If present, a message that should be printed after all other 87 response processing is done. 88 end_message_end: end arg for end_message print() call. 89 end_command: If present, this command is run with these args at the end 90 of response processing. 91 """ 92 93 @ioprepped 94 @dataclass 95 class Downloads: 96 """Info about downloads included in a response.""" 97 98 @ioprepped 99 @dataclass 100 class Entry: 101 """Individual download.""" 102 103 path: Annotated[str, IOAttrs('p')] 104 # Args include with this particular request (combined with 105 # baseargs). 106 args: Annotated[dict[str, str], IOAttrs('a')] 107 # TODO: could add a hash here if we want the client to 108 # verify hashes. 109 110 # If present, will be prepended to all entry paths via os.path.join. 111 basepath: Annotated[str | None, IOAttrs('p')] 112 113 # Server command that should be called for each download. The 114 # server command is expected to respond with a downloads_inline 115 # containing a single 'default' entry. In the future this may 116 # be expanded to a more streaming-friendly process. 117 cmd: Annotated[str, IOAttrs('c')] 118 119 # Args that should be included with all download requests. 120 baseargs: Annotated[dict[str, str], IOAttrs('a')] 121 122 # Everything that should be downloaded. 123 entries: Annotated[list[Entry], IOAttrs('e')] 124 125 message: Annotated[str | None, IOAttrs('m', store_default=False)] = None 126 message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n' 127 error: Annotated[str | None, IOAttrs('e', store_default=False)] = None 128 delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0 129 login: Annotated[str | None, IOAttrs('l', store_default=False)] = None 130 logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False 131 dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = ( 132 None 133 ) 134 uploads: Annotated[ 135 tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False) 136 ] = None 137 uploads_inline: Annotated[ 138 list[str] | None, IOAttrs('uinl', store_default=False) 139 ] = None 140 deletes: Annotated[ 141 list[str] | None, IOAttrs('dlt', store_default=False) 142 ] = None 143 downloads: Annotated[ 144 Downloads | None, IOAttrs('dl', store_default=False) 145 ] = None 146 downloads_inline: Annotated[ 147 dict[str, bytes] | None, IOAttrs('dinl', store_default=False) 148 ] = None 149 dir_prune_empty: Annotated[ 150 str | None, IOAttrs('dpe', store_default=False) 151 ] = None 152 open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None 153 input_prompt: Annotated[ 154 tuple[str, bool] | None, IOAttrs('inp', store_default=False) 155 ] = None 156 end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = ( 157 None 158 ) 159 end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n' 160 end_command: Annotated[ 161 tuple[str, dict] | None, IOAttrs('ec', store_default=False) 162 ] = None
21def asset_file_cache_path(filehash: str) -> str: 22 """Given a sha256 hex file hash, return a storage path.""" 23 24 # We expect a 64 byte hex str with only lowercase letters and 25 # numbers. Note to self: I considered base64 hashes to save space 26 # but then remembered that lots of filesystems out there ignore case 27 # so that would not end well. 28 assert len(filehash) == 64 29 assert filehash.islower() 30 assert filehash.isalnum() 31 32 # Split into a few levels of directories to keep directory listings 33 # and operations reasonable. This will give 256 top level dirs, each 34 # with 256 subdirs. So if we have 65,536 files in our cache then 35 # dirs will average 1 file each. That seems like a reasonable spread 36 # I think. 37 return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}'
Given a sha256 hex file hash, return a storage path.
40@ioprepped 41@dataclass 42class RequestData: 43 """Request sent to bacloud server.""" 44 45 command: Annotated[str, IOAttrs('c')] 46 token: Annotated[str | None, IOAttrs('t')] 47 payload: Annotated[dict, IOAttrs('p')] 48 tzoffset: Annotated[float, IOAttrs('z')] 49 isatty: Annotated[bool, IOAttrs('y')]
Request sent to bacloud server.
52@ioprepped 53@dataclass 54class ResponseData: 55 """Response sent from the bacloud server to the client. 56 57 Attributes: 58 message: If present, client should print this message before any other 59 response processing (including error handling) occurs. 60 message_end: end arg for message print() call. 61 error: If present, client should abort with this error message. 62 delay_seconds: How long to wait before proceeding with remaining 63 response (can be useful when waiting for server progress in a loop). 64 login: If present, a token that should be stored client-side and passed 65 with subsequent commands. 66 logout: If True, any existing client-side token should be discarded. 67 dir_manifest: If present, client should generate a manifest of this dir. 68 It should be added to end_command args as 'manifest'. 69 uploads: If present, client should upload the requested files (arg1) 70 individually to a server command (arg2) with provided args (arg3). 71 uploads_inline: If present, a list of pathnames that should be gzipped 72 and uploaded to an 'uploads_inline' bytes dict in end_command args. 73 This should be limited to relatively small files. 74 deletes: If present, file paths that should be deleted on the client. 75 downloads: If present, describes files the client should individually 76 request from the server if not already present on the client. 77 downloads_inline: If present, pathnames mapped to gzipped data to 78 be written to the client. This should only be used for relatively 79 small files as they are all included inline as part of the response. 80 dir_prune_empty: If present, all empty dirs under this one should be 81 removed. 82 open_url: If present, url to display to the user. 83 input_prompt: If present, a line of input is read and placed into 84 end_command args as 'input'. The first value is the prompt printed 85 before reading and the second is whether it should be read as a 86 password (without echoing to the terminal). 87 end_message: If present, a message that should be printed after all other 88 response processing is done. 89 end_message_end: end arg for end_message print() call. 90 end_command: If present, this command is run with these args at the end 91 of response processing. 92 """ 93 94 @ioprepped 95 @dataclass 96 class Downloads: 97 """Info about downloads included in a response.""" 98 99 @ioprepped 100 @dataclass 101 class Entry: 102 """Individual download.""" 103 104 path: Annotated[str, IOAttrs('p')] 105 # Args include with this particular request (combined with 106 # baseargs). 107 args: Annotated[dict[str, str], IOAttrs('a')] 108 # TODO: could add a hash here if we want the client to 109 # verify hashes. 110 111 # If present, will be prepended to all entry paths via os.path.join. 112 basepath: Annotated[str | None, IOAttrs('p')] 113 114 # Server command that should be called for each download. The 115 # server command is expected to respond with a downloads_inline 116 # containing a single 'default' entry. In the future this may 117 # be expanded to a more streaming-friendly process. 118 cmd: Annotated[str, IOAttrs('c')] 119 120 # Args that should be included with all download requests. 121 baseargs: Annotated[dict[str, str], IOAttrs('a')] 122 123 # Everything that should be downloaded. 124 entries: Annotated[list[Entry], IOAttrs('e')] 125 126 message: Annotated[str | None, IOAttrs('m', store_default=False)] = None 127 message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n' 128 error: Annotated[str | None, IOAttrs('e', store_default=False)] = None 129 delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0 130 login: Annotated[str | None, IOAttrs('l', store_default=False)] = None 131 logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False 132 dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = ( 133 None 134 ) 135 uploads: Annotated[ 136 tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False) 137 ] = None 138 uploads_inline: Annotated[ 139 list[str] | None, IOAttrs('uinl', store_default=False) 140 ] = None 141 deletes: Annotated[ 142 list[str] | None, IOAttrs('dlt', store_default=False) 143 ] = None 144 downloads: Annotated[ 145 Downloads | None, IOAttrs('dl', store_default=False) 146 ] = None 147 downloads_inline: Annotated[ 148 dict[str, bytes] | None, IOAttrs('dinl', store_default=False) 149 ] = None 150 dir_prune_empty: Annotated[ 151 str | None, IOAttrs('dpe', store_default=False) 152 ] = None 153 open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None 154 input_prompt: Annotated[ 155 tuple[str, bool] | None, IOAttrs('inp', store_default=False) 156 ] = None 157 end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = ( 158 None 159 ) 160 end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n' 161 end_command: Annotated[ 162 tuple[str, dict] | None, IOAttrs('ec', store_default=False) 163 ] = None
Response sent from the bacloud server to the client.
Attributes: message: If present, client should print this message before any other response processing (including error handling) occurs. message_end: end arg for message print() call. error: If present, client should abort with this error message. delay_seconds: How long to wait before proceeding with remaining response (can be useful when waiting for server progress in a loop). login: If present, a token that should be stored client-side and passed with subsequent commands. logout: If True, any existing client-side token should be discarded. dir_manifest: If present, client should generate a manifest of this dir. It should be added to end_command args as 'manifest'. uploads: If present, client should upload the requested files (arg1) individually to a server command (arg2) with provided args (arg3). uploads_inline: If present, a list of pathnames that should be gzipped and uploaded to an 'uploads_inline' bytes dict in end_command args. This should be limited to relatively small files. deletes: If present, file paths that should be deleted on the client. downloads: If present, describes files the client should individually request from the server if not already present on the client. downloads_inline: If present, pathnames mapped to gzipped data to be written to the client. This should only be used for relatively small files as they are all included inline as part of the response. dir_prune_empty: If present, all empty dirs under this one should be removed. open_url: If present, url to display to the user. input_prompt: If present, a line of input is read and placed into end_command args as 'input'. The first value is the prompt printed before reading and the second is whether it should be read as a password (without echoing to the terminal). end_message: If present, a message that should be printed after all other response processing is done. end_message_end: end arg for end_message print() call. end_command: If present, this command is run with these args at the end of response processing.
94 @ioprepped 95 @dataclass 96 class Downloads: 97 """Info about downloads included in a response.""" 98 99 @ioprepped 100 @dataclass 101 class Entry: 102 """Individual download.""" 103 104 path: Annotated[str, IOAttrs('p')] 105 # Args include with this particular request (combined with 106 # baseargs). 107 args: Annotated[dict[str, str], IOAttrs('a')] 108 # TODO: could add a hash here if we want the client to 109 # verify hashes. 110 111 # If present, will be prepended to all entry paths via os.path.join. 112 basepath: Annotated[str | None, IOAttrs('p')] 113 114 # Server command that should be called for each download. The 115 # server command is expected to respond with a downloads_inline 116 # containing a single 'default' entry. In the future this may 117 # be expanded to a more streaming-friendly process. 118 cmd: Annotated[str, IOAttrs('c')] 119 120 # Args that should be included with all download requests. 121 baseargs: Annotated[dict[str, str], IOAttrs('a')] 122 123 # Everything that should be downloaded. 124 entries: Annotated[list[Entry], IOAttrs('e')]
Info about downloads included in a response.
99 @ioprepped 100 @dataclass 101 class Entry: 102 """Individual download.""" 103 104 path: Annotated[str, IOAttrs('p')] 105 # Args include with this particular request (combined with 106 # baseargs). 107 args: Annotated[dict[str, str], IOAttrs('a')] 108 # TODO: could add a hash here if we want the client to 109 # verify hashes.
Individual download.