bacommon.bacloud     
                        Functionality related to the bacloud tool.
1# Released under the MIT License. See LICENSE for details. 2# 3"""Functionality related to the bacloud tool.""" 4 5from __future__ import annotations 6 7from dataclasses import dataclass 8from typing import TYPE_CHECKING, Annotated 9 10from efro.dataclassio import ioprepped, IOAttrs 11 12if TYPE_CHECKING: 13 pass 14 15# Version is sent to the master-server with all commands. Can be incremented 16# if we need to change behavior server-side to go along with client changes. 17BACLOUD_VERSION = 13 18 19 20def asset_file_cache_path(filehash: str) -> str: 21 """Given a sha256 hex file hash, return a storage path.""" 22 23 # We expect a 64 byte hex str with only lowercase letters and 24 # numbers. Note to self: I considered base64 hashes to save space 25 # but then remembered that lots of filesystems out there ignore case 26 # so that would not end well. 27 assert len(filehash) == 64 28 assert filehash.islower() 29 assert filehash.isalnum() 30 31 # Split into a few levels of directories to keep directory listings 32 # and operations reasonable. This will give 256 top level dirs, each 33 # with 256 subdirs. So if we have 65,536 files in our cache then 34 # dirs will average 1 file each. That seems like a reasonable spread 35 # I think. 36 return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}' 37 38 39@ioprepped 40@dataclass 41class RequestData: 42 """Request sent to bacloud server.""" 43 44 command: Annotated[str, IOAttrs('c')] 45 token: Annotated[str | None, IOAttrs('t')] 46 payload: Annotated[dict, IOAttrs('p')] 47 tzoffset: Annotated[float, IOAttrs('z')] 48 isatty: Annotated[bool, IOAttrs('y')] 49 50 51@ioprepped 52@dataclass 53class ResponseData: 54 """Response sent from the bacloud server to the client.""" 55 56 @ioprepped 57 @dataclass 58 class Downloads: 59 """Info about downloads included in a response.""" 60 61 @ioprepped 62 @dataclass 63 class Entry: 64 """Individual download.""" 65 66 path: Annotated[str, IOAttrs('p')] 67 68 #: Args include with this particular request (combined with 69 #: baseargs). 70 args: Annotated[dict[str, str], IOAttrs('a')] 71 72 # TODO: could add a hash here if we want the client to 73 # verify hashes. 74 75 #: If present, will be prepended to all entry paths via os.path.join. 76 basepath: Annotated[str | None, IOAttrs('p')] 77 78 #: Server command that should be called for each download. The 79 #: server command is expected to respond with a downloads_inline 80 #: containing a single 'default' entry. In the future this may 81 #: be expanded to a more streaming-friendly process. 82 cmd: Annotated[str, IOAttrs('c')] 83 84 #: Args that should be included with all download requests. 85 baseargs: Annotated[dict[str, str], IOAttrs('a')] 86 87 #: Everything that should be downloaded. 88 entries: Annotated[list[Entry], IOAttrs('e')] 89 90 #: If present, client should print this message before any other 91 #: response processing (including error handling) occurs. 92 message: Annotated[str | None, IOAttrs('m', store_default=False)] = None 93 94 #: End arg for message print() call. 95 message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n' 96 97 #: If present, client should abort with this error message. 98 error: Annotated[str | None, IOAttrs('e', store_default=False)] = None 99 100 #: How long to wait before proceeding with remaining response (can 101 #: be useful when waiting for server progress in a loop). 102 delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0 103 104 #: If present, a token that should be stored client-side and passed 105 #: with subsequent commands. 106 login: Annotated[str | None, IOAttrs('l', store_default=False)] = None 107 108 #: If True, any existing client-side token should be discarded. 109 logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False 110 111 #: If present, client should generate a manifest of this dir. 112 #: It should be added to end_command args as 'manifest'. 113 dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = ( 114 None 115 ) 116 117 #: If present, client should upload the requested files (arg1) 118 #: individually to a server command (arg2) with provided args (arg3). 119 uploads: Annotated[ 120 tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False) 121 ] = None 122 123 #: If present, a list of pathnames that should be gzipped 124 #: and uploaded to an 'uploads_inline' bytes dict in end_command args. 125 #: This should be limited to relatively small files. 126 uploads_inline: Annotated[ 127 list[str] | None, IOAttrs('uinl', store_default=False) 128 ] = None 129 130 #: If present, file paths that should be deleted on the client. 131 deletes: Annotated[ 132 list[str] | None, IOAttrs('dlt', store_default=False) 133 ] = None 134 135 #: If present, describes files the client should individually 136 #: request from the server if not already present on the client. 137 downloads: Annotated[ 138 Downloads | None, IOAttrs('dl', store_default=False) 139 ] = None 140 141 #: If present, pathnames mapped to gzipped data to 142 #: be written to the client. This should only be used for relatively 143 #: small files as they are all included inline as part of the response. 144 downloads_inline: Annotated[ 145 dict[str, bytes] | None, IOAttrs('dinl', store_default=False) 146 ] = None 147 148 #: If present, all empty dirs under this one should be removed. 149 dir_prune_empty: Annotated[ 150 str | None, IOAttrs('dpe', store_default=False) 151 ] = None 152 153 #: If present, url to display to the user. 154 open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None 155 156 #: If present, a line of input is read and placed into 157 #: end_command args as 'input'. The first value is the prompt printed 158 #: before reading and the second is whether it should be read as a 159 #: password (without echoing to the terminal). 160 input_prompt: Annotated[ 161 tuple[str, bool] | None, IOAttrs('inp', store_default=False) 162 ] = None 163 164 #: If present, a message that should be printed after all other 165 #: response processing is done. 166 end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = ( 167 None 168 ) 169 170 #: End arg for end_message print() call. 171 end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n' 172 173 #: If present, this command is run with these args at the end 174 #: of response processing. 175 end_command: Annotated[ 176 tuple[str, dict] | None, IOAttrs('ec', store_default=False) 177 ] = None
            BACLOUD_VERSION        =
13
        
    
    
    
    
                
            
        def
        asset_file_cache_path(filehash: str) -> str:
                
    
    
            21def asset_file_cache_path(filehash: str) -> str: 22 """Given a sha256 hex file hash, return a storage path.""" 23 24 # We expect a 64 byte hex str with only lowercase letters and 25 # numbers. Note to self: I considered base64 hashes to save space 26 # but then remembered that lots of filesystems out there ignore case 27 # so that would not end well. 28 assert len(filehash) == 64 29 assert filehash.islower() 30 assert filehash.isalnum() 31 32 # Split into a few levels of directories to keep directory listings 33 # and operations reasonable. This will give 256 top level dirs, each 34 # with 256 subdirs. So if we have 65,536 files in our cache then 35 # dirs will average 1 file each. That seems like a reasonable spread 36 # I think. 37 return f'{filehash[:2]}/{filehash[2:4]}/{filehash[4:]}'
Given a sha256 hex file hash, return a storage path.
@ioprepped
        @dataclass
    class
    RequestData:
                
    40@ioprepped 41@dataclass 42class RequestData: 43 """Request sent to bacloud server.""" 44 45 command: Annotated[str, IOAttrs('c')] 46 token: Annotated[str | None, IOAttrs('t')] 47 payload: Annotated[dict, IOAttrs('p')] 48 tzoffset: Annotated[float, IOAttrs('z')] 49 isatty: Annotated[bool, IOAttrs('y')]
Request sent to bacloud server.
            
        RequestData(	command: Annotated[str, <efro.dataclassio.IOAttrs object>],	token: Annotated[str | None, <efro.dataclassio.IOAttrs object>],	payload: Annotated[dict, <efro.dataclassio.IOAttrs object>],	tzoffset: Annotated[float, <efro.dataclassio.IOAttrs object>],	isatty: Annotated[bool, <efro.dataclassio.IOAttrs object>])
        
    
    
    
    
                            
            command: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1069f35c0>]
        
    
    
    
    
                            
            token: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x1069f3560>]
        
    
    
    
    
                            
            payload: Annotated[dict, <efro.dataclassio.IOAttrs object at 0x1069f1f10>]
        
    
    
    
    
                            
            tzoffset: Annotated[float, <efro.dataclassio.IOAttrs object at 0x1069f2c00>]
        
    
    
    
    
                            
            isatty: Annotated[bool, <efro.dataclassio.IOAttrs object at 0x1069f1b50>]
        
    
    
    
    
                            @ioprepped
        @dataclass
    class
    ResponseData:
                
    52@ioprepped 53@dataclass 54class ResponseData: 55 """Response sent from the bacloud server to the client.""" 56 57 @ioprepped 58 @dataclass 59 class Downloads: 60 """Info about downloads included in a response.""" 61 62 @ioprepped 63 @dataclass 64 class Entry: 65 """Individual download.""" 66 67 path: Annotated[str, IOAttrs('p')] 68 69 #: Args include with this particular request (combined with 70 #: baseargs). 71 args: Annotated[dict[str, str], IOAttrs('a')] 72 73 # TODO: could add a hash here if we want the client to 74 # verify hashes. 75 76 #: If present, will be prepended to all entry paths via os.path.join. 77 basepath: Annotated[str | None, IOAttrs('p')] 78 79 #: Server command that should be called for each download. The 80 #: server command is expected to respond with a downloads_inline 81 #: containing a single 'default' entry. In the future this may 82 #: be expanded to a more streaming-friendly process. 83 cmd: Annotated[str, IOAttrs('c')] 84 85 #: Args that should be included with all download requests. 86 baseargs: Annotated[dict[str, str], IOAttrs('a')] 87 88 #: Everything that should be downloaded. 89 entries: Annotated[list[Entry], IOAttrs('e')] 90 91 #: If present, client should print this message before any other 92 #: response processing (including error handling) occurs. 93 message: Annotated[str | None, IOAttrs('m', store_default=False)] = None 94 95 #: End arg for message print() call. 96 message_end: Annotated[str, IOAttrs('m_end', store_default=False)] = '\n' 97 98 #: If present, client should abort with this error message. 99 error: Annotated[str | None, IOAttrs('e', store_default=False)] = None 100 101 #: How long to wait before proceeding with remaining response (can 102 #: be useful when waiting for server progress in a loop). 103 delay_seconds: Annotated[float, IOAttrs('d', store_default=False)] = 0.0 104 105 #: If present, a token that should be stored client-side and passed 106 #: with subsequent commands. 107 login: Annotated[str | None, IOAttrs('l', store_default=False)] = None 108 109 #: If True, any existing client-side token should be discarded. 110 logout: Annotated[bool, IOAttrs('lo', store_default=False)] = False 111 112 #: If present, client should generate a manifest of this dir. 113 #: It should be added to end_command args as 'manifest'. 114 dir_manifest: Annotated[str | None, IOAttrs('man', store_default=False)] = ( 115 None 116 ) 117 118 #: If present, client should upload the requested files (arg1) 119 #: individually to a server command (arg2) with provided args (arg3). 120 uploads: Annotated[ 121 tuple[list[str], str, dict] | None, IOAttrs('u', store_default=False) 122 ] = None 123 124 #: If present, a list of pathnames that should be gzipped 125 #: and uploaded to an 'uploads_inline' bytes dict in end_command args. 126 #: This should be limited to relatively small files. 127 uploads_inline: Annotated[ 128 list[str] | None, IOAttrs('uinl', store_default=False) 129 ] = None 130 131 #: If present, file paths that should be deleted on the client. 132 deletes: Annotated[ 133 list[str] | None, IOAttrs('dlt', store_default=False) 134 ] = None 135 136 #: If present, describes files the client should individually 137 #: request from the server if not already present on the client. 138 downloads: Annotated[ 139 Downloads | None, IOAttrs('dl', store_default=False) 140 ] = None 141 142 #: If present, pathnames mapped to gzipped data to 143 #: be written to the client. This should only be used for relatively 144 #: small files as they are all included inline as part of the response. 145 downloads_inline: Annotated[ 146 dict[str, bytes] | None, IOAttrs('dinl', store_default=False) 147 ] = None 148 149 #: If present, all empty dirs under this one should be removed. 150 dir_prune_empty: Annotated[ 151 str | None, IOAttrs('dpe', store_default=False) 152 ] = None 153 154 #: If present, url to display to the user. 155 open_url: Annotated[str | None, IOAttrs('url', store_default=False)] = None 156 157 #: If present, a line of input is read and placed into 158 #: end_command args as 'input'. The first value is the prompt printed 159 #: before reading and the second is whether it should be read as a 160 #: password (without echoing to the terminal). 161 input_prompt: Annotated[ 162 tuple[str, bool] | None, IOAttrs('inp', store_default=False) 163 ] = None 164 165 #: If present, a message that should be printed after all other 166 #: response processing is done. 167 end_message: Annotated[str | None, IOAttrs('em', store_default=False)] = ( 168 None 169 ) 170 171 #: End arg for end_message print() call. 172 end_message_end: Annotated[str, IOAttrs('eme', store_default=False)] = '\n' 173 174 #: If present, this command is run with these args at the end 175 #: of response processing. 176 end_command: Annotated[ 177 tuple[str, dict] | None, IOAttrs('ec', store_default=False) 178 ] = None
Response sent from the bacloud server to the client.
            
        ResponseData(	message: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None,	message_end: Annotated[str, <efro.dataclassio.IOAttrs object>] = '\n',	error: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None,	delay_seconds: Annotated[float, <efro.dataclassio.IOAttrs object>] = 0.0,	login: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None,	logout: Annotated[bool, <efro.dataclassio.IOAttrs object>] = False,	dir_manifest: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None,	uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio.IOAttrs object>] = None,	uploads_inline: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object>] = None,	deletes: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object>] = None,	downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio.IOAttrs object>] = None,	downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio.IOAttrs object>] = None,	dir_prune_empty: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None,	open_url: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None,	input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio.IOAttrs object>] = None,	end_message: Annotated[str | None, <efro.dataclassio.IOAttrs object>] = None,	end_message_end: Annotated[str, <efro.dataclassio.IOAttrs object>] = '\n',	end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio.IOAttrs object>] = None)
        
    
    
    
    
                            
            uploads: Annotated[tuple[list[str], str, dict] | None, <efro.dataclassio.IOAttrs object at 0x1069f4950>]        =
None
        
    
    
    
    
                            
            uploads_inline: Annotated[list[str] | None, <efro.dataclassio.IOAttrs object at 0x1069f4b00>]        =
None
        
    
    
    
    
                            
            downloads: Annotated[ResponseData.Downloads | None, <efro.dataclassio.IOAttrs object at 0x1069f4d70>]        =
None
        
    
    
    
    
                            
            downloads_inline: Annotated[dict[str, bytes] | None, <efro.dataclassio.IOAttrs object at 0x1069f54f0>]        =
None
        
    
    
    
    
                            
            input_prompt: Annotated[tuple[str, bool] | None, <efro.dataclassio.IOAttrs object at 0x1069f5f70>]        =
None
        
    
    
    
    
                            
            end_command: Annotated[tuple[str, dict] | None, <efro.dataclassio.IOAttrs object at 0x1069f62d0>]        =
None
        
    
    
    
    
                            @ioprepped
        @dataclass
    class
    ResponseData.Downloads:
                
    57 @ioprepped 58 @dataclass 59 class Downloads: 60 """Info about downloads included in a response.""" 61 62 @ioprepped 63 @dataclass 64 class Entry: 65 """Individual download.""" 66 67 path: Annotated[str, IOAttrs('p')] 68 69 #: Args include with this particular request (combined with 70 #: baseargs). 71 args: Annotated[dict[str, str], IOAttrs('a')] 72 73 # TODO: could add a hash here if we want the client to 74 # verify hashes. 75 76 #: If present, will be prepended to all entry paths via os.path.join. 77 basepath: Annotated[str | None, IOAttrs('p')] 78 79 #: Server command that should be called for each download. The 80 #: server command is expected to respond with a downloads_inline 81 #: containing a single 'default' entry. In the future this may 82 #: be expanded to a more streaming-friendly process. 83 cmd: Annotated[str, IOAttrs('c')] 84 85 #: Args that should be included with all download requests. 86 baseargs: Annotated[dict[str, str], IOAttrs('a')] 87 88 #: Everything that should be downloaded. 89 entries: Annotated[list[Entry], IOAttrs('e')]
Info about downloads included in a response.
            
        ResponseData.Downloads(	basepath: Annotated[str | None, <efro.dataclassio.IOAttrs object>],	cmd: Annotated[str, <efro.dataclassio.IOAttrs object>],	baseargs: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object>],	entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio.IOAttrs object>])
        
    
    
    
    
                            
            basepath: Annotated[str | None, <efro.dataclassio.IOAttrs object at 0x106405b50>]
        
    
    
    
    
                            
            cmd: Annotated[str, <efro.dataclassio.IOAttrs object at 0x106872390>]
        
    
    
    
    
                            
            baseargs: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object at 0x106872630>]
        
    
    
    
    
                            
            entries: Annotated[list[ResponseData.Downloads.Entry], <efro.dataclassio.IOAttrs object at 0x106871940>]
        
    
    
    
    
                            @ioprepped
        @dataclass
    class
    ResponseData.Downloads.Entry:
                
    62 @ioprepped 63 @dataclass 64 class Entry: 65 """Individual download.""" 66 67 path: Annotated[str, IOAttrs('p')] 68 69 #: Args include with this particular request (combined with 70 #: baseargs). 71 args: Annotated[dict[str, str], IOAttrs('a')] 72 73 # TODO: could add a hash here if we want the client to 74 # verify hashes.
Individual download.
            
        ResponseData.Downloads.Entry(	path: Annotated[str, <efro.dataclassio.IOAttrs object>],	args: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object>])
        
    
    
    
    
                            
            path: Annotated[str, <efro.dataclassio.IOAttrs object at 0x1071975c0>]
        
    
    
    
    
                            
            args: Annotated[dict[str, str], <efro.dataclassio.IOAttrs object at 0x1071953a0>]