Edit on GitHub

sqlmesh.utils.aws 

 1import typing as t
 2from urllib.parse import urlparse
 3
 4from sqlmesh.utils.errors import SQLMeshError
 5
 6
 7def validate_s3_uri(
 8    value: str, base: bool = False, error_type: t.Type[Exception] = SQLMeshError
 9) -> str:
10    if not value.startswith("s3://"):
11        raise error_type(f"Location '{value}' must be a s3:// URI")
12
13    if base and not value.endswith("/"):
14        value = value + "/"
15
16    # To avoid HIVE_METASTORE_ERROR: S3 resource path length must be less than or equal to 700.
17    if len(value) > 700:
18        raise error_type(f"Location '{value}' cannot be more than 700 characters")
19
20    return value
21
22
23def parse_s3_uri(s3_uri: str) -> t.Tuple[str, str]:
24    """
25    Given a s3:// URI, parse it into a pair of (bucket, key)
26
27    Note that this could be any URI, including a file key, so we dont add a trailing / unlike validate_s3_base_uri
28    """
29    validate_s3_uri(s3_uri)
30
31    parsed_uri = urlparse(s3_uri)
32
33    bucket = parsed_uri.netloc
34    key = parsed_uri.path
35
36    if key:
37        key = key[1:]  # trim off leading /
38
39    return bucket, key
def validate_s3_uri( value: str, base: bool = False, error_type: Type[Exception] = <class 'sqlmesh.utils.errors.SQLMeshError'>) -> str: 
 8def validate_s3_uri(
 9    value: str, base: bool = False, error_type: t.Type[Exception] = SQLMeshError
10) -> str:
11    if not value.startswith("s3://"):
12        raise error_type(f"Location '{value}' must be a s3:// URI")
13
14    if base and not value.endswith("/"):
15        value = value + "/"
16
17    # To avoid HIVE_METASTORE_ERROR: S3 resource path length must be less than or equal to 700.
18    if len(value) > 700:
19        raise error_type(f"Location '{value}' cannot be more than 700 characters")
20
21    return value
def parse_s3_uri(s3_uri: str) -> Tuple[str, str]: 
24def parse_s3_uri(s3_uri: str) -> t.Tuple[str, str]:
25    """
26    Given a s3:// URI, parse it into a pair of (bucket, key)
27
28    Note that this could be any URI, including a file key, so we dont add a trailing / unlike validate_s3_base_uri
29    """
30    validate_s3_uri(s3_uri)
31
32    parsed_uri = urlparse(s3_uri)
33
34    bucket = parsed_uri.netloc
35    key = parsed_uri.path
36
37    if key:
38        key = key[1:]  # trim off leading /
39
40    return bucket, key

Given a s3:// URI, parse it into a pair of (bucket, key)

Note that this could be any URI, including a file key, so we dont add a trailing / unlike validate_s3_base_uri