Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: FileBone(public=True) for public files #1241

Merged
merged 39 commits into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
42c8936
feat: added public flag to filebone and file entity. default: public …
Aug 15, 2024
b703f1b
fix: missing public flag
Aug 20, 2024
74dac89
feat: added servingurls for publicfiles
Aug 20, 2024
466728c
feat: added public bucket support.
Aug 20, 2024
740b3ea
chore: pep8
Aug 20, 2024
2678224
Merge branch 'main' into feat/public-files
akelch Aug 20, 2024
abc5110
Merge branch 'main' into feat/public-files
phorward Aug 20, 2024
0abb466
Merge branch 'develop' into feat/public-files
phorward Aug 27, 2024
8fc1475
Refactoring the new public file repo
phorward Aug 27, 2024
abd1ae7
Merge branch 'develop' into feat/public-files
phorward Aug 27, 2024
68281db
Cleaning up code for working order
phorward Aug 27, 2024
5c65dbd
Minor fixes
phorward Aug 27, 2024
dbc347c
Provide refactored File.serve function
phorward Aug 27, 2024
d8841b9
Ha ha ha
phorward Aug 27, 2024
67b09c7
clean code
phorward Aug 27, 2024
b8ba387
Consequent use of PUBLIC_DLKEY_POSTFIX
phorward Aug 28, 2024
14854d7
Fixed File.read() and some split calls.
phorward Aug 28, 2024
c8f8c21
fix: added public attribute to structure
Aug 28, 2024
610f59c
Always write Exception to log
phorward Aug 29, 2024
32bde87
Renamed PUBLIC_DLKEY_POSTFIX into PUBLIC_DLKEY_SUFFIX
phorward Aug 30, 2024
e19a204
Keep bucket lookups low
phorward Aug 30, 2024
eb4bc77
Fixed missing trailing commas
phorward Sep 2, 2024
6511311
Raise UnprocessableEntity on invalid format parameter
phorward Sep 2, 2024
9aadd06
Provide filename Content-Disposition with quotes
phorward Sep 2, 2024
6d835c0
Eliminated poor error handling with try...except
phorward Sep 2, 2024
9579523
Fixed pep-8 issues and broken suggestion
phorward Sep 2, 2024
d2d8cc5
Move serve-endpoint validiy dicts to File
phorward Sep 2, 2024
e539538
Merge remote-tracking branch 'origin/develop' into feat/public-files
Sep 4, 2024
74ae93e
fix: unlimit image sizes
Sep 4, 2024
935c03f
fix: serve now takes a host and a key parameter
Sep 4, 2024
599adfb
fix: added create_serve_parameters and create_serve_url function
Sep 4, 2024
eab49a1
chore: linting
Sep 4, 2024
716a9ec
Apply suggestions from code review
phorward Sep 18, 2024
ed7bca6
Merge branch 'develop' into feat/public-files
phorward Sep 20, 2024
f254a86
create_internal_serving_url() and Jinja wrapper
phorward Sep 20, 2024
f9d7166
fix: bucket object
Sep 20, 2024
7bffdee
Merge branch 'develop' into feat/public-files
phorward Sep 23, 2024
a94b922
Simplifying `create_internal_serving_url`
phorward Sep 23, 2024
fd1cb6b
Remove unused import for itertools
phorward Sep 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/viur/core/bones/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,5 +314,5 @@ def recreateFileEntryIfNeeded(val):
def structure(self) -> dict:
return super().structure() | {
"valid_mime_types": self.validMimeTypes,
"public": self.public
"public": self.public,
}
163 changes: 103 additions & 60 deletions src/viur/core/modules/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -429,6 +429,7 @@ class FileNodeSkel(TreeSkel):
class File(Tree):
PENDING_POSTFIX = " (pending)"
DOWNLOAD_URL_PREFIX = "/file/download/"
SERVE_URL_PREFIX = "/file/serve"
MAX_FILENAME_LEN = 256

leafSkelCls = FileLeafSkel
Expand Down Expand Up @@ -492,6 +493,53 @@ def hmac_sign(data: t.Any) -> str:
def hmac_verify(data: t.Any, signature: str) -> bool:
return hmac.compare_digest(File.hmac_sign(data.encode("ASCII")), signature)

@staticmethod
def create_serve_parameters(serving_url: str) -> t.Iterable[str] | None:
phorward marked this conversation as resolved.
Show resolved Hide resolved
"""
Splits a serving URL into its components, used by serve function.
:param serving_url: the serving URL to be split
:return: return a tuple of host and key
"""
regex_result = re.match(
r"^https:\/\/(.*?)\.googleusercontent\.com\/(.*?)$",
serving_url
)

if regex_result:
return regex_result.groups()
return regex_result
phorward marked this conversation as resolved.
Show resolved Hide resolved

@staticmethod
def create_serve_url(
serving_url: str,
phorward marked this conversation as resolved.
Show resolved Hide resolved
size: t.Optional[int] = None,
filename: t.Optional[str] = None,
options: str = "",
download: bool = False
) -> str:

serve_url = f"{File.SERVE_URL_PREFIX}"
phorward marked this conversation as resolved.
Show resolved Hide resolved

serving_params = File.create_serve_parameters(serving_url)
if not serving_params:
raise errors.UnprocessableEntity(f"Invalid serving_url {serving_url!r} provided")
phorward marked this conversation as resolved.
Show resolved Hide resolved

serve_url += f"/{'/'.join(serving_params)}"

path_parameters = [f"{x}" for x in [size, filename] if x is not None]
phorward marked this conversation as resolved.
Show resolved Hide resolved
if path_parameters:
serve_url += f"/{'/'.join(path_parameters)}"

query_parameters = [
f"{k}={str(v).lower()}"
for k, v in {"options": options, "download": download}.items()
if bool(v) is not False]

if query_parameters:
serve_url += f"?{'&'.join(query_parameters)}"
phorward marked this conversation as resolved.
Show resolved Hide resolved

return serve_url

@staticmethod
def create_download_url(
dlkey: str,
Expand Down Expand Up @@ -944,68 +992,60 @@ def download(self, blobKey: str, fileName: str = "", download: bool = False, sig

raise errors.Redirect(signedUrl)

SERVE_VALID_OPTIONS = {
"c",
"p",
"fv",
"fh",
"r90",
"r180",
"r270",
"nu",
}
"""
Valid modification option shorts for the serve-function.
This is passed-through to the Google UserContent API, and hast to be supported there.
"""

SERVE_VALID_FORMATS = {
"jpg": "rj",
"jpeg": "rj",
"png": "rp",
"webp": "rw",
}
"""
Valid file-formats to the serve-function.
This is passed-through to the Google UserContent API, and hast to be supported there.
"""

@exposed
def serve(
akelch marked this conversation as resolved.
Show resolved Hide resolved
self,
host: str,
key: str,
size: t.Optional[int] = None,
filename: t.Optional[str] = None, # random string with .ext
filename: t.Optional[str] = None,
options: str = "",
download: bool = False,
):
"""
Requests an image using the serving url to bypass direct Google requests.

:param key: a string with one __ seperator. The first part is the host prefix, the second part is the key
:param size: the target image size, take a look at the VALID_SIZES
:param filename: a random string with an extention, valid extentions are jpg,png,webp
:param options: - seperated options:
:param host: the google host prefix i.e. lh3
:param key: the serving url key
:param size: the target image size
:param filename: a random string with an extention, valid extentions are (defined in File.SERVE_VALID_FORMATS).
:param options: - seperated options (defined in File.SERVE_VALID_OPTIONS).
c - crop
p - face crop
fv - vertrical flip
fh - horizontal flip
rXXX - rotate 90, 180, 270
nu - no upscale
:param download: if value = 1 force download, else not
:return:
"""

VALID_OPTIONS = {
"c",
"p",
"fv",
"fh",
"r90",
"r180",
"r270",
"nu"
}

VALID_SIZES = {
32, 48, 64, 72, 80, 90, 94,
104, 110, 120, 128, 144, 150, 160,
200, 220, 288,
320,
400,
512, 576,
640,
720,
800,
912,
1024, 1152, 1280, 1440, 1600
}
:param download: Serves the content as download (Content-Disposition) or not.

VALID_FMTS = {
"jpg": "rj",
"jpeg": "rj",
"png": "rp",
"webp": "rw",
}

try:
host, value = key.split("__", 1)
except ValueError:
raise errors.BadRequest("Invalid key provided for serving url")
:return: Returns the requested content on success, raises a proper HTTP exception otherwise.
"""

if any(c not in conf.search_valid_chars for c in host):
raise errors.BadRequest("key contains invalid characters")
Expand All @@ -1015,34 +1055,37 @@ def serve(

if filename:
fmt = filename.rsplit(".", 1)[-1].lower()
if fmt in VALID_FMTS:
if fmt in self.SERVE_VALID_FORMATS:
file_fmt = fmt
else:
raise errors.UnprocessableEntity(f"Unsupported filetype {fmt}")

url = f"https://{host}.googleusercontent.com/{value}"
url = f"https://{host}.googleusercontent.com/{key}"

if options and not all(param in VALID_OPTIONS for param in options.split("-")):
if options and not all(param in self.SERVE_VALID_OPTIONS for param in options.split("-")):
raise errors.BadRequest("Invalid options provided")

options += f"-{VALID_FMTS[file_fmt]}"
options += f"-{self.SERVE_VALID_FORMATS[file_fmt]}"

if size and size in VALID_SIZES:
if size:
options = f"s{size}-" + options

url += "=" + options

try:
response = current.request.get().response
response.headers["Content-Type"] = f"image/{file_fmt}"
response.headers["Cache-Control"] = "public, max-age=604800" # 7 Days
if download:
response.headers["Content-Disposition"] = f"attachment; filename={filename}"
else:
response.headers["Content-Disposition"] = f"filename={filename}"
response = current.request.get().response
response.headers["Content-Type"] = f"image/{file_fmt}"
response.headers["Cache-Control"] = "public, max-age=604800" # 7 Days
if download:
response.headers["Content-Disposition"] = f"attachment; filename={filename}"
else:
response.headers["Content-Disposition"] = f"filename={filename}"

return requests.get(url).content
answ = requests.get(url, timeout=20)
if not answ.ok:
logging.error(f"{answ.status_code} {answ.text}")
raise errors.BadRequest("Unable to fetch a file with these parameters")

except Exception as e:
raise errors.BadRequest("Invalid Url")
return answ.content

@exposed
@force_ssl
Expand Down
Loading